def train(self, data, train_arguments):
[cache_prefix, epochs, sample_interval] = train_arguments
#Create a summary file
train_summary_writer = tf.summary.create_file_writer(
path.join('.', 'summaries', 'train'))
# Adversarial ground truths
valid = np.ones((self.batch_size, 1))
fake = -np.ones((self.batch_size, 1))
with train_summary_writer.as_default():
for epoch in tqdm.trange(epochs, desc='Epoch Iterations'):
for _ in range(self.n_critic):
# ---------------------
# Train the Critic
# ---------------------
batch_data = self.get_data_batch(data, self.batch_size)
noise = tf.random.normal((self.batch_size, self.noise_dim))
# Generate a batch of events
gen_data = self.generator(noise)
# Train the Critic
d_loss_real = self.critic.train_on_batch(batch_data, valid)
d_loss_fake = self.critic.train_on_batch(gen_data, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
for l in self.critic.layers:
weights = l.get_weights()
weights = [
np.clip(w, -self.clip_value, self.clip_value)
for w in weights
]
l.set_weights(weights)
# ---------------------
# Train Generator
# ---------------------
noise = tf.random.normal((self.batch_size, self.noise_dim))
# Train the generator (to have the critic label samples as valid)
g_loss = self._model.train_on_batch(noise, valid)
# Plot the progress
print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" %
(epoch, d_loss[0], 100 * d_loss[1], g_loss))
#If at save interval => save generated events
if epoch % sample_interval == 0:
# Test here data generation step
# save model checkpoints
if path.exists('./cache') is False:
os.mkdir('./cache')
model_checkpoint_base_name = './cache/' + cache_prefix + '_{}_model_weights_step_{}.h5'
self.generator.save_weights(
model_checkpoint_base_name.format('generator', epoch))
self.critic.save_weights(
model_checkpoint_base_name.format('critic', epoch))
def get_unique_label(self, lbl_list):
uniq_label = []
for i in tqdm.trange(len(lbl_list)):
lbl_file = lbl_list[i]
lbl = Image.open(lbl_file)
for l in set(lbl.flatten()):
if not l in uniq_label:
uniq_label.append(l)
return uniq_label
def get_coeffs(self, predicts, coefficients, alpha=0.001, iterations=100):
best_coefficients = coefficients.clone()
best_score = self._compute_score_with_coefficients(predicts, coefficients)
for _ in tqdm.trange(iterations):
counter = self._get_labels_distribution(predicts, coefficients)
label = int(torch.argmax(counter).cpu())
coefficients[label] -= alpha
score = self._compute_score_with_coefficients(predicts, coefficients)
if score > best_score:
best_score = score
best_coefficients = coefficients.clone()
return best_coefficients.numpy()
def getEveryVideoCaptureImgDarkChannel(base_dir):
count = 1
with tqdm.trange(len(os.listdir('../data/airport/capture'))) as tbar:
disp_dict = {}
for name in glob('{}/*.jpg'.format(base_dir)):
count += 1
# print(name.split('\\')[1])
m = deHaze(cv2.imread(name) / 255.0) * 255
scipy.misc.imsave(
'../data/airport/defog/defog_{}.jpg'.format(
name.split('/')[4].split('.')[0]), m)
disp_dict.update({'num': count})
tbar.update()
tbar.set_description(desc='num')
tbar.set_postfix(disp_dict)
tbar.refresh()
def get_dark_channel_img(base_dir):
with tqdm.trange(len(os.listdir(base_dir))) as tbar:
count = 1
disp_dict = {}
for name in glob('{}/*'.format(base_dir)):
count = count + 1
# print(name.split('\\')[1])
haze = np.array(Image.open(name))[:, :, 0:3] / 255
dark = calculate_dark(haze)
scipy.misc.imsave(
'../data/airport/dark_channel/darkchannel_{}.jpg'.format(
name.split('/')[4].split('.')[0]), dark)
disp_dict.update({'num': count})
tbar.update()
tbar.set_description(desc='num')
tbar.set_postfix(disp_dict)
tbar.refresh()
def query2(uids, batch_size=400, n_tries=5, timeout=5):
"""
query: verbose, batched, unique uids, retry on failure
"""
uids = np.unique(uids)
all_ok, all_errors = {}, []
for i in tqdm.trange(0, len(uids), batch_size):
_uids = uids[i:i + batch_size]
for _ in range(n_tries):
ok, errors = query(_uids)
if ok is not None:
break
print("query failed, waiting {} seconds until next try...".format(
timeout))
time.sleep(timeout)
if ok is None:
raise RuntimeError(
"query failed {} times, aborting...".format(n_tries))
all_ok.update(ok)
all_errors += errors
return all_ok, all_errors
def processData(data):
import tqdm
filteredData = data
stemmer = PorterStemmer()
stopWords = set(stopwords.words('english'))
#process each review individually
for i in tqdm.trange(len(data)):
#Replace ' hex code with \'
rev = str.replace(''',"'" ,filteredData.review[i])
#Remove tokens not needed
rev = str.replace(r'[^a-zA-Z \']+', '', rev)
#Translate to lower case
#rev = rev.lower()
#Tokenize using NLTK's tokenizer
rev = nltk.word_tokenize(rev)
#Stem the words
rev = [stemmer.stem(word) for word in rev]
#negate
rev = negate(rev)
#Remove stopwords
rev = [word for word in rev if word not in stopWords]
filteredData.review[i] = rev
return filteredData
def get_activations_from_ims(ims,
model,
batch_size=50,
dims=2048,
device=None,
verbose=False,
tqdm_position=None):
"""Calculates the activations of the pool_3 layer for all images.
Params:
-- files : List of image files paths
-- model : Instance of inception model
-- batch_size : Batch size of images for the model to process at once.
Make sure that the number of samples is a multiple of
the batch size, otherwise some samples are ignored. This
behavior is retained to match the original FID score
implementation.
-- dims : Dimensionality of features returned by Inception
-- cuda : If set to True, use GPU
-- verbose : If set to True and parameter out_step is given, the number
of calculated batches is reported.
Returns:
-- A numpy array of dimension (num images, dims) that contains the
activations of the given tensor when feeding inception with the
query tensor.
"""
model.eval()
# if len(ims) % batch_size != 0:
# print(('Warning: number of images is not a multiple of the '
# 'batch size. Some samples are going to be ignored.'))
# if batch_size > len(ims):
# print(('Warning: batch size is bigger than the datasets size. '
# 'Setting batch size to datasets size'))
# batch_size = len(ims)
# print(len(files), batch_size)
n_batches = (len(ims) + batch_size - 1) // batch_size
n_used_imgs = len(ims)
pred_arr = np.empty((n_used_imgs, dims))
if tqdm_position is None or tqdm_position >= 0:
import tqdm
dataloader_tqdm = tqdm.trange(n_batches,
desc='FID ',
position=tqdm_position,
leave=False)
else:
dataloader_tqdm = range(n_batches)
for i in dataloader_tqdm:
start = i * batch_size
end = start + batch_size
if end > len(ims):
end = len(ims)
images = ims[start:end]
if images.shape[1] != 3:
images = images.transpose((0, 3, 1, 2))
images /= 255
batch = torch.from_numpy(images).type(torch.FloatTensor).to(device)
# if cuda:
# batch = batch.cuda()
with torch.no_grad():
pred = model(batch)[0]
# If model output is not scalar, apply global spatial average pooling.
# This happens if you choose a dimensionality not equal 2048.
# print(pred.shape)
if pred.shape[2] != 1 or pred.shape[3] != 1:
pred = adaptive_avg_pool2d(pred, output_size=(1, 1))
pred_arr[start:end] = pred.cpu().data.numpy().reshape(end - start, -1)
if verbose:
print(' done')
return pred_arr
elif a.startswith("rmdir "):
os.rmdir(a[6:])
print("directory removed!")
elif a == "usb-devices":
myblkd = blkinfo.BlkDiskInfo()
all_my_disks = myblkd.get_disks()
for i in all_my_disks:
print(i, "\n")
elif a == "sleep":
while True:
time.sleep(1)
elif a.startswith("sleep "):
time.sleep(a[6:])
elif a.startswith("compile "):
os.system('nuitka3 ' + a[8:])
for i in tqdm.trange(100):
time.sleep(0.01)
elif a == "hardware":
hardware()
elif a.startswith("pwd"):
print(os.getcwd())
elif a == "whoami":
print(getpass.getuser())
elif a.startswith("python"):
os.system("python3.8")
elif a == "myip":
ip()
elif a.startswith("ping "):
while True:
ping(a[5:], verbose=True, count=1)
elif a.startswith("say "):
