else:
net_ = 'error'
interface_name_ = ''
for i_ in net_:
if str(i_['color']) == str(data_['COLOR']) and str(i_['number']) == str(
data_['NUMBER']) and str(i_['address']) == str(data_['ADDRESS']):
interface_name_ = i_['name']
print('Имя интерфейса для которого производится проверка: ' + interface_name_)
print('----------------------------------------------------------')
print('\x1b[1;32;40m' + ' Шаг-6: Проверка по словарю: ' + '\x1b[0m')
# post-запрос на изменение параметров
bar_ = progressbar.ProgressBar(
maxval=(len(dictionary_) + 1),
widgets=[progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage()])
bar_.start()
number_ = 0
table_data_ = [[
'Имя', 'Описание', 'Тип', 'Параметр', 'Ожидаемый', 'Код post',
'Фактический (sysifaces)', 'Фактический (netstatus)', 'Результат'
]]
for i_ in dictionary_:
# Перед проверкой выставляем значение по умолчанию:
urn_ = str(parameters_['post']['sysifaces_interfaces']['urn'])
data_ = parameters_['post']['sysifaces_interfaces']['data'].copy()
req_ = librequests.post(protocol_, host_, port_, urn_, login_, password_,
timeout_, data_)
def detect_write_images(self):
batch_beg = self.args.data_inst.train_test_split
outdir_good = os.path.join(self.args.log_dir_t, 'good')
if not os.path.exists(outdir_good):
os.makedirs(outdir_good)
outdir_bad = os.path.join(self.args.log_dir_t, 'bad')
if not os.path.exists(outdir_bad):
os.makedirs(outdir_bad)
with h5py.File(self.predict_file, 'r') as pred_h5:
fig = mpplot.figure(figsize=(5, 5))
ax = fig.add_axes([0, 0, 1, 1])
poses_pred_h5 = pred_h5['poses']
num_line = poses_pred_h5.shape[0]
print('start writing {} images ...'.format(num_line))
timerbar = progressbar.ProgressBar(maxval=num_line,
widgets=[
progressbar.Percentage(),
' ',
progressbar.Bar(
'=', '[', ']'), ' ',
progressbar.ETA()
]).start()
store_handle = self.store_handle['test']
frame_h5 = store_handle[self.frame_type]
poses_echt_h5 = store_handle['poses']
resce_h5 = store_handle['resce']
index_h5 = store_handle['index']
error_cap = (10, 20)
bad_cnt = 0
good_cnt = 0
for li in np.arange(num_line):
poses_pred = poses_pred_h5[li, ...].reshape(-1, 3)
poses_echt = poses_echt_h5[(batch_beg + li),
...].reshape(-1, 3)
error = np.mean(
np.sqrt(np.sum((poses_pred - poses_echt)**2, axis=1)))
if error_cap[1] < error:
bad_cnt += 1
# # self._draw_image_pose(
# # ax,
# # frame_h5[(batch_beg + li), ...],
# # poses_pred,
# # resce_h5[(batch_beg + li), ...],
# # self.caminfo)
# self._draw_image_pose_compare(
# ax, self.caminfo,
# frame_h5[(batch_beg + li), ...],
# resce_h5[(batch_beg + li), ...],
# poses_pred, poses_echt)
# ax.axis('off')
# mpplot.savefig(os.path.join(
# outdir_bad,
# self.args.data_io.index2imagename(index_h5[li])))
# ax.clear()
if error_cap[0] > error:
good_cnt += 1
# if 0 == (good_cnt % 100):
# # self._draw_image_pose(
# # ax,
# # frame_h5[(batch_beg + li), ...],
# # poses_pred,
# # resce_h5[(batch_beg + li), ...],
# # self.caminfo)
# self._draw_image_pose_compare(
# ax, self.caminfo,
# frame_h5[(batch_beg + li), ...],
# resce_h5[(batch_beg + li), ...],
# poses_pred, poses_echt)
# ax.axis('off')
# mpplot.savefig(os.path.join(
# outdir_good,
# self.args.data_io.index2imagename(index_h5[li])))
# ax.clear()
if 0 == (li % 1000):
self._draw_image_pose_compare(
ax, self.caminfo, frame_h5[(batch_beg + li), ...],
resce_h5[(batch_beg + li),
...], poses_pred, poses_echt)
ax.axis('off')
mpplot.savefig(
os.path.join(
outdir_bad,
self.args.data_io.index2imagename(index_h5[li])))
ax.clear()
timerbar.update(li)
timerbar.finish()
mpplot.close(fig)
self.logger.info(
'{} good detections, {} bad detections for error cap [{}]'.
format(good_cnt, bad_cnt, error_cap))
criterion_yaw = criterion_yaw.cuda()
criterion_d = criterion_d.cuda()
model_g.train()
model_f1.train()
model_f2.train()
print('Will train from epoch %d to epoch %d (exclusively)' %
(start_epoch, args.epochs))
for epoch in range(start_epoch, args.epochs):
widgets = [
'Epoch %d/%d,' % (epoch, args.epochs), ' ',
progressbar.Counter('batch %(value)d/%(max_value)d')
]
bar = progressbar.ProgressBar(widgets=widgets,
max_value=len(train_loader),
redirect_stdout=True)
for ibatch, batch in bar(enumerate(train_loader)):
src_imgs, src_masks, tgt_imgs = batch['S_image'], batch[
'S_mask'], batch['T_image']
src_imgs, src_masks, tgt_imgs = Variable(src_imgs), Variable(
src_masks), Variable(tgt_imgs)
if args.num_of_labelled_target > 0:
tgt_masks, tgt_use_masks = batch[
'T_mask'], batch['T_index'] < args.num_of_labelled_target
tgt_masks, tgt_use_masks = Variable(tgt_masks), Variable(
tgt_use_masks)
if torch.cuda.is_available():
def train(model, optimizer, criterion, dataset, train_params, test_data=None):
max_iter = train_params['max_iter']
seed = train_params['seed']
class_rate = False
#_utils.touch_pytorch(train_params['logdir'])
torch.manual_seed(seed)
loss = np.zeros(max_iter)
if train_params.has_key('class_rate') and train_params['class_rate']:
class_rate_train = np.zeros(max_iter)
class_rate_test = np.zeros(max_iter)
class_rate = True
bar = progressbar.ProgressBar(max_value=max_iter)
for n in range(max_iter): # loop over the dataset multiple times
data_batch = dataset.get_next_batch()
if train_params['cuda']:
data_batch['x'] = data_batch['x'].cuda()
data_batch['y'] = data_batch['y'].cuda()
inputs = Variable(data_batch['x'])
targets = Variable(data_batch['y'])
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(inputs)
l = criterion(outputs, targets)
# print('loss {} {}'.format(n, l.data[0]))
# print('outputs {} targets {}'.format(outputs.data[0], targets.data[0]))
# print('x1 {} x2 {}'.format(model.x1.data[0], model.x2.data[0]))
loss[n] = l.data[0]
if class_rate:
class_rate_train[n] = compute_class_rate(outputs, targets)
if test_data is not None:
class_rate_test[n] = compute_test_class_rate(
test_data, model, train_params['cuda'])
l.backward()
optimizer.step()
if train_params['debug'] and not train_params['cuda']:
if hasattr(model, 'forward_pass_data'):
file_path = os.path.join(train_params['logdir'],
"forward_pass_{}.pkl".format(n))
#_tf_utils.pkl_save(file_path, model.forward_pass_data)
file_path = os.path.join(train_params['logdir'],
"loss_{}.pkl".format(n))
#_tf_utils.pkl_save(file_path, {'loss': loss[n]})
bar.update(n)
print('Finished Training')
res = {}
res['loss'] = loss
if class_rate:
res['class_rate_train'] = class_rate_train
res['class_rate_test'] = class_rate_test
return res
def filter_parallel_onenode(
ct_series, output_img_series, desc, filter_one, **kwds):
"""
* ct_series: an image series for ct scan
* output_img_series: output image series
* desc: description of the filter action
* filter_one: function to filter one image
* kwds: additional kargs for filter_one
"""
import logging; logger = logging.getLogger("mpi")
from mpi4py import MPI
comm = MPI.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()
# start processing at different time
time.sleep(rank*0.2)
# create the output dir
dir = os.path.dirname(output_img_series.filename_template)
if not os.path.exists(dir):
if rank == 0:
os.makedirs(dir)
else:
created = False
for i in range(WAIT_COUNT):
time.sleep(WAIT_SECONDS)
if os.path.exists(dir):
created = True
break
continue
if not created:
raise IOError("Waited %s seconds, %s still not created" % (
WAIT_COUNT*WAIT_SECONDS, dir))
#
prefix = "%s %s:" % (desc, ct_series.name or "")
if rank==0:
logger.info( "Running %s on %s" % (desc, ct_series.name) )
pass
totalN = ct_series.nImages
# number of images to process in this process
N = int(np.ceil(totalN*1. / size))
# start and stop index
start, stop = rank*N, min(totalN, (rank+1)*N)
# progress bar
# for simplicity, we just report the progress at rank 0, which should be a
# good indication of progress of all nodes any way
if rank==0:
bar = progressbar.ProgressBar(
widgets=[
prefix,
progressbar.Percentage(),
progressbar.Bar(),
' [', progressbar.ETA(), '] ',
],
max_value = stop-start-1,
**pb_config
)
for i, angle in enumerate(ct_series.identifiers[start: stop]):
# sys.stderr.write("%s: %s\n" % (prefix, angle))
# skip over existing results
if not output_img_series.exists(angle):
data = ct_series.getData(angle)
output_img_series.putImage(angle, filter_one(data, **kwds))
if rank==0: bar.update(i)
continue
comm.Barrier()
if rank==0: print('\n')
return
curtailment = pd.DataFrame(columns=columns, index=["curtailment"])
widgets = [
pgb.widgets.Percentage(),
" ",
pgb.widgets.SimpleProgress(format="(%s)" %
pgb.widgets.SimpleProgress.DEFAULT_FORMAT),
" ",
pgb.widgets.Bar(),
" ",
pgb.widgets.Timer(),
" ",
pgb.widgets.ETA(),
]
progressbar = pgb.ProgressBar(prefix="Extract results: ",
widgets=widgets,
max_value=len(snakemake.input))
for k, v in progressbar(columns_dict.items()):
ids = v.values()
n = pypsa.Network(k)
investment_results[(*ids, )] = get_investments(n)
gen_capacity_mix[(*ids, )] = get_capacity_mix(n.generators)
sto_capacity_mix[(*ids, )] = get_capacity_mix(n.storage_units)
energy_mix[(*ids, )] = get_energy_mix(n)
line_transmission_capacity[(*ids, )] = get_transmission(n, "lines")
line_transmission_volume[(*ids, )] = get_transmission(n,
"lines",
length=True)
link_transmission_capacity[(*ids, )] = get_transmission(n, "links")
count_params(trainable_variables_rgb) + count_params(trainable_variables_depth)) + \
"\n{} Generate features from training set".format(datetime.now())
sLog(makelog, simpleLog)
tb_train_count = 0
tb_val_count = 0
best_loss_val = 1000
# Loop over number of epochs
num_samples = 0
# Training set
sess.run(training_init_op)
# Progress bar setting
bar = progressbar.ProgressBar(maxval=tr_batches_per_epoch,
widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
bar.start()
train_loss = 0
train_acc = 0
for i in range(tr_batches_per_epoch):
bar.update(i + 1)
tb_train_count += 1
rgb_batch, depth_batch, label_batch = sess.run(next_batch)
num_samples += np.shape(rgb_batch)[0]
feed_dict = {x_rgb: rgb_batch, x_depth: depth_batch, y: label_batch, keep_prob: (1 - do),
training_phase: True, K.learning_phase(): 1}
batch_loss, summary = sess.run([loss, summary_op], feed_dict=feed_dict)
train_loss += batch_loss
train_writer.add_summary(summary, tb_train_count)
def _maybe_convert_sets(target_dir, extracted_data):
extracted_dir = os.path.join(target_dir, extracted_data)
# override existing CSV with normalized one
target_csv_template = os.path.join(
target_dir, ARCHIVE_DIR_NAME,
ARCHIVE_NAME.replace(".tar.gz", "_{}.csv"))
if os.path.isfile(target_csv_template):
return
wav_root_dir = os.path.join(extracted_dir)
all_files = [
"transcripts/train/yaounde/fn_text.txt",
"transcripts/train/ca16_conv/transcripts.txt",
"transcripts/train/ca16_read/conditioned.txt",
"transcripts/dev/niger_west_african_fr/transcripts.txt",
"speech/dev/niger_west_african_fr/niger_wav_file_name_transcript.tsv",
"transcripts/devtest/ca16_read/conditioned.txt",
"transcripts/test/ca16/prompts.txt",
]
transcripts = {}
for tr in all_files:
with open(os.path.join(target_dir, ARCHIVE_DIR_NAME, tr),
"r") as tr_source:
for line in tr_source.readlines():
line = line.strip()
if ".tsv" in tr:
sep = " "
else:
sep = " "
audio = os.path.basename(line.split(sep)[0])
if not (".wav" in audio):
if ".tdf" in audio:
audio = audio.replace(".tdf", ".wav")
else:
audio += ".wav"
transcript = " ".join(line.split(sep)[1:])
transcripts[audio] = transcript
# Get audiofile path and transcript for each sentence in tsv
samples = []
glob_dir = os.path.join(wav_root_dir, "**/*.wav")
for record in glob(glob_dir, recursive=True):
record_file = os.path.basename(record)
if record_file in transcripts:
samples.append((record, transcripts[record_file]))
# Keep track of how many samples are good vs. problematic
counter = get_counter()
num_samples = len(samples)
rows = []
print("Importing WAV files...")
pool = Pool()
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
for i, processed in enumerate(pool.imap_unordered(one_sample, samples),
start=1):
counter += processed[0]
rows += processed[1]
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
with open(target_csv_template.format("train"),
"w",
encoding="utf-8",
newline="") as train_csv_file: # 80%
with open(target_csv_template.format("dev"),
"w",
encoding="utf-8",
newline="") as dev_csv_file: # 10%
with open(target_csv_template.format("test"),
"w",
encoding="utf-8",
newline="") as test_csv_file: # 10%
train_writer = csv.DictWriter(train_csv_file,
fieldnames=FIELDNAMES)
train_writer.writeheader()
dev_writer = csv.DictWriter(dev_csv_file,
fieldnames=FIELDNAMES)
dev_writer.writeheader()
test_writer = csv.DictWriter(test_csv_file,
fieldnames=FIELDNAMES)
test_writer.writeheader()
for i, item in enumerate(rows):
transcript = validate_label(item[2])
if not transcript:
continue
wav_filename = item[0]
i_mod = i % 10
if i_mod == 0:
writer = test_writer
elif i_mod == 1:
writer = dev_writer
else:
writer = train_writer
writer.writerow(
dict(
wav_filename=wav_filename,
wav_filesize=os.path.getsize(wav_filename),
transcript=transcript,
))
imported_samples = get_imported_samples(counter)
assert counter["all"] == num_samples
assert len(rows) == imported_samples
print_import_report(counter, SAMPLE_RATE, MAX_SECS)
transform_to_lfw_format)
# pylint: disable=no-member
from .config import AlignConfig
# pylint: disable=no-member
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.logging.set_verbosity(tf.logging.ERROR)
WIDGETS = ['Aligning Dataset: ', pb.Percentage(), ' ',
pb.Bar(), ' ', pb.ETA()]
TIMER = pb.ProgressBar(widgets=WIDGETS)
NUM_SUCESSFUL = Value(c_int) # defaults to 0
NUM_SUCESSFUL_LOCK = Lock()
NUM_UNSECESSFUL = Value(c_int)
NUM_UNSUCESSFUL_LOCK = Lock()
NUM_IMAGES_TOTAL = Value(c_int)
NUM_IMAGES_TOTAL_LOCK = Lock()
def align_dataset(config_file: str):
"""Aligns an image dataset
"""
config = AlignConfig(config_file)
output_dir = os.path.expanduser(config.output_dir)
os.makedirs(output_dir, exist_ok=True)
#initialize the image preprocessors and the list of RGB channel averages
aap = AspectAwarePreprocessor(width=256, height=256)
itap = ImageToArrayPreprocessor(data_format=None)
R, G, B = [], [], []
#loop over the datasets tuples
for dType, paths, labels, outputPath in datasets:
#create HDF5 writer
print('Building {}...'.format(outputPath))
writer = HDF5DatasetWriter(dims=(len(paths), 256, 256, 3), outputPath=outputPath)
# initialize the progress bar
widgets = ["Building Dataset: ", progressbar.Percentage(), " ",
progressbar.Bar(), " ", progressbar.ETA()]
pbar = progressbar.ProgressBar(maxval=len(paths),
widgets=widgets).start()
#loop over the image paths
for i, (path, label) in enumerate(zip(paths, labels)):
#load the image and preprocess it
image = cv2.imread(path)
image = aap.preprocess(image)
image = itap.preprocess(image)
#compute the mean of each channel in each image in the training set and update the lists
if dType=='train':
g, b, r = cv2.mean(image)[:3]
B.append(b)
G.append(g)
R.append(r)
#add the image and label to the hdf5 writer
def parmap(f,
iterable,
nprocs=multiprocessing.cpu_count(),
show_progress=True,
size=None):
"""
@param f
function to be applied to the items in iterable
@param iterable
...
@param nprocs
number of processes
@param show_progress
True <-> show a progress bar
@param size
number of items in iterable.
If show_progress == True and size is None and iterable is not already a
list, it is converted to a list first. This could be bad for generators!
(If size is not needed right away for the progress bar, all input items
are enqueued before reading the results from the output queue.)
TLDR: If you know it, tell us the size of your iterable.
"""
q_in = multiprocessing.Queue(1)
q_out = multiprocessing.Queue()
progress = None
if show_progress:
if not isinstance(iterable, list):
iterable = list(iterable)
size = len(iterable)
widgets = [
progressbar.Percentage(), ' ',
progressbar.Bar(), ' ',
progressbar.ETA()
]
progress = progressbar.ProgressBar(widgets=widgets, maxval=size)
proc = [
multiprocessing.Process(target=spawn(f), args=(q_in, q_out))
for _ in range(nprocs)
]
for p in proc:
p.daemon = True
p.start()
if progress is not None:
progress.start()
def enqueue():
s = 0
for i, x in enumerate(iterable):
q_in.put((i, x))
s += 1
for _ in range(nprocs):
q_in.put((None, None))
return s
pool = ThreadPool(processes=1)
async_size = pool.apply_async(enqueue)
if size is None:
# this is the old behavior
size = async_size.get()
res = []
progress_value = 0
for _ in range(size):
r = q_out.get()
res.append(r)
# we could: insert sorted, yield all results we have so far
if progress is not None:
progress_value += 1
progress.update(progress_value)
del pool
for p in proc:
p.join()
if progress is not None:
progress.finish()
return [x for _, x in sorted(res)]
def train(self):
#write to file
feat_trix = pickle.load(open(self.school + ".timetable.features",
"rb"))
#prep sizes
num_fts = feat_trix.shape[1]
num_tts = feat_trix.shape[0]
print "Training {0} on {1} features, {2} timetables".format(
self.school, num_fts, num_tts)
#dictionary mapping from coursef
#to tuple: (related course, similarity)
similarities = {}
# For each Course, C1
# For each Timetable TT with Course C1
# For each Course C2 in Timetable TT
# Record that a Timetable has courses C1 and C2
# For each Course C2
# Compute the similarity between C1 and C2
print "EXECUTING FIRST PASS"
bar = progressbar.ProgressBar()
low_data_courses = {}
for c1 in bar(range(num_fts)):
similar = set()
c1_rows = filter(lambda ptt_idx: feat_trix[ptt_idx, c1],
range(num_tts))
for tt in c1_rows:
row = feat_trix[tt].toarray()[0]
similar = similar.union(set(np.where(row > 0)[0].flatten()))
#use len c1_rows < 5 OR len similar < 3
if len(similar) > 0 and (len(c1_rows) < 5 or len(similar) < 3):
low_data_courses[c1] = []
for c2 in similar:
css = self.compute_similarity(feat_trix[:, c1].toarray(),
feat_trix[:, c2].toarray())
if c1 not in similarities:
similarities[c1] = []
similarities[c1].append((c2, css))
if c1 in low_data_courses:
low_data_courses[c1].append((c2, css))
#low data pass
print "EXECUTING SECOND PASS"
bar2 = progressbar.ProgressBar()
for c1, similar_css in bar2(low_data_courses.items()):
agg_sims = {}
norm_factor = {}
for c2, css in similar_css:
if c2 not in agg_sims:
agg_sims[c2] = css
norm_factor[c2] = 1
for c3, css3 in filter(lambda x: x[0] != c2, similarities[c2]):
if c3 not in agg_sims:
agg_sims[c3] = css
norm_factor[c3] = 1
agg_sims[c3] += css3
norm_factor[c3] += 1
similarities[c1] = []
for c in agg_sims:
similarities[c1].append((c, agg_sims[c] / norm_factor[c]))
pickle.dump(similarities, open(self.school + ".recommended.model",
"wb"))
cv2.namedWindow('result', cv2.WINDOW_NORMAL)
cv2.resizeWindow('result', (int(width/3), int(height/3)))
cv2.moveWindow("result", 1000,1000);
images.sort()
images.reverse()
result_img = None
old_img = None
counter = 0
diff = None
# bar = progressbar.progressbar(range(len(images)))
print("Processing", len(images), "images...")
with progressbar.ProgressBar(max_value=len(images)) as bar:
for image in images:
counter += 1
if result_img is None:
result_img = cv2.imread(os.path.join(image_folder, image))
old_img = result_img
else:
new = cv2.imread(os.path.join(image_folder, image))
# new = cv2.GaussianBlur(new, (3,3), 0, 0 )
# result_img[:,:,1] = max(new[:,:,1], result_img[:,:,1])
# result_img[:,:,2] = max(new[:,:,2], result_img[:,:,2])
# diff = cv2.absdiff(new, old_img)
def main():
print("WELCOME TO ZEPHYRUS.")
# choose user to recommend movies to
print("\nChoose user (ID): ", end="")
user = int(input())
# read dataset
movies_data = pandas.read_csv("csv/movies_data.csv")
test_data = pandas.read_csv("csv/test_data.csv")
train_data = pandas.read_csv("csv/train_data.csv")
movie_reviews = pandas.read_csv("csv/movie_reviews.csv")
# initialize our data matrix (0 = unknown rating)
n_users = train_data['user_id'].max()
n_items = movies_data['movie_id'].max()
# generate (user x movie) ratings matrix
print("\nGenerating user x movie ratings matrix...")
ratings = np.full((n_users, n_items), 0)
for row in train_data.itertuples():
user_id = getattr(row, "user_id")
movie = getattr(row, "movie_id")
rating = getattr(row, "rating")
ratings[user_id - 1][movie - 1] = rating
# calculate biases
# print("Calculating biases...")
# global_avg = global_average(ratings)
# print("Global Average: {}".format(global_avg))
# bu = bias_user(ratings, user, global_avg)
# print("Bias User: {}".format(bu))
# bi = []
# print("Calculating biases for every item...")
# with progressbar.ProgressBar(max_value=n_items) as bar:
# for i in range(n_items):
# bi.append(bias_item(ratings, i, global_avg))
# bar.update(i)
# calculate prediction for every item for chosen user
print("\nPredicting ratings with RF_Rec...")
predictions = np.zeros(n_items)
with progressbar.ProgressBar(max_value=n_items) as bar:
for i in range(n_items):
# predictions[i] = baseline(bu, bi[i], global_avg)
predictions[i] = RF_Rec(ratings, user, i)
bar.update(i)
# sorting and getting top rated items
# sorted_predictions = predictions.argsort()[:5]
sorted_predictions = nlargest(10,
enumerate(predictions),
key=lambda x: x[1])
print("Top 10 predictions:")
print(sorted_predictions)
rf_rec_results = []
for prediction in sorted_predictions:
rf_rec_results.append(prediction[0])
print("\nRF_Rec results:")
for movie in rf_rec_results:
title = movies_data['title'][movie]
print("* " + title)
# use these for FBC-Knn
start = timer()
fbc_knn(rf_rec_results, movie_reviews, movies_data)
end = timer()
time_elapsed = end - start
print("\nElapsed time for FBC-Knn: {}".format(time_elapsed))
def extract_data(args):
''' Extracts data from .bag file given. '''
# open the bag file
print('Loading .bag file')
bag = rosbag.Bag(args['bag'], 'r')
# assumption:
# the matching message will have been received within 150 ms of the pivot message
# this is used to narrow the search for the matching message
t_tol = rospy.Duration(secs=0, nsecs=150000000) # 150000000 ns = 150 ms
# start and end time for the data extraction
start_at = rospy.Time(secs=int(bag.get_start_time()) + 1,
nsecs=0) # 1 second offset for the start
end_at = rospy.Time(secs=int(bag.get_end_time()) - 1,
nsecs=0) # -1 second offset for the end
# pose is the slowest topic
pose_bagmsgs = bag.read_messages(topics='pose',
start_time=start_at,
end_time=end_at)
# joy is faster and is out of sync, which is why a tolerance is needed
joy_bagmsgs = bag.read_messages(topics='joy',
start_time=start_at - t_tol,
end_time=end_at + t_tol)
# load the messages into memory
print('Loading messages into memory')
pose_bagmsgs = [p for p in pose_bagmsgs]
joy_bagmsgs = [j for j in joy_bagmsgs]
# last messages saved. this will be used to obtain incremental
# measures and ground truth for joystick measures
last_pose_msg = None
last_joy_msg = None
second_last_pose_msg = None
second_last_joy_msg = None
# setup the progress bar
c = 0
c_max = len(pose_bagmsgs)
bar = progressbar.ProgressBar(max_value=c_max)
# create pandas dataframe for the data
data = pd.DataFrame(columns=[
'dtns', # 'dts', 'dtns'
# 'dx', 'dy', 'dq.w', 'dq.z'
'x',
'y', ####
'vx',
'dvx',
'vz',
'dvz',
'jx',
'djx',
'jy',
'djy', # 'jb', 'djb'
'jx_gt',
'jy_gt' # 'jb_gt'
])
# counter for the joy list
joy_idx = 0
# read messages from the pose topic
print('Processing and saving data')
for pose_bagmsg in pose_bagmsgs:
# get the pose message and it's timestamp
pose_msg = pose_bagmsg.message
pose_t = pose_bagmsg.timestamp # pose_msg.header.stamp # when it was generated/captured
# read from the joy topic
joy_msg = None
min_diff = rospy.Duration(
secs=10, nsecs=0) # starting the minimum with 10 seconds
for joy_bagmsg in joy_bagmsgs[joy_idx:]:
joy_t = joy_bagmsg.timestamp # joy_bagmsg.message.header.stamp
diff_t = abs(pose_t - joy_t)
if diff_t < min_diff or joy_msg is None:
joy_msg = joy_bagmsg.message
min_diff = diff_t
joy_idx += 1
else:
break
# to proceed, we need a last message for pose and joy
if last_pose_msg is not None and last_joy_msg is not None:
# and we also need a second last to get the ground truth
if second_last_pose_msg is not None and second_last_joy_msg is not None:
# extract the relevant data from the messages
row = {}
# datetime
row['datetime'] = datetime.datetime.fromtimestamp(pose_t.secs) \
+ datetime.timedelta(microseconds=pose_t.nsecs//1000)
# time
ti = last_pose_msg.header.stamp - second_last_pose_msg.header.stamp
# row['dts'] = ti.secs
row['dtns'] = ti.nsecs
# position
'''
po = second_last_pose_msg.pose.pose.position
pn = last_pose_msg.pose.pose.position
row['dx'] = pn.x - po.x
row['dy'] = pn.y - po.y
'''
row['x'] = last_pose_msg.pose.pose.position.x
row['y'] = last_pose_msg.pose.pose.position.y
# orientation
'''
qo = second_last_pose_msg.pose.pose.orientation
qn = last_pose_msg.pose.pose.orientation
qo = np.quaternion(qo.w, qo.x, qo.y, qo.z)
qn = np.quaternion(qn.w, qn.x, qn.y, qn.z)
qi = qn/qo
row['dq.w'] = qi.w
row['dq.z'] = qi.z
'''
# velocity
row['vx'] = last_pose_msg.twist.twist.linear.x
row['dvx'] = last_pose_msg.twist.twist.linear.x - second_last_pose_msg.twist.twist.linear.x
row['vz'] = last_pose_msg.twist.twist.angular.z
row['dvz'] = last_pose_msg.twist.twist.angular.z - second_last_pose_msg.twist.twist.angular.z
# joy (data)
row['jx'] = last_joy_msg.axes[0]
row['djx'] = last_joy_msg.axes[0] - second_last_joy_msg.axes[0]
row['jy'] = last_joy_msg.axes[1]
row['djy'] = last_joy_msg.axes[1] - second_last_joy_msg.axes[1]
# row['jb'] = last_joy_msg.buttons[0]
# row['djb'] = last_joy_msg.buttons[0] - second_last_joy_msg.buttons[0]
# joy (ground-truth)
row['jx_gt'] = joy_msg.axes[0]
row['jy_gt'] = joy_msg.axes[1]
# row['jb_gt'] = joy_msg.buttons[0]
# append the row to the data
data = data.append(pd.DataFrame(row, index=[0]),
ignore_index=True,
sort=False)
# save the messages for the next iteration
second_last_pose_msg = last_pose_msg
second_last_joy_msg = last_joy_msg
# save the messages for the next iteration
last_pose_msg = pose_msg
last_joy_msg = joy_msg
# update the progress
c += 1
bar.update(c)
# mark the processing as finished
bar.finish()
# save the data to disk
set_folder = os.path.join('data', args['dest'], args['set'])
if not os.path.isdir(set_folder):
os.makedirs(set_folder)
if args['append']:
data = pd.concat(
[pd.read_csv(os.path.join(set_folder, 'data.csv')), data])
data.to_csv(path_or_buf=os.path.join(set_folder, 'data.csv'),
header=True,
index=False)
# close the bag file
bag.close()
def optimize(loss_function,
model,
optimizer_str,
lr=1e-4,
nb_epochs=1000,
verbose=True,
num_workers=0,
max_norm=1.0,
**kargs):
"""
Providing the schema of the iterative method for optimizing a
differentiable objective function for models that use gradient centric
schemas (a.k.a order 1 optimization)
Parameters:
-----------
* loss_function: function
Loss function of the model
* model: torch object
Actual model to optimize
* optimizer_str: str
Defines the type of optimizer to use. Here are the possible options:
- adadelta
- adagrad
- adam
- adamax
- rmsprop
- sparseadam
- sgd
* lr: float (default=1e-4)
learning rate used in the optimization
* nb_epochs: int (default=1000)
The number of iterations in the optimization
* verbose: bool (default=True)
Whether or not producing detailed logging about the modeling
* max_norm: float (default=1.0)
Max l2 norm for gradient clipping
"""
# Choosing an optimizer
W = model.parameters()
if optimizer_str.lower() == 'adadelta':
optimizer = torch.optim.Adadelta(W, lr=lr)
elif optimizer_str.lower() == 'adagrad':
optimizer = torch.optim.Adagrad(W, lr=lr)
elif optimizer_str.lower() == 'adam':
optimizer = torch.optim.Adam(W, lr=lr)
elif optimizer_str.lower() == 'adamax':
optimizer = torch.optim.Adamax(W, lr=lr)
elif optimizer_str.lower() == 'rmsprop':
optimizer = torch.optim.RMSprop(W, lr=lr)
elif optimizer_str.lower() == 'sparseadam':
optimizer = torch.optim.SparseAdam(W, lr=lr)
elif optimizer_str.lower() == 'sgd':
optimizer = torch.optim.SGD(W, lr=lr)
elif optimizer_str.lower() == 'lbfgs':
optimizer = torch.optim.LBFGS(W, lr=lr)
elif optimizer_str.lower() == 'rprop':
optimizer = torch.optim.Rprop(W, lr=lr)
else:
error = "{} optimizer isn't implemented".format(optimizer_str)
raise NotImplementedError(error)
# Initializing the Progress Bar
loss_values = []
if verbose:
widgets = [
'% Completion: ',
progressbar.Percentage(),
progressbar.Bar('*'), ''
]
bar = progressbar.ProgressBar(maxval=nb_epochs, widgets=widgets)
bar.start()
# Updating the weights at each training epoch
temp_model = None
for epoch in range(nb_epochs):
# Backward pass and optimization
def closure():
optimizer.zero_grad()
loss = loss_function(model, **kargs)
#print(f"loss raw: {loss}")
#loss = loss * 1/1000000
#print(f"loss: {loss}")
loss.backward()
clip_grad_norm_(model.parameters(), max_norm)
return loss
if 'lbfgs' in optimizer_str.lower():
optimizer.step(closure)
else:
optimizer.step()
loss = closure()
loss_value = loss.item()
# Printing error message if the gradient didn't explode
if np.isnan(loss_value) or np.isinf(loss_value):
error = "The gradient exploded... "
error += "You should reduce the learning"
error += "rate (lr) of your optimizer"
if verbose:
widgets[-1] = error
else:
print(error)
break
# Otherwise, printing value of loss function
else:
temp_model = copy.deepcopy(model)
loss_values.append(loss_value)
if verbose:
widgets[-1] = "Loss: {:6.2f}".format(loss_value)
# Updating the progressbar
if verbose:
bar.update(epoch + 1)
# Terminating the progressbar
if verbose:
bar.finish()
# Finilazing the model
if temp_model is not None:
temp_model = temp_model.eval()
model = copy.deepcopy(temp_model)
else:
raise ValueError(error)
return model, loss_values
def main():
parser = argparse.ArgumentParser(description='Interactive input')
parser.add_argument('-n',
default=1,
type=checkSize,
help='Size of a file(default: 1Mb)')
parser.add_argument('--k',
nargs='+',
type=checkTuple,
default=(10, 100),
help='Amount of words in a string(default: (10, 100))')
parser.add_argument('--l',
nargs='+',
type=checkTuple,
default=(3, 10),
help='Length of a word(default: (3, 10))')
parser.add_argument('--name',
default='output',
help='Give a name to output file.')
args = parser.parse_args()
k = tuple(args.k)
rand_for_str = []
for i in range(k[0], k[1] + 1):
rand_for_str.append(i)
checkDimension(k)
l = tuple(args.l)
rand_for_word = []
for i in range(l[0], l[1] + 1):
rand_for_word.append(i)
checkDimension(l)
file_name = args.name
file = open(file_name + ".txt", "w")
alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
n = args.n
length = random.choice(rand_for_word)
string_len = random.choice(rand_for_str)
spaces = 0
size_in_bytes = 0
str_help = 0
bar = progressbar.ProgressBar(maxval=int(1024**2 * n + 1),
widgets=[
'Working...: ',
progressbar.Bar(left='[',
marker='*',
right=']'),
progressbar.Percentage(),
]).start()
start_time = time.time()
while size_in_bytes < int(1024**2 * n):
if size_in_bytes != 0 and spaces != 0 and (spaces % (string_len) == 0):
spaces = 0
file.write('\n')
size_in_bytes += 2
string_len = random.choice(rand_for_str)
file.write(random.choice(alphabet))
size_in_bytes += 1
str_help += 1
if str_help % length == 0:
spaces += 1
file.write(' ')
size_in_bytes += 1
length = random.choice(rand_for_word)
str_help = 0
bar.update(size_in_bytes)
bar.finish()
file.close()
print("Total time complexity", (time.time() - start_time))
def run(template_traces: np.ndarray,
template_keys: np.ndarray,
template_plaintext: np.ndarray,
attack_traces: np.ndarray,
attack_keys: np.ndarray,
attack_plain: np.ndarray,
pooled: bool,
num_points_of_interest: int,
spacing_points_of_interest: int,
subkey: int,
gpu: bool = False,
leakage_model: bool = True,
debug_mode_enabled: bool = False,
feature_select: int = 0) -> np.array:
""" Method used to select correct version of Template Attack.
:param template_traces: the traces to use
:param template_keys: the keys to use
:param template_plaintext: the plaintexts to use
:param attack_traces: the traces to use for attacking
:param attack_keys: the keys to use for attacking
:param attack_plain: the plaintexts to use for attacking
:param pooled: whether to do a pooled attack
:param num_points_of_interest: number of points of interest to use
:param spacing_points_of_interest: spacing between the points of interest
:param subkey: the subkey index to analyze. Must be in the range [0-16]. 16 signals the full key.
:param gpu: whether or not to use gpu for this attack
:param leakage_model: the leakage model to use
:param debug_mode_enabled: whether to enable debug mode
:param feature_select: which feature selection method to use, see main for which number is which.
:return: array containing the calculated key
"""
# Init progress bar with rough amount of iterations
num_subkeys = 16 if subkey == 16 else 1 # Attack takes roughly equal time per subkey so * 16 for whole key
max_value = len(attack_traces) * 256 * num_subkeys
bar = progressbar.ProgressBar(
max_value=max_value,
widgets=progress_bar_util.get_widgets(debug_mode_enabled))
ta = TA(template_traces, template_keys, template_plaintext,
attack_traces, attack_keys, attack_plain)
indices = []
if feature_select > 0:
print("Feature selection is being calculated...")
if num_subkeys == 16:
for i in range(16):
temp = DataPartitioner.select_features(
template_traces, template_keys, template_plaintext, i,
feature_select, num_points_of_interest, 1, 0,
leakage_model)
for j in temp:
indices.append(j)
else:
# Select at least 10 features
num_features = max(num_points_of_interest, 10)
indices = DataPartitioner.select_features(
template_traces, template_keys, template_plaintext, subkey,
feature_select, num_features, 1, 0, leakage_model)
ta.template_traces, ta.attack_traces = template_traces[:,
indices], attack_traces[:,
indices]
if pooled:
result = ta.run_pooled(num_points_of_interest,
spacing_points_of_interest, subkey, bar,
gpu, leakage_model, debug_mode_enabled)
else:
result = ta.run_normal(num_points_of_interest,
spacing_points_of_interest, subkey, bar,
gpu, leakage_model, debug_mode_enabled)
bar.finish()
print('The final key is: ', result)
return result
def create_benchmark_archive(blk_hash):
blk = json.loads(
subprocess.check_output([ZELCASH_CLI, 'getblock', blk_hash]))
print 'Height: %d' % blk['height']
print 'Transactions: %d' % len(blk['tx'])
os.mkdir('benchmark')
with open('benchmark/block-%d.dat' % blk['height'], 'wb') as f:
f.write(
binascii.unhexlify(
subprocess.check_output(
[ZELCASH_CLI, 'getblock', blk_hash, 'false']).strip()))
txs = [
json.loads(
subprocess.check_output(
[ZELCASH_CLI, 'getrawtransaction', tx, '1']))
for tx in blk['tx']
]
js_txs = len([tx for tx in txs if len(tx['vjoinsplit']) > 0])
if js_txs:
print 'Block contains %d JoinSplit-containing transactions' % js_txs
return
inputs = [(x['txid'], x['vout']) for tx in txs for x in tx['vin']
if x.has_key('txid')]
print 'Total inputs: %d' % len(inputs)
unique_inputs = {}
for i in sorted(inputs):
if unique_inputs.has_key(i[0]):
unique_inputs[i[0]].append(i[1])
else:
unique_inputs[i[0]] = [i[1]]
print 'Unique input transactions: %d' % len(unique_inputs)
db_path = 'benchmark/block-%d-inputs' % blk['height']
db = plyvel.DB(db_path, create_if_missing=True)
wb = db.write_batch()
bar = progressbar.ProgressBar(redirect_stdout=True)
print 'Collecting input coins for block'
for tx in bar(unique_inputs.keys()):
rawtx = json.loads(
subprocess.check_output(
[ZELCASH_CLI, 'getrawtransaction', tx, '1']))
mask_size = 0
mask_code = 0
b = 0
while 2 + b * 8 < len(rawtx['vout']):
zero = True
i = 0
while i < 8 and 2 + b * 8 + i < len(rawtx['vout']):
if 2 + b * 8 + i in unique_inputs[tx]:
zero = False
i += 1
if not zero:
mask_size = b + 1
mask_code += 1
b += 1
coinbase = len(rawtx['vin']) == 1 and 'coinbase' in rawtx['vin'][0]
first = len(rawtx['vout']) > 0 and 0 in unique_inputs[tx]
second = len(rawtx['vout']) > 1 and 1 in unique_inputs[tx]
code = 8*(mask_code - (0 if first or second else 1)) + \
(1 if coinbase else 0) + \
(2 if first else 0) + \
(4 if second else 0)
coins = bytearray()
# Serialized format:
# - VARINT(nVersion)
coins.extend(encode_varint(rawtx['version']))
# - VARINT(nCode)
coins.extend(encode_varint(code))
# - unspentness bitvector, for vout[2] and further; least significant byte first
for b in range(mask_size):
avail = 0
i = 0
while i < 8 and 2 + b * 8 + i < len(rawtx['vout']):
if 2 + b * 8 + i in unique_inputs[tx]:
avail |= (1 << i)
i += 1
coins.append(avail)
# - the non-spent CTxOuts (via CTxOutCompressor)
for i in range(len(rawtx['vout'])):
if i in unique_inputs[tx]:
coins.extend(
encode_varint(
compress_amount(int(rawtx['vout'][i]['valueZat']))))
coins.extend(
compress_script(
binascii.unhexlify(
rawtx['vout'][i]['scriptPubKey']['hex'])))
# - VARINT(nHeight)
coins.extend(
encode_varint(
json.loads(
subprocess.check_output(
[ZELCASH_CLI, 'getblockheader',
rawtx['blockhash']]))['height']))
db_key = b'c' + bytes(binascii.unhexlify(tx)[::-1])
db_val = bytes(coins)
wb.put(db_key, db_val)
wb.write()
db.close()
# Make reproducible archive
os.remove('%s/LOG' % db_path)
files = subprocess.check_output(['find', 'benchmark']).strip().split('\n')
archive_name = 'block-%d.tar' % blk['height']
tar = tarfile.open(archive_name, 'w')
for name in sorted(files):
tar.add(name, recursive=False, filter=deterministic_filter)
tar.close()
subprocess.check_call(['xz', '-6', archive_name])
print 'Created archive %s.xz' % archive_name
subprocess.call(['rm', '-r', 'benchmark'])
def new_data() -> None:
"""
Function for generating images and XML markup and storing this data in the directories specified in the config file
:return: None
"""
# array with full parameters for each class
item = []
random.seed()
# getting class names and their number from a file
with open(config.file_with_classnames, 'r') as file:
for line in file:
str_ = line.split(' ')
name = str_[0]
count = str_[1][:-1]
item.append(it.Item(name, count))
# function for converting the source instances of each class to the initial dimensions
change_size(item)
# current class number
index = 0
# creating a variable responsible for the file number
_count_ = 1
print('\n*****Create image and xml for pix: *****')
number_classes = len(item)
# the list of accounting for the number of elements of each class
list_count_of_each_class = [0] * number_classes
progress_count = 0
# counter of the total number of objects in the future dataset
count_of_image = 0
name_to_xml = str(_count_)
# cycle through all class parameters
for x in item:
count_of_image += int(x.count)
bar = progressbar.ProgressBar(maxval=count_of_image,
widgets=config.widgets).start()
# flag that is responsible for stopping the formation of the dataset
flag = True
# cycle for creating a dataset
while flag:
# a tuple that stores the size of an image
size_ = (config.image_size['width'], config.image_size['height'])
# creating an xml file
root = Element("annotation")
file = SubElement(root, "filename")
file.text = str(name_to_xml + '.jpg')
path = SubElement(root, "path")
path.text = str(
f'{config.dir_output_data}/{config.dir_result_img}/{name_to_xml}.jpg'
)
size_tree = SubElement(root, "size")
width = SubElement(size_tree, "width")
width.text = str(int(size_[0]))
height = SubElement(size_tree, "height")
height.text = str(int(size_[1]))
depth = SubElement(size_tree, "depth")
depth.text = "3"
# creating auxiliary arrays
object_, name, xmin_, xmax_, ymax_, ymin_, bndbox = [], [], [], [], [], [], []
# creating a new image with the specified dimensions and a black background
img = Image.new(
"RGB", (config.image_size['width'], config.image_size['height']),
'#191919')
# flag that controls the fullness of the dataset
flag_end = True
# variable responsible for the number of files created
end_file = 0
# auxiliary variable for accessing arrays for xml markup
number = 0
# creating a matrix for the location of class objects
matrix = create_matrix()
# checking for boundary conditions
while flag_end:
end = 0
if index >= number_classes:
index = 0
if list_count_of_each_class[index] >= int(item[index].count):
while list_count_of_each_class[index] >= int(
item[index].count):
index += 1
end += 1
if index == number_classes:
index = 0
if end == number_classes:
flag = False
break
if not flag:
break
coeff_orientation = None
# orientation check to generate the orientation coefficient
if item[index].orientation == 0: # original
coeff_orientation = 1
# elif(item[index].orientation == 1): # horizontal
# coeff_orientation = random.randrange(70, 80, 1) / 100
# elif(item[index].orientation == 2): # vertical
# coeff_orientation = 100 / random.randrange(70, 80, 1)
# if (item[index].current_size[1] + item[index].config.step_change_size <= item[index].size_finish[1]):
# item[index].orientation = (item[index].orientation + 1 if (item[index].orientation < 2) else 0)
# the number of occupied cells in the matrix by width
count_of_x_boxes_item = int((
item[index].current_size[1] // config.step['width'] +
1) if item[index].current_size[1] % config.step['width'] > 0
else int(item[index].current_size[1] //
config.step['width']))
# the number of occupied cells in the matrix by height
count_of_y_boxes_item = int((
item[index].current_size[0] // config.step['height'] +
1) if item[index].current_size[0] % config.step['height'] > 0
else int(item[index].current_size[0] //
config.step['height']))
# the path to the prepared image of the class object
full_file_name = os.path.join(config.work_dir,
config.dir_output_data,
config.dir_prepare_boxes,
f'{item[index].name}.png')
# the path to the prepared image of the class object opening the image and converting to RGBA
button = Image.open(full_file_name).convert("RGBA")
# flag for determining whether an object of the class will be added to the image
done_item_box = False
# cycle through the matrix in height
for i in range(4):
global j
# cycle through the matrix in width
for j in range(4):
# flag for adding a class object to an image
is_fit = True
# checking that an object of the class with the new dimensions is placed on the image
if (count_of_x_boxes_item + j <
5) and (count_of_y_boxes_item + i < 5):
# cycle through the allowed cells in height
for y in range(i, i + count_of_y_boxes_item):
# cycle through the allowed cells in width
for x in range(j, j + count_of_x_boxes_item):
# checking that the cells of the matrix in which we want to place the object are free
if matrix[y, x] > 0:
# we transfer the flag to the stage of the impossibility of adding it
is_fit = False
break
if not is_fit:
break
# checking whether an object of the class can be added to the image
if is_fit:
# cycle through the allowed cells in height
for y in range(i, i + count_of_y_boxes_item):
# cycle through the allowed cells in width
for x in range(j, j + count_of_x_boxes_item):
# note in the matrix that these cells are filled in
matrix[y, x] = 1
# setting the flag for adding a class object to the true state
done_item_box = True
break
# checking whether an object of the class can be added to the image
if done_item_box:
# resetting the variable number of filled files
end_file = 0
# the width of the area required for the class object
width = (j + count_of_x_boxes_item) * config.step['width']
# the height of the area required for the class object
height = (i +
count_of_y_boxes_item) * config.step['height']
# offset from the edge
random_indent = random.randrange(
5, 15, 5) if ((item[index].current_size[1] == 100) or
(item[index].current_size[1] == 120)
) else random.randrange(20, 40, 5)
width_min, height_min, width_max, height_max = None, None, None, None
# checking that the class object should be shifted to the upper-left corner relative to the start
# cell
if item[index].current_position == 1:
width_min = width - count_of_x_boxes_item * config.step[
'width'] + random_indent
height_min = height - count_of_y_boxes_item * config.step[
'height'] + random_indent
width_max = width - count_of_x_boxes_item * config.step[
'width'] + item[index].current_size[
1] + random_indent
height_max = height - count_of_y_boxes_item * config.step[
'height'] + item[index].current_size[
0] + random_indent
item[index].current_position += 1
# checking that the class object should be shifted to the upper-right corner relative to the start
# cell
elif item[index].current_position == 2:
width_min = width - item[index].current_size[
1] - random_indent
height_min = height - count_of_y_boxes_item * config.step[
'height'] + random_indent
width_max = width - random_indent
height_max = height - count_of_y_boxes_item * config.step[
'height'] + item[index].current_size[
0] + random_indent
item[index].current_position += 1
# checking that the class object should be shifted to the lower-left corner relative to the start
# cell
elif item[index].current_position == 3:
width_min = width - count_of_x_boxes_item * config.step[
'width'] + random_indent
height_min = height - item[index].current_size[
0] - random_indent
width_max = width - count_of_x_boxes_item * config.step[
'width'] + item[index].current_size[
1] + random_indent
height_max = height - random_indent
item[index].current_position += 1
# checking that the class object should be shifted to the lower-right corner relative to the start
# cell
elif item[index].current_position == 4:
width_min = width - item[index].current_size[
1] - random_indent
height_min = height - item[index].current_size[
0] - random_indent
width_max = width - random_indent
height_max = height - random_indent
item[index].current_position += 1
# checking that the class object should be shifted to the center part relative to the start cell
elif item[index].current_position == 5:
width_min = width - count_of_x_boxes_item * config.step['width'] / 2 - \
item[index].current_size[1] / 2
height_min = height - count_of_y_boxes_item * config.step['height'] / 2 - \
item[index].current_size[0] / 2
width_max = width - count_of_x_boxes_item * config.step['width'] / 2 + \
item[index].current_size[1] / 2
height_max = height - count_of_y_boxes_item * config.step['height'] / 2 + \
item[index].current_size[0] / 2
item[index].current_position = 1
progress_count += 1
bar.update(progress_count)
# filter for resizing an image
resample = Image.ANTIALIAS
button.load()
# splitting an image into strips (R, G, B, A)
bands = button.split()
# getting new object sizes
dim = (int(width_max - width_min),
int(height_max - height_min))
# converting each color-light strip of the image to a new size
bands = [b.resize(dim, resample) for b in bands]
# combining all the stripes into a single image
button = Image.merge('RGBA', bands)
# inserting the resulting class object into the image
img.paste(button, (int(width_min), int(height_min)),
button)
# adding a class name to xml
object_.append(SubElement(root, "object_"))
name.append(SubElement(object_[number], "name"))
name[number].text = str(item[index].name)
bndbox.append(SubElement(object_[number], "bndbox"))
xmin_.append(SubElement(bndbox[number], "xmin"))
xmin_[number].text = str(int(width_min))
ymin_.append(SubElement(bndbox[number], "ymin"))
ymin_[number].text = str(int(height_min))
xmax_.append(SubElement(bndbox[number], "xmax"))
xmax_[number].text = str(int(width_max))
ymax_.append(SubElement(bndbox[number], "ymax"))
ymax_[number].text = str(int(height_max))
# increasing the number of added objects for the current class
list_count_of_each_class[index] += 1
# increasing the auxiliary variable by 1
number += 1
# getting a new object height
height_current = (item[index].current_size[1] + item[index].step_change_size) * \
item[index].current_size[0] / item[index].current_size[1]
# getting a new object width
item[index].current_size[1] = (
item[index].current_size[1] +
item[index].step_change_size
if item[index].current_size[1] +
item[index].step_change_size <=
item[index].size_finish[1] else
item[index].size_start[1])
# checking that the current width is equal to the initial width
if item[index].current_size[1] == item[index].size_start[
1]:
height_current = item[index].size_start[0]
# getting the height
item[index].current_size[
0] = height_current * coeff_orientation
break
# checking that the class object did not fit into the file
if not done_item_box:
end_file += 1
# checking that the number of filled files == the number of classes
if end_file == number_classes:
flag_end = False
# moving the class number variable to the next class
index += 1
# saving the image
img.save(
os.path.join(config.work_dir, config.dir_output_data,
config.dir_result_img, f'{str(_count_)}.jpg'))
# path to the xml file
name_ = os.path.join(config.work_dir, config.dir_output_data,
config.dir_result_xml, f'{str(_count_)}.xml')
# calling the function that saves the xml file
save_xml(name_, root)
# increasing the file number by 1
_count_ += 1
bar.finish()
logfile.write(str(getattr(args, arg)))
logfile.write("\n")
cmdfile = open("3_cmds", 'w')
bcin = open(bcfile, 'r')
bc = []
for bcinline in bcin:
bc.append(bcinline)
bclength = len(bc)
print "Number of entries: ", bclength
print "\nWorking..."
bar = progressbar.ProgressBar()
for i in bar(range(bclength)):
bcline = bc[i]
bccols = bcline.split()
sample = bccols[1]
output = wd + "/" + sample
if seqprep_output_in_output:
seqprep_output = output + "/" + seqprep_output_orig
else:
seqprep_output = seqprep_output_orig
so_s = seqprep_output + "/" + sample
mo_s = metadir + "/" + sample
alllist = []
if not os.path.isfile(so_s + ".all.SP.fq"):
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout):
# Currently only 'marathon' instances are supported for wait_for_deployment because they
# are the only thing that are worth waiting on.
service_configs = PaastaServiceConfig(service=service,
soa_dir=soa_dir,
load_deployments=False)
total_instances = 0
clusters_data = []
api_endpoints = load_system_paasta_config().get_api_endpoints()
for cluster in service_configs.clusters:
if cluster not in api_endpoints:
paasta_print(
PaastaColors.red(
'Cluster %s is NOT in paasta-api endpoints config.' %
cluster, ))
raise NoSuchCluster
instances_queue = Queue()
for instance_config in service_configs.instance_configs(
cluster=cluster, instance_type='marathon'):
if instance_config.get_deploy_group() == deploy_group:
instances_queue.put(instance_config)
total_instances += 1
if not instances_queue.empty():
clusters_data.append(
ClusterData(
cluster=cluster,
service=service,
git_sha=git_sha,
instances_queue=instances_queue,
))
if not clusters_data:
_log(
service=service,
component='deploy',
line=
("Couldn't find any marathon instances for service {} in deploy group {}. Exiting."
.format(service, deploy_group)),
level='event',
)
return
paasta_print("Waiting for deployment of {} for '{}' to complete...".format(
git_sha, deploy_group))
deadline = time.time() + timeout
green_light = Event()
green_light.set()
with progressbar.ProgressBar(maxval=total_instances) as bar:
while time.time() < deadline:
_query_clusters(clusters_data, green_light)
if not green_light.is_set():
raise KeyboardInterrupt
bar.update(total_instances - sum((c.instances_queue.qsize()
for c in clusters_data)))
if all((cluster.instances_queue.empty()
for cluster in clusters_data)):
sys.stdout.flush()
return 0
else:
time.sleep(min(60, timeout))
sys.stdout.flush()
_log(
service=service,
component='deploy',
line=compose_timeout_message(clusters_data, timeout, deploy_group,
service, git_sha),
level='event',
)
raise TimeoutError
import pickle
import math
import progressbar
loc = input('Enter the path to the file you want to prune: ')
data = None
try:
data = open(loc, 'rb')
except Exception as e:
raise Exception("Could not read file")
menance = pickle.load(data)
done = 0
length = len(menance.moves)
pbartmp = progressbar.ProgressBar(maxval=1).default_widgets()
pbar = progressbar.ProgressBar(widgets=pbartmp[:], maxval=length)
del pbartmp
for key in menance.moves:
done += 1
state = menance.moves[key].state
if len(state) > 0:
unique = list(set(state))
unique_count = [state.count(num) for num in unique]
mmin = min(unique_count)
new_count = [math.ceil(count / mmin) for count in unique_count]
tmp = []
for i in range(len(new_count)):
orig = unique[i]
for j in range(new_count[i]):
tmp.append(orig)
def _consume_multi_display_multi_pbar(progress_queue, num_workers,
num_simulations, max_width, fd):
# In order to display multiple progress bars, we need to manipulate the
# terminal/console to move up lines. Colorama is used to wrap stderr such
# that ANSI escape sequences are mapped to equivalent win32 API calls.
fd = colorama.AnsiToWin32(fd).stream
def ansi_up(n):
return b'\x1b[{}A'.decode('latin1').format(n)
ansi_bold = b'\x1b[1m'.decode('latin1')
ansi_norm = b'\x1b[0m'.decode('latin1')
overall_pbar = _get_overall_pbar(num_simulations, max_width, fd)
try:
worker_progress = OrderedDict()
completed = set()
while len(completed) < num_simulations:
sim_index, now, t_stop, timescale = progress_queue.get()
if now == t_stop:
completed.add(sim_index)
if worker_progress:
print(ansi_up(len(worker_progress)), end='', file=fd)
if sim_index in worker_progress:
for pindex, pbar in worker_progress.items():
if sim_index == pindex and pbar:
if now == t_stop:
pbar.finish()
worker_progress[sim_index] = None
else:
if t_stop and pbar.max_value != t_stop:
pbar.max_value = t_stop
pbar.widgets = _get_progressbar_widgets(
sim_index, timescale, know_stop_time=True)
pbar.update(now)
print(file=fd)
else:
print(file=fd)
else:
for pindex, pbar in worker_progress.items():
if pbar is None:
worker_progress.pop(pindex)
break
print('\n' * len(worker_progress), file=fd)
pbar = progressbar.ProgressBar(
fd=fd,
term_width=overall_pbar.term_width,
min_value=0,
max_value=(progressbar.UnknownLength
if t_stop is None else t_stop),
widgets=_get_progressbar_widgets(sim_index,
timescale,
know_stop_time=t_stop
is not None))
worker_progress[sim_index] = pbar
print(ansi_bold, end='', file=fd)
overall_pbar.update(len(completed))
print(ansi_norm, end='', file=fd)
finally:
print(ansi_bold, end='', file=fd)
overall_pbar.finish()
print(ansi_norm, end='', file=fd)
def compute(self,
networkinfo,
elon,
elat,
edep,
vp=6.5,
vs=3.5,
nnst=6,
procdelay=False,
target=None,
nmaps=500,
resultsfn='p_wave_tt_6_stations.npz',
datadir='./data',
latencies=None,
vp3d=False):
# If given a target location (lon,lat) it will also compute
# the lead time from any potential epicenter to the target site
if target is not None:
tln, tlt = target
# get the station locations
networks = networkinfo.get_networks()
# get the delay measurement distributions:
# pkdel = pick delays
# envdel = envelope delays
# ascdel = associator/locator delays
# magdel = magnitude delays
# pkdefault = pick default delay
# envdefault = envelope default delay
# Note pkdel and envdel have to be Python dictionaries that have
# a Numpy array of delay measurements for every station name key
# ascdel and magdel are simply Numpy arrays containing delay
# observations
# pkdefault and envdefault are Python dictionaries that contain a
# default delay value for every network name key. These default values
# are used if a station is in the list of stations returned by
# networkinfo but does not have delay observations in pkdel and envdel
if procdelay and latencies is not None:
pkdel = latencies.get_pick_delays()
envdel = latencies.get_envelope_delays()
ascdel = latencies.get_associator_delays()
magdel = latencies.get_magnitude_delays()
pkdefault = latencies.get_pick_default()
envdefault = latencies.get_envelope_default()
# do we have a grid or a set of points?
if elon.shape != elat.shape or elon.shape != edep.shape:
raise DelayException(
"elon, elat, and edep must have the same shape!")
outshape = [nmaps]
for _s in elon.shape:
outshape.append(_s)
lat = elat.ravel()
lon = elon.ravel()
dep = edep.ravel()
ngp = lon.size
ttP = np.zeros(outshape)
ttPtmp = np.zeros(ngp)
tstarget = ttPtmp.copy()
g = pyproj.Geod(ellps='sphere')
# Setup progressbar
widgets = ['tt: ', pg.Percentage(), ' ', pg.Bar('#'), ' ', pg.ETA()]
pbar = pg.ProgressBar(widgets=widgets, maxval=nmaps * ngp).start()
no_env_dl = []
no_pk_dl = []
# loop over samples (Monte-Carlo simulation)
for nm in range(nmaps):
idx = 0
# loop over earthquake locations
for _lat, _lon, _dep in zip(lat, lon, dep):
pbar.update(nm * ngp + idx)
min_dt = 1.e38
# loop over networks; all stations within a network are
# considered for delay computations but only the fastest alert
# is stored
for net in networks.keys():
if len(networks[net]['lat']) < 1:
continue
stlat = networks[net]['lat']
stlon = networks[net]['lon']
nwcode = np.array(networks[net]['nw'])
names = np.array(networks[net]['nm'])
maxnstat = min(10, len(stlat))
nnst = min(nnst, len(stlat))
# An option to used traveltimes from a 3D model instead
# of a homogeneous halfspace; only works on my computer
# at the moment
if vp3d:
datadir = './data/ttime_ch/'
fin = os.path.join(datadir, "%.4f_%.4f_%.1f.npy" % \
(_lat, _lon, _dep))
if not os.path.isfile(fin):
raise DelayException('File %s does not exist!' %
fin)
a = np.load(fin)
pt = a['ttime'][:, 0]
dt = max(pt[0:nnst])
stat_names = a['name']
else:
# Find the <nnst> nearest stations
lats = np.ones((len(stlat), )) * _lat
lons = np.ones((len(stlon), )) * _lon
az, baz, dist = g.inv(lons, lats, stlon, stlat)
dist_sorted = np.sort(dist)
stat_names = names[np.argsort(dist)]
dz = np.ones((maxnstat, )) * _dep
pt = np.sqrt(dz * dz +
dist_sorted[0:maxnstat] / 1000. *
dist_sorted[0:maxnstat] / 1000.) / vp
dt = max(pt[0:nnst])
if target is not None:
azt, bazt, distt = g.inv(_lon, _lat, tln, tlt)
distt /= 1000.
tstarget[idx] = np.sqrt(distt * distt +
_dep * _dep) / vs
# Add delays
# Note: if a close by station has very long delays, the
# algorithm will pick one of the next further stations
# if they can send data before the closer station
if procdelay:
pk_delays = []
env_delays = []
for stat in stat_names[0:maxnstat]:
if stat in pkdel.keys():
pk_delays.append(pkdel[stat][np.random.randint(
0, len(pkdel[stat]))])
else:
if stat not in no_pk_dl:
no_pk_dl.append(stat)
pk_delays.append(pkdefault[net])
if stat in envdel.keys():
env_delays.append(
envdel[stat][np.random.randint(
0, len(envdel[stat]))])
else:
if stat not in no_env_dl:
no_env_dl.append(stat)
env_delays.append(envdefault[net])
if len(pk_delays) != maxnstat or len(
env_delays) != maxnstat:
raise DelayException(
'Number of delays is not equal to %d' % nnst)
temp = np.sort(pt + np.array(pk_delays)) + ascdel[
np.random.randint(0, len(ascdel))]
origin_delay = max(temp[0:nnst])
waveform_delay = min(pt + np.array(env_delays))
dt = max(origin_delay,
waveform_delay) + magdel[np.random.randint(
0, len(magdel))]
if dt < min_dt:
min_dt = dt
ttPtmp[idx] = min_dt
idx += 1
ttP[nm] = ttPtmp.reshape(elon.shape)
tstarget = tstarget.reshape(elon.shape)
# store the results for plotting
if resultsfn is not None:
np.savez(resultsfn,
ttP=ttP,
tstarget=tstarget,
lat=elat,
lon=elon,
dep=edep)
pbar.finish()
print "No envelope delay data available for the following stations:"
print ' '.join(no_env_dl)
print "No pick delay info available for the following stations:"
print ' '.join(no_pk_dl)
return elat, elon, edep, ttP, tstarget
def generate(self):
""" Gradient descent based PCF matching for point synthesis """
const = self.edge_correction()
sigma = self.choose_sigma()
g0, g_init, D, D_old, weight, G_init_local, X_init = self.initial_calculation(
)
max_r = 2 * self.r_1
r = np.linspace(0.0001, max_r, self.len_r)
g = g_init
""" Initialize parameters """
j = 1
X = dc(X_init)
G_local_old = (G_init_local)
error_func = []
error = np.square(np.subtract(g, g0)).mean()
error_func.append(error)
losses = []
alpha = self.lr
beta1 = 0.9
beta2 = 0.999
eps = 1 / np.power(10, 8)
m_t = np.zeros((self.N, self.d))
v_t = np.zeros((self.N, self.d))
m_hat = np.zeros((self.N, self.d))
v_hat = np.zeros((self.N, self.d))
""" Gradient Descent Loop """
with progressbar.ProgressBar(max_value=self.maxGD) as bar:
while (error > self.MSE and j < self.maxGD):
j = j + 1
for i in range(1, self.N + 1):
G = np.divide(np.multiply(g - g0, const), weight)
U = X[i - 1, :] - X
N1 = np.linalg.norm(U, axis=1) + 0.0001
D_old[i - 1] = dc(N1)
D_old[i - 1][D_old[i - 1] == 0] = float('inf')
Y_old = (np.mat(D_old[i - 1])) - (np.transpose(
np.matrix(r)))
Lo_old = self.G_kern(Y_old, sigma)
N1[i - 1] = 1
N2 = np.tile(np.transpose(np.matrix(N1)), (1, self.d))
U = np.divide(U, N2)
Y_old[np.isinf(Y_old)] = 0
Z = np.multiply(
np.multiply(np.transpose(np.tile(G, (self.N, 1))),
Y_old), Lo_old)
W = Z.sum(axis=0)
M = np.multiply(U, np.transpose(np.tile(W, (self.d, 1))))
grad = -(M.sum(axis=0) /
np.linalg.norm(M.sum(axis=0) + 0.0001))
""" ADAM gradient update """
m_t[i - 1, :] = beta1 * m_t[i - 1, :] + (1 - beta1) * grad
v_t[i - 1, :] = beta2 * v_t[i - 1, :] + (
1 - beta2) * np.square(grad)
m_hat[i - 1, :] = m_t[i - 1, :] / (1 - beta1)
v_hat[i - 1, :] = v_t[i - 1, :] / (1 - beta2)
adam_grad = alpha * m_hat[i - 1, :] / (
np.sqrt(v_hat[i - 1, :]) + eps)
X[i - 1, :] = np.subtract(X[i - 1, :], adam_grad)
X[np.isnan(X)] = 0
""" Points should be in the bounded region """
for jj in range(self.d):
if np.mod(i, 3) == 0:
if X[i - 1, jj] > 1:
X[i - 1, jj] = np.random.rand(1)
if X[i - 1, jj] < 0:
X[i - 1, jj] = np.random.rand(1)
else:
if X[i - 1, jj] > 1:
X[i - 1, jj] = .999
if X[i - 1, jj] < 0:
X[i - 1, jj] = 0.001
U = X[i - 1, :] - X
D[i - 1] = np.linalg.norm(U, axis=1)
D[i - 1][D[i - 1] == 0] = float('inf')
Y = (np.mat(D[i - 1])) - (np.transpose(np.matrix(r)))
Lo = self.G_kern(Y, sigma)
Ks = Lo.sum(axis=1)
G_local_new = np.multiply(
np.array(const).flatten(),
np.array(Ks).flatten())
G_local_old[i - 1, :] = G_local_new
""" Update local PCF of all the point """
g = G_local_old.sum(axis=0)
g[g < 0] = 0
Loss = np.mean(np.divide(np.multiply(g - g0, const), weight))
losses.append(Loss)
error = np.square(np.subtract(g, g0)).mean()
error_func.append(error)
# print ( 'epoch :'+str(j)+' MSE error:'+str(error) +' Loss:'+str(Loss))
bar.update(j)
return X, g, g0, error_func, losses
def main():
# 默认路径
# CONFIG_FILE = 'configs/mask_rcnn/mask_rcnn_r50_fpn_1x_coco.py' # 模型的配置文件
# CONFIG_FILE = 'configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco.py' # 模型的配置文件
# CONFIG_FILE = 'configs/yolo/yolov3_d53_mstrain-608_273e_coco.py' # 模型的配置文件
CONFIG_FILE = 'configs/ssd/ssd300_coco.py' # 模型的配置文件
# CONFIG_FILE = 'configs/retinanet/retinanet_r50_fpn_1x_coco.py' # 模型的配置文件
CHECKPOINT_FILE = 'work_dirs/ssd300_coco/epoch_12.pth' # 训练好的模型权重
# IMG_PATH = '/home/watson/Documents/mask_THzDatasets/val'
IMG_PATH = '/home/watson/Documents/mask_THzDatasets/test2/imgs'
RESULT_ROOT = "/home/watson/Documents/mmdet_result/{}".format(
os.path.splitext(os.path.basename(CONFIG_FILE))[0])
# 解析参数
args = parse_args()
if args.config is not None:
CONFIG_FILE = args.config
if args.checkpoint is not None:
CHECKPOINT_FILE = args.checkpoint
if args.img_input is not None:
IMG_PATH = args.img_input
if args.img_output is not None:
RESULT_ROOT = args.img_output
RESULT_IMG_PATH = os.path.join(RESULT_ROOT, "imgs")
RESULT_XML_PATH = os.path.join(RESULT_ROOT, "xmls")
if not os.path.exists(RESULT_IMG_PATH):
os.makedirs(RESULT_IMG_PATH)
if not os.path.exists(RESULT_XML_PATH):
os.makedirs(RESULT_XML_PATH)
# 初始化模型
print('=== initialing detector ===')
model = init_detector(CONFIG_FILE, CHECKPOINT_FILE)
# 推断图片
print('inference start!')
img_list = os.listdir(IMG_PATH)
img_cnt = 0
img_totalNum = len(img_list)
bar = progressbar.ProgressBar(max_value=img_totalNum).start()
for img_fileName in img_list:
img_cnt += 1
bar.update(img_cnt)
img_fullFileName = os.path.join(IMG_PATH, img_fileName)
result_img_fullName = os.path.join(RESULT_IMG_PATH, img_fileName)
result_xml_fullName = os.path.join(
RESULT_XML_PATH,
img_fileName.split('.')[0] + "_result.xml")
result = inference_detector(model, img_fullFileName)
img = model.show_result(img_fullFileName,
result,
score_thr=0.3,
show=False,
out_file=result_img_fullName)
# show_result_pyplot(model, img_fullFileName, result, score_thr=0.3)
detections = resolve_result(result,
class_names=model.CLASSES,
score_thr=0.3)
store_voc_result(img_fullFileName, detections, result_xml_fullName)
bar.finish()
print('inference finished!')
return finale
# create custEmails dataframe with regular customers' ids and emails (below)
custEmails = pd.DataFrame(orig_data[orig_data['Customer'].str.startswith('CC')]
['Customer'].drop_duplicates())
#save count to var howmany (only regular customers)
howmany = len(custEmails)
counter = 0 #counter for While loop
emailarray = [] #empty array for loop
print "Creating email addresses..."
prebar = progressbar.ProgressBar(maxval=int(howmany))
for i in prebar(range(howmany)):
while counter < howmany:
emailarray.append(str(makeEmail()))
counter = counter + 1
prebar.update(i)
print "Email creation completed."
bar = progressbar.ProgressBar(maxval=int(howmany))
custEmails['Email'] = emailarray
# save regular customer Ids and Emails in custEmails.csv
custEmails.to_csv('../custEmails.csv', index=None, header=False)
"""
Command line options
"""
parser.add_argument(
"--download_folder",
type=str,
default=download_folder,
help="Folder to download weights to. Default is " + download_folder,
)
FLAGS = parser.parse_args()
url = "https://pjreddie.com/media/files/yolov3.weights"
r = requests.get(url, stream=True)
f = open(os.path.join(download_folder, "yolov3.weights"), "wb")
file_size = int(r.headers.get("content-length"))
chunk = 100
num_bars = file_size // chunk
bar = progressbar.ProgressBar(maxval=num_bars).start()
i = 0
for chunk in r.iter_content(chunk):
f.write(chunk)
bar.update(i)
i += 1
f.close()
call_string = "python convert.py yolov3.cfg yolov3.weights yolo.h5"
subprocess.call(call_string, shell=True, cwd=download_folder)
def main():
workspace_path = os.environ.get('AE_WORKSPACE_PATH')
if workspace_path is None:
print('Please define a workspace path:\n')
print('export AE_WORKSPACE_PATH=/path/to/workspace\n')
exit(-1)
gentle_stop = np.array((1, ), dtype=np.bool)
gentle_stop[0] = False
def on_ctrl_c(signal, frame):
gentle_stop[0] = True
signal.signal(signal.SIGINT, on_ctrl_c)
parser = argparse.ArgumentParser()
parser.add_argument("experiment_name")
parser.add_argument("-d", action='store_true', default=False)
parser.add_argument("-gen", action='store_true', default=False)
arguments = parser.parse_args()
full_name = arguments.experiment_name.split('/')
experiment_name = full_name.pop()
experiment_group = full_name.pop() if len(full_name) > 0 else ''
debug_mode = arguments.d
generate_data = arguments.gen
cfg_file_path = u.get_config_file_path(workspace_path, experiment_name,
experiment_group)
log_dir = u.get_log_dir(workspace_path, experiment_name, experiment_group)
checkpoint_file = u.get_checkpoint_basefilename(log_dir)
ckpt_dir = u.get_checkpoint_dir(log_dir)
train_fig_dir = u.get_train_fig_dir(log_dir)
dataset_path = u.get_dataset_path(workspace_path)
if not os.path.exists(cfg_file_path):
print('Could not find config file:\n')
print('{}\n'.format(cfg_file_path))
exit(-1)
if not os.path.exists(ckpt_dir):
os.makedirs(ckpt_dir)
if not os.path.exists(train_fig_dir):
os.makedirs(train_fig_dir)
if not os.path.exists(dataset_path):
os.makedirs(dataset_path)
args = configparser.ConfigParser()
args.read(cfg_file_path)
shutil.copy2(cfg_file_path, log_dir)
with tf.variable_scope(experiment_name):
dataset = factory.build_dataset(dataset_path, args)
queue = factory.build_queue(dataset, args)
encoder = factory.build_encoder(queue.x, args, is_training=True)
decoder = factory.build_decoder(queue.y,
encoder,
args,
is_training=True)
ae = factory.build_ae(encoder, decoder, args)
codebook = factory.build_codebook(encoder, dataset, args)
train_op = factory.build_train_op(ae, args)
saver = tf.train.Saver(save_relative_paths=True)
num_iter = args.getint('Training',
'NUM_ITER') if not debug_mode else 100000
save_interval = args.getint('Training', 'SAVE_INTERVAL')
model_type = args.get('Dataset', 'MODEL')
print("there will be %d iterations" % num_iter)
if model_type == 'dsprites':
dataset.get_sprite_training_images(args)
else:
dataset.get_training_images(dataset_path, args)
dataset.load_bg_images(dataset_path)
if generate_data:
print('finished generating synthetic training data for ' +
experiment_name)
print('exiting...')
exit()
widgets = [
'Training: ',
progressbar.Percentage(), ' ',
progressbar.Bar(), ' ',
progressbar.Counter(),
' / %s' % num_iter, ' ',
progressbar.ETA(), ' '
]
bar = progressbar.ProgressBar(maxval=num_iter, widgets=widgets)
gpu_options = tf.GPUOptions(allow_growth=True,
per_process_gpu_memory_fraction=0.9)
config = tf.ConfigProto(gpu_options=gpu_options)
with tf.Session(config=config) as sess:
chkpt = tf.train.get_checkpoint_state(ckpt_dir)
if chkpt and chkpt.model_checkpoint_path:
saver.restore(sess, chkpt.model_checkpoint_path)
else:
sess.run(tf.global_variables_initializer())
merged_loss_summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(ckpt_dir, sess.graph)
if not debug_mode:
print('Training with %s model' % args.get('Dataset', 'MODEL'),
os.path.basename(args.get('Paths', 'MODEL_PATH')))
bar.start()
queue.start(sess)
for i in range(ae.global_step.eval(), num_iter):
print("step %d" % i)
if not debug_mode:
sess.run(train_op)
if i % 10 == 0:
loss = sess.run(merged_loss_summary)
summary_writer.add_summary(loss, i)
bar.update(i)
if (i + 1) % save_interval == 0:
saver.save(sess,
checkpoint_file,
global_step=ae.global_step)
this_x, this_y = sess.run([queue.x, queue.y])
reconstr_train = sess.run(decoder.x,
feed_dict={queue.x: this_x})
train_imgs = np.hstack(
(u.tiles(this_x, 4,
4), u.tiles(reconstr_train, 4,
4), u.tiles(this_y, 4, 4)))
cv2.imwrite(
os.path.join(train_fig_dir,
'training_images_%s.png' % i),
train_imgs * 255)
else:
this_x, this_y = sess.run([queue.x, queue.y])
reconstr_train = sess.run(decoder.x,
feed_dict={queue.x: this_x})
cv2.imshow(
'sample batch',
np.hstack((u.tiles(this_x, 3,
3), u.tiles(reconstr_train, 3,
3), u.tiles(this_y, 3, 3))))
k = cv2.waitKey(0)
if k == 27:
break
if gentle_stop[0]:
break
queue.stop(sess)
if not debug_mode:
bar.finish()
if not gentle_stop[0] and not debug_mode:
print('To create the embedding run:\n')
print('ae_embed {}\n'.format(full_name))
