else: net_ = 'error' interface_name_ = '' for i_ in net_: if str(i_['color']) == str(data_['COLOR']) and str(i_['number']) == str( data_['NUMBER']) and str(i_['address']) == str(data_['ADDRESS']): interface_name_ = i_['name'] print('Имя интерфейса для которого производится проверка: ' + interface_name_) print('----------------------------------------------------------') print('\x1b[1;32;40m' + ' Шаг-6: Проверка по словарю: ' + '\x1b[0m') # post-запрос на изменение параметров bar_ = progressbar.ProgressBar( maxval=(len(dictionary_) + 1), widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()]) bar_.start() number_ = 0 table_data_ = [[ 'Имя', 'Описание', 'Тип', 'Параметр', 'Ожидаемый', 'Код post', 'Фактический (sysifaces)', 'Фактический (netstatus)', 'Результат' ]] for i_ in dictionary_: # Перед проверкой выставляем значение по умолчанию: urn_ = str(parameters_['post']['sysifaces_interfaces']['urn']) data_ = parameters_['post']['sysifaces_interfaces']['data'].copy() req_ = librequests.post(protocol_, host_, port_, urn_, login_, password_, timeout_, data_)
def detect_write_images(self): batch_beg = self.args.data_inst.train_test_split outdir_good = os.path.join(self.args.log_dir_t, 'good') if not os.path.exists(outdir_good): os.makedirs(outdir_good) outdir_bad = os.path.join(self.args.log_dir_t, 'bad') if not os.path.exists(outdir_bad): os.makedirs(outdir_bad) with h5py.File(self.predict_file, 'r') as pred_h5: fig = mpplot.figure(figsize=(5, 5)) ax = fig.add_axes([0, 0, 1, 1]) poses_pred_h5 = pred_h5['poses'] num_line = poses_pred_h5.shape[0] print('start writing {} images ...'.format(num_line)) timerbar = progressbar.ProgressBar(maxval=num_line, widgets=[ progressbar.Percentage(), ' ', progressbar.Bar( '=', '[', ']'), ' ', progressbar.ETA() ]).start() store_handle = self.store_handle['test'] frame_h5 = store_handle[self.frame_type] poses_echt_h5 = store_handle['poses'] resce_h5 = store_handle['resce'] index_h5 = store_handle['index'] error_cap = (10, 20) bad_cnt = 0 good_cnt = 0 for li in np.arange(num_line): poses_pred = poses_pred_h5[li, ...].reshape(-1, 3) poses_echt = poses_echt_h5[(batch_beg + li), ...].reshape(-1, 3) error = np.mean( np.sqrt(np.sum((poses_pred - poses_echt)**2, axis=1))) if error_cap[1] < error: bad_cnt += 1 # # self._draw_image_pose( # # ax, # # frame_h5[(batch_beg + li), ...], # # poses_pred, # # resce_h5[(batch_beg + li), ...], # # self.caminfo) # self._draw_image_pose_compare( # ax, self.caminfo, # frame_h5[(batch_beg + li), ...], # resce_h5[(batch_beg + li), ...], # poses_pred, poses_echt) # ax.axis('off') # mpplot.savefig(os.path.join( # outdir_bad, # self.args.data_io.index2imagename(index_h5[li]))) # ax.clear() if error_cap[0] > error: good_cnt += 1 # if 0 == (good_cnt % 100): # # self._draw_image_pose( # # ax, # # frame_h5[(batch_beg + li), ...], # # poses_pred, # # resce_h5[(batch_beg + li), ...], # # self.caminfo) # self._draw_image_pose_compare( # ax, self.caminfo, # frame_h5[(batch_beg + li), ...], # resce_h5[(batch_beg + li), ...], # poses_pred, poses_echt) # ax.axis('off') # mpplot.savefig(os.path.join( # outdir_good, # self.args.data_io.index2imagename(index_h5[li]))) # ax.clear() if 0 == (li % 1000): self._draw_image_pose_compare( ax, self.caminfo, frame_h5[(batch_beg + li), ...], resce_h5[(batch_beg + li), ...], poses_pred, poses_echt) ax.axis('off') mpplot.savefig( os.path.join( outdir_bad, self.args.data_io.index2imagename(index_h5[li]))) ax.clear() timerbar.update(li) timerbar.finish() mpplot.close(fig) self.logger.info( '{} good detections, {} bad detections for error cap [{}]'. format(good_cnt, bad_cnt, error_cap))
criterion_yaw = criterion_yaw.cuda() criterion_d = criterion_d.cuda() model_g.train() model_f1.train() model_f2.train() print('Will train from epoch %d to epoch %d (exclusively)' % (start_epoch, args.epochs)) for epoch in range(start_epoch, args.epochs): widgets = [ 'Epoch %d/%d,' % (epoch, args.epochs), ' ', progressbar.Counter('batch %(value)d/%(max_value)d') ] bar = progressbar.ProgressBar(widgets=widgets, max_value=len(train_loader), redirect_stdout=True) for ibatch, batch in bar(enumerate(train_loader)): src_imgs, src_masks, tgt_imgs = batch['S_image'], batch[ 'S_mask'], batch['T_image'] src_imgs, src_masks, tgt_imgs = Variable(src_imgs), Variable( src_masks), Variable(tgt_imgs) if args.num_of_labelled_target > 0: tgt_masks, tgt_use_masks = batch[ 'T_mask'], batch['T_index'] < args.num_of_labelled_target tgt_masks, tgt_use_masks = Variable(tgt_masks), Variable( tgt_use_masks) if torch.cuda.is_available():
def train(model, optimizer, criterion, dataset, train_params, test_data=None): max_iter = train_params['max_iter'] seed = train_params['seed'] class_rate = False #_utils.touch_pytorch(train_params['logdir']) torch.manual_seed(seed) loss = np.zeros(max_iter) if train_params.has_key('class_rate') and train_params['class_rate']: class_rate_train = np.zeros(max_iter) class_rate_test = np.zeros(max_iter) class_rate = True bar = progressbar.ProgressBar(max_value=max_iter) for n in range(max_iter): # loop over the dataset multiple times data_batch = dataset.get_next_batch() if train_params['cuda']: data_batch['x'] = data_batch['x'].cuda() data_batch['y'] = data_batch['y'].cuda() inputs = Variable(data_batch['x']) targets = Variable(data_batch['y']) # zero the parameter gradients optimizer.zero_grad() # forward + backward + optimize outputs = model(inputs) l = criterion(outputs, targets) # print('loss {} {}'.format(n, l.data[0])) # print('outputs {} targets {}'.format(outputs.data[0], targets.data[0])) # print('x1 {} x2 {}'.format(model.x1.data[0], model.x2.data[0])) loss[n] = l.data[0] if class_rate: class_rate_train[n] = compute_class_rate(outputs, targets) if test_data is not None: class_rate_test[n] = compute_test_class_rate( test_data, model, train_params['cuda']) l.backward() optimizer.step() if train_params['debug'] and not train_params['cuda']: if hasattr(model, 'forward_pass_data'): file_path = os.path.join(train_params['logdir'], "forward_pass_{}.pkl".format(n)) #_tf_utils.pkl_save(file_path, model.forward_pass_data) file_path = os.path.join(train_params['logdir'], "loss_{}.pkl".format(n)) #_tf_utils.pkl_save(file_path, {'loss': loss[n]}) bar.update(n) print('Finished Training') res = {} res['loss'] = loss if class_rate: res['class_rate_train'] = class_rate_train res['class_rate_test'] = class_rate_test return res
def filter_parallel_onenode( ct_series, output_img_series, desc, filter_one, **kwds): """ * ct_series: an image series for ct scan * output_img_series: output image series * desc: description of the filter action * filter_one: function to filter one image * kwds: additional kargs for filter_one """ import logging; logger = logging.getLogger("mpi") from mpi4py import MPI comm = MPI.COMM_WORLD size = comm.Get_size() rank = comm.Get_rank() # start processing at different time time.sleep(rank*0.2) # create the output dir dir = os.path.dirname(output_img_series.filename_template) if not os.path.exists(dir): if rank == 0: os.makedirs(dir) else: created = False for i in range(WAIT_COUNT): time.sleep(WAIT_SECONDS) if os.path.exists(dir): created = True break continue if not created: raise IOError("Waited %s seconds, %s still not created" % ( WAIT_COUNT*WAIT_SECONDS, dir)) # prefix = "%s %s:" % (desc, ct_series.name or "") if rank==0: logger.info( "Running %s on %s" % (desc, ct_series.name) ) pass totalN = ct_series.nImages # number of images to process in this process N = int(np.ceil(totalN*1. / size)) # start and stop index start, stop = rank*N, min(totalN, (rank+1)*N) # progress bar # for simplicity, we just report the progress at rank 0, which should be a # good indication of progress of all nodes any way if rank==0: bar = progressbar.ProgressBar( widgets=[ prefix, progressbar.Percentage(), progressbar.Bar(), ' [', progressbar.ETA(), '] ', ], max_value = stop-start-1, **pb_config ) for i, angle in enumerate(ct_series.identifiers[start: stop]): # sys.stderr.write("%s: %s\n" % (prefix, angle)) # skip over existing results if not output_img_series.exists(angle): data = ct_series.getData(angle) output_img_series.putImage(angle, filter_one(data, **kwds)) if rank==0: bar.update(i) continue comm.Barrier() if rank==0: print('\n') return
curtailment = pd.DataFrame(columns=columns, index=["curtailment"]) widgets = [ pgb.widgets.Percentage(), " ", pgb.widgets.SimpleProgress(format="(%s)" % pgb.widgets.SimpleProgress.DEFAULT_FORMAT), " ", pgb.widgets.Bar(), " ", pgb.widgets.Timer(), " ", pgb.widgets.ETA(), ] progressbar = pgb.ProgressBar(prefix="Extract results: ", widgets=widgets, max_value=len(snakemake.input)) for k, v in progressbar(columns_dict.items()): ids = v.values() n = pypsa.Network(k) investment_results[(*ids, )] = get_investments(n) gen_capacity_mix[(*ids, )] = get_capacity_mix(n.generators) sto_capacity_mix[(*ids, )] = get_capacity_mix(n.storage_units) energy_mix[(*ids, )] = get_energy_mix(n) line_transmission_capacity[(*ids, )] = get_transmission(n, "lines") line_transmission_volume[(*ids, )] = get_transmission(n, "lines", length=True) link_transmission_capacity[(*ids, )] = get_transmission(n, "links")
count_params(trainable_variables_rgb) + count_params(trainable_variables_depth)) + \ "\n{} Generate features from training set".format(datetime.now()) sLog(makelog, simpleLog) tb_train_count = 0 tb_val_count = 0 best_loss_val = 1000 # Loop over number of epochs num_samples = 0 # Training set sess.run(training_init_op) # Progress bar setting bar = progressbar.ProgressBar(maxval=tr_batches_per_epoch, widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()]) bar.start() train_loss = 0 train_acc = 0 for i in range(tr_batches_per_epoch): bar.update(i + 1) tb_train_count += 1 rgb_batch, depth_batch, label_batch = sess.run(next_batch) num_samples += np.shape(rgb_batch)[0] feed_dict = {x_rgb: rgb_batch, x_depth: depth_batch, y: label_batch, keep_prob: (1 - do), training_phase: True, K.learning_phase(): 1} batch_loss, summary = sess.run([loss, summary_op], feed_dict=feed_dict) train_loss += batch_loss train_writer.add_summary(summary, tb_train_count)
def _maybe_convert_sets(target_dir, extracted_data): extracted_dir = os.path.join(target_dir, extracted_data) # override existing CSV with normalized one target_csv_template = os.path.join( target_dir, ARCHIVE_DIR_NAME, ARCHIVE_NAME.replace(".tar.gz", "_{}.csv")) if os.path.isfile(target_csv_template): return wav_root_dir = os.path.join(extracted_dir) all_files = [ "transcripts/train/yaounde/fn_text.txt", "transcripts/train/ca16_conv/transcripts.txt", "transcripts/train/ca16_read/conditioned.txt", "transcripts/dev/niger_west_african_fr/transcripts.txt", "speech/dev/niger_west_african_fr/niger_wav_file_name_transcript.tsv", "transcripts/devtest/ca16_read/conditioned.txt", "transcripts/test/ca16/prompts.txt", ] transcripts = {} for tr in all_files: with open(os.path.join(target_dir, ARCHIVE_DIR_NAME, tr), "r") as tr_source: for line in tr_source.readlines(): line = line.strip() if ".tsv" in tr: sep = " " else: sep = " " audio = os.path.basename(line.split(sep)[0]) if not (".wav" in audio): if ".tdf" in audio: audio = audio.replace(".tdf", ".wav") else: audio += ".wav" transcript = " ".join(line.split(sep)[1:]) transcripts[audio] = transcript # Get audiofile path and transcript for each sentence in tsv samples = [] glob_dir = os.path.join(wav_root_dir, "**/*.wav") for record in glob(glob_dir, recursive=True): record_file = os.path.basename(record) if record_file in transcripts: samples.append((record, transcripts[record_file])) # Keep track of how many samples are good vs. problematic counter = get_counter() num_samples = len(samples) rows = [] print("Importing WAV files...") pool = Pool() bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR) for i, processed in enumerate(pool.imap_unordered(one_sample, samples), start=1): counter += processed[0] rows += processed[1] bar.update(i) bar.update(num_samples) pool.close() pool.join() with open(target_csv_template.format("train"), "w", encoding="utf-8", newline="") as train_csv_file: # 80% with open(target_csv_template.format("dev"), "w", encoding="utf-8", newline="") as dev_csv_file: # 10% with open(target_csv_template.format("test"), "w", encoding="utf-8", newline="") as test_csv_file: # 10% train_writer = csv.DictWriter(train_csv_file, fieldnames=FIELDNAMES) train_writer.writeheader() dev_writer = csv.DictWriter(dev_csv_file, fieldnames=FIELDNAMES) dev_writer.writeheader() test_writer = csv.DictWriter(test_csv_file, fieldnames=FIELDNAMES) test_writer.writeheader() for i, item in enumerate(rows): transcript = validate_label(item[2]) if not transcript: continue wav_filename = item[0] i_mod = i % 10 if i_mod == 0: writer = test_writer elif i_mod == 1: writer = dev_writer else: writer = train_writer writer.writerow( dict( wav_filename=wav_filename, wav_filesize=os.path.getsize(wav_filename), transcript=transcript, )) imported_samples = get_imported_samples(counter) assert counter["all"] == num_samples assert len(rows) == imported_samples print_import_report(counter, SAMPLE_RATE, MAX_SECS)
transform_to_lfw_format) # pylint: disable=no-member from .config import AlignConfig # pylint: disable=no-member warnings.filterwarnings("ignore", message="numpy.dtype size changed") warnings.filterwarnings("ignore", message="numpy.ufunc size changed") os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' tf.logging.set_verbosity(tf.logging.ERROR) WIDGETS = ['Aligning Dataset: ', pb.Percentage(), ' ', pb.Bar(), ' ', pb.ETA()] TIMER = pb.ProgressBar(widgets=WIDGETS) NUM_SUCESSFUL = Value(c_int) # defaults to 0 NUM_SUCESSFUL_LOCK = Lock() NUM_UNSECESSFUL = Value(c_int) NUM_UNSUCESSFUL_LOCK = Lock() NUM_IMAGES_TOTAL = Value(c_int) NUM_IMAGES_TOTAL_LOCK = Lock() def align_dataset(config_file: str): """Aligns an image dataset """ config = AlignConfig(config_file) output_dir = os.path.expanduser(config.output_dir) os.makedirs(output_dir, exist_ok=True)
#initialize the image preprocessors and the list of RGB channel averages aap = AspectAwarePreprocessor(width=256, height=256) itap = ImageToArrayPreprocessor(data_format=None) R, G, B = [], [], [] #loop over the datasets tuples for dType, paths, labels, outputPath in datasets: #create HDF5 writer print('Building {}...'.format(outputPath)) writer = HDF5DatasetWriter(dims=(len(paths), 256, 256, 3), outputPath=outputPath) # initialize the progress bar widgets = ["Building Dataset: ", progressbar.Percentage(), " ", progressbar.Bar(), " ", progressbar.ETA()] pbar = progressbar.ProgressBar(maxval=len(paths), widgets=widgets).start() #loop over the image paths for i, (path, label) in enumerate(zip(paths, labels)): #load the image and preprocess it image = cv2.imread(path) image = aap.preprocess(image) image = itap.preprocess(image) #compute the mean of each channel in each image in the training set and update the lists if dType=='train': g, b, r = cv2.mean(image)[:3] B.append(b) G.append(g) R.append(r) #add the image and label to the hdf5 writer
def parmap(f, iterable, nprocs=multiprocessing.cpu_count(), show_progress=True, size=None): """ @param f function to be applied to the items in iterable @param iterable ... @param nprocs number of processes @param show_progress True <-> show a progress bar @param size number of items in iterable. If show_progress == True and size is None and iterable is not already a list, it is converted to a list first. This could be bad for generators! (If size is not needed right away for the progress bar, all input items are enqueued before reading the results from the output queue.) TLDR: If you know it, tell us the size of your iterable. """ q_in = multiprocessing.Queue(1) q_out = multiprocessing.Queue() progress = None if show_progress: if not isinstance(iterable, list): iterable = list(iterable) size = len(iterable) widgets = [ progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA() ] progress = progressbar.ProgressBar(widgets=widgets, maxval=size) proc = [ multiprocessing.Process(target=spawn(f), args=(q_in, q_out)) for _ in range(nprocs) ] for p in proc: p.daemon = True p.start() if progress is not None: progress.start() def enqueue(): s = 0 for i, x in enumerate(iterable): q_in.put((i, x)) s += 1 for _ in range(nprocs): q_in.put((None, None)) return s pool = ThreadPool(processes=1) async_size = pool.apply_async(enqueue) if size is None: # this is the old behavior size = async_size.get() res = [] progress_value = 0 for _ in range(size): r = q_out.get() res.append(r) # we could: insert sorted, yield all results we have so far if progress is not None: progress_value += 1 progress.update(progress_value) del pool for p in proc: p.join() if progress is not None: progress.finish() return [x for _, x in sorted(res)]
def train(self): #write to file feat_trix = pickle.load(open(self.school + ".timetable.features", "rb")) #prep sizes num_fts = feat_trix.shape[1] num_tts = feat_trix.shape[0] print "Training {0} on {1} features, {2} timetables".format( self.school, num_fts, num_tts) #dictionary mapping from coursef #to tuple: (related course, similarity) similarities = {} # For each Course, C1 # For each Timetable TT with Course C1 # For each Course C2 in Timetable TT # Record that a Timetable has courses C1 and C2 # For each Course C2 # Compute the similarity between C1 and C2 print "EXECUTING FIRST PASS" bar = progressbar.ProgressBar() low_data_courses = {} for c1 in bar(range(num_fts)): similar = set() c1_rows = filter(lambda ptt_idx: feat_trix[ptt_idx, c1], range(num_tts)) for tt in c1_rows: row = feat_trix[tt].toarray()[0] similar = similar.union(set(np.where(row > 0)[0].flatten())) #use len c1_rows < 5 OR len similar < 3 if len(similar) > 0 and (len(c1_rows) < 5 or len(similar) < 3): low_data_courses[c1] = [] for c2 in similar: css = self.compute_similarity(feat_trix[:, c1].toarray(), feat_trix[:, c2].toarray()) if c1 not in similarities: similarities[c1] = [] similarities[c1].append((c2, css)) if c1 in low_data_courses: low_data_courses[c1].append((c2, css)) #low data pass print "EXECUTING SECOND PASS" bar2 = progressbar.ProgressBar() for c1, similar_css in bar2(low_data_courses.items()): agg_sims = {} norm_factor = {} for c2, css in similar_css: if c2 not in agg_sims: agg_sims[c2] = css norm_factor[c2] = 1 for c3, css3 in filter(lambda x: x[0] != c2, similarities[c2]): if c3 not in agg_sims: agg_sims[c3] = css norm_factor[c3] = 1 agg_sims[c3] += css3 norm_factor[c3] += 1 similarities[c1] = [] for c in agg_sims: similarities[c1].append((c, agg_sims[c] / norm_factor[c])) pickle.dump(similarities, open(self.school + ".recommended.model", "wb"))
cv2.namedWindow('result', cv2.WINDOW_NORMAL) cv2.resizeWindow('result', (int(width/3), int(height/3))) cv2.moveWindow("result", 1000,1000); images.sort() images.reverse() result_img = None old_img = None counter = 0 diff = None # bar = progressbar.progressbar(range(len(images))) print("Processing", len(images), "images...") with progressbar.ProgressBar(max_value=len(images)) as bar: for image in images: counter += 1 if result_img is None: result_img = cv2.imread(os.path.join(image_folder, image)) old_img = result_img else: new = cv2.imread(os.path.join(image_folder, image)) # new = cv2.GaussianBlur(new, (3,3), 0, 0 ) # result_img[:,:,1] = max(new[:,:,1], result_img[:,:,1]) # result_img[:,:,2] = max(new[:,:,2], result_img[:,:,2]) # diff = cv2.absdiff(new, old_img)
def main(): print("WELCOME TO ZEPHYRUS.") # choose user to recommend movies to print("\nChoose user (ID): ", end="") user = int(input()) # read dataset movies_data = pandas.read_csv("csv/movies_data.csv") test_data = pandas.read_csv("csv/test_data.csv") train_data = pandas.read_csv("csv/train_data.csv") movie_reviews = pandas.read_csv("csv/movie_reviews.csv") # initialize our data matrix (0 = unknown rating) n_users = train_data['user_id'].max() n_items = movies_data['movie_id'].max() # generate (user x movie) ratings matrix print("\nGenerating user x movie ratings matrix...") ratings = np.full((n_users, n_items), 0) for row in train_data.itertuples(): user_id = getattr(row, "user_id") movie = getattr(row, "movie_id") rating = getattr(row, "rating") ratings[user_id - 1][movie - 1] = rating # calculate biases # print("Calculating biases...") # global_avg = global_average(ratings) # print("Global Average: {}".format(global_avg)) # bu = bias_user(ratings, user, global_avg) # print("Bias User: {}".format(bu)) # bi = [] # print("Calculating biases for every item...") # with progressbar.ProgressBar(max_value=n_items) as bar: # for i in range(n_items): # bi.append(bias_item(ratings, i, global_avg)) # bar.update(i) # calculate prediction for every item for chosen user print("\nPredicting ratings with RF_Rec...") predictions = np.zeros(n_items) with progressbar.ProgressBar(max_value=n_items) as bar: for i in range(n_items): # predictions[i] = baseline(bu, bi[i], global_avg) predictions[i] = RF_Rec(ratings, user, i) bar.update(i) # sorting and getting top rated items # sorted_predictions = predictions.argsort()[:5] sorted_predictions = nlargest(10, enumerate(predictions), key=lambda x: x[1]) print("Top 10 predictions:") print(sorted_predictions) rf_rec_results = [] for prediction in sorted_predictions: rf_rec_results.append(prediction[0]) print("\nRF_Rec results:") for movie in rf_rec_results: title = movies_data['title'][movie] print("* " + title) # use these for FBC-Knn start = timer() fbc_knn(rf_rec_results, movie_reviews, movies_data) end = timer() time_elapsed = end - start print("\nElapsed time for FBC-Knn: {}".format(time_elapsed))
def extract_data(args): ''' Extracts data from .bag file given. ''' # open the bag file print('Loading .bag file') bag = rosbag.Bag(args['bag'], 'r') # assumption: # the matching message will have been received within 150 ms of the pivot message # this is used to narrow the search for the matching message t_tol = rospy.Duration(secs=0, nsecs=150000000) # 150000000 ns = 150 ms # start and end time for the data extraction start_at = rospy.Time(secs=int(bag.get_start_time()) + 1, nsecs=0) # 1 second offset for the start end_at = rospy.Time(secs=int(bag.get_end_time()) - 1, nsecs=0) # -1 second offset for the end # pose is the slowest topic pose_bagmsgs = bag.read_messages(topics='pose', start_time=start_at, end_time=end_at) # joy is faster and is out of sync, which is why a tolerance is needed joy_bagmsgs = bag.read_messages(topics='joy', start_time=start_at - t_tol, end_time=end_at + t_tol) # load the messages into memory print('Loading messages into memory') pose_bagmsgs = [p for p in pose_bagmsgs] joy_bagmsgs = [j for j in joy_bagmsgs] # last messages saved. this will be used to obtain incremental # measures and ground truth for joystick measures last_pose_msg = None last_joy_msg = None second_last_pose_msg = None second_last_joy_msg = None # setup the progress bar c = 0 c_max = len(pose_bagmsgs) bar = progressbar.ProgressBar(max_value=c_max) # create pandas dataframe for the data data = pd.DataFrame(columns=[ 'dtns', # 'dts', 'dtns' # 'dx', 'dy', 'dq.w', 'dq.z' 'x', 'y', #### 'vx', 'dvx', 'vz', 'dvz', 'jx', 'djx', 'jy', 'djy', # 'jb', 'djb' 'jx_gt', 'jy_gt' # 'jb_gt' ]) # counter for the joy list joy_idx = 0 # read messages from the pose topic print('Processing and saving data') for pose_bagmsg in pose_bagmsgs: # get the pose message and it's timestamp pose_msg = pose_bagmsg.message pose_t = pose_bagmsg.timestamp # pose_msg.header.stamp # when it was generated/captured # read from the joy topic joy_msg = None min_diff = rospy.Duration( secs=10, nsecs=0) # starting the minimum with 10 seconds for joy_bagmsg in joy_bagmsgs[joy_idx:]: joy_t = joy_bagmsg.timestamp # joy_bagmsg.message.header.stamp diff_t = abs(pose_t - joy_t) if diff_t < min_diff or joy_msg is None: joy_msg = joy_bagmsg.message min_diff = diff_t joy_idx += 1 else: break # to proceed, we need a last message for pose and joy if last_pose_msg is not None and last_joy_msg is not None: # and we also need a second last to get the ground truth if second_last_pose_msg is not None and second_last_joy_msg is not None: # extract the relevant data from the messages row = {} # datetime row['datetime'] = datetime.datetime.fromtimestamp(pose_t.secs) \ + datetime.timedelta(microseconds=pose_t.nsecs//1000) # time ti = last_pose_msg.header.stamp - second_last_pose_msg.header.stamp # row['dts'] = ti.secs row['dtns'] = ti.nsecs # position ''' po = second_last_pose_msg.pose.pose.position pn = last_pose_msg.pose.pose.position row['dx'] = pn.x - po.x row['dy'] = pn.y - po.y ''' row['x'] = last_pose_msg.pose.pose.position.x row['y'] = last_pose_msg.pose.pose.position.y # orientation ''' qo = second_last_pose_msg.pose.pose.orientation qn = last_pose_msg.pose.pose.orientation qo = np.quaternion(qo.w, qo.x, qo.y, qo.z) qn = np.quaternion(qn.w, qn.x, qn.y, qn.z) qi = qn/qo row['dq.w'] = qi.w row['dq.z'] = qi.z ''' # velocity row['vx'] = last_pose_msg.twist.twist.linear.x row['dvx'] = last_pose_msg.twist.twist.linear.x - second_last_pose_msg.twist.twist.linear.x row['vz'] = last_pose_msg.twist.twist.angular.z row['dvz'] = last_pose_msg.twist.twist.angular.z - second_last_pose_msg.twist.twist.angular.z # joy (data) row['jx'] = last_joy_msg.axes[0] row['djx'] = last_joy_msg.axes[0] - second_last_joy_msg.axes[0] row['jy'] = last_joy_msg.axes[1] row['djy'] = last_joy_msg.axes[1] - second_last_joy_msg.axes[1] # row['jb'] = last_joy_msg.buttons[0] # row['djb'] = last_joy_msg.buttons[0] - second_last_joy_msg.buttons[0] # joy (ground-truth) row['jx_gt'] = joy_msg.axes[0] row['jy_gt'] = joy_msg.axes[1] # row['jb_gt'] = joy_msg.buttons[0] # append the row to the data data = data.append(pd.DataFrame(row, index=[0]), ignore_index=True, sort=False) # save the messages for the next iteration second_last_pose_msg = last_pose_msg second_last_joy_msg = last_joy_msg # save the messages for the next iteration last_pose_msg = pose_msg last_joy_msg = joy_msg # update the progress c += 1 bar.update(c) # mark the processing as finished bar.finish() # save the data to disk set_folder = os.path.join('data', args['dest'], args['set']) if not os.path.isdir(set_folder): os.makedirs(set_folder) if args['append']: data = pd.concat( [pd.read_csv(os.path.join(set_folder, 'data.csv')), data]) data.to_csv(path_or_buf=os.path.join(set_folder, 'data.csv'), header=True, index=False) # close the bag file bag.close()
def optimize(loss_function, model, optimizer_str, lr=1e-4, nb_epochs=1000, verbose=True, num_workers=0, max_norm=1.0, **kargs): """ Providing the schema of the iterative method for optimizing a differentiable objective function for models that use gradient centric schemas (a.k.a order 1 optimization) Parameters: ----------- * loss_function: function Loss function of the model * model: torch object Actual model to optimize * optimizer_str: str Defines the type of optimizer to use. Here are the possible options: - adadelta - adagrad - adam - adamax - rmsprop - sparseadam - sgd * lr: float (default=1e-4) learning rate used in the optimization * nb_epochs: int (default=1000) The number of iterations in the optimization * verbose: bool (default=True) Whether or not producing detailed logging about the modeling * max_norm: float (default=1.0) Max l2 norm for gradient clipping """ # Choosing an optimizer W = model.parameters() if optimizer_str.lower() == 'adadelta': optimizer = torch.optim.Adadelta(W, lr=lr) elif optimizer_str.lower() == 'adagrad': optimizer = torch.optim.Adagrad(W, lr=lr) elif optimizer_str.lower() == 'adam': optimizer = torch.optim.Adam(W, lr=lr) elif optimizer_str.lower() == 'adamax': optimizer = torch.optim.Adamax(W, lr=lr) elif optimizer_str.lower() == 'rmsprop': optimizer = torch.optim.RMSprop(W, lr=lr) elif optimizer_str.lower() == 'sparseadam': optimizer = torch.optim.SparseAdam(W, lr=lr) elif optimizer_str.lower() == 'sgd': optimizer = torch.optim.SGD(W, lr=lr) elif optimizer_str.lower() == 'lbfgs': optimizer = torch.optim.LBFGS(W, lr=lr) elif optimizer_str.lower() == 'rprop': optimizer = torch.optim.Rprop(W, lr=lr) else: error = "{} optimizer isn't implemented".format(optimizer_str) raise NotImplementedError(error) # Initializing the Progress Bar loss_values = [] if verbose: widgets = [ '% Completion: ', progressbar.Percentage(), progressbar.Bar('*'), '' ] bar = progressbar.ProgressBar(maxval=nb_epochs, widgets=widgets) bar.start() # Updating the weights at each training epoch temp_model = None for epoch in range(nb_epochs): # Backward pass and optimization def closure(): optimizer.zero_grad() loss = loss_function(model, **kargs) #print(f"loss raw: {loss}") #loss = loss * 1/1000000 #print(f"loss: {loss}") loss.backward() clip_grad_norm_(model.parameters(), max_norm) return loss if 'lbfgs' in optimizer_str.lower(): optimizer.step(closure) else: optimizer.step() loss = closure() loss_value = loss.item() # Printing error message if the gradient didn't explode if np.isnan(loss_value) or np.isinf(loss_value): error = "The gradient exploded... " error += "You should reduce the learning" error += "rate (lr) of your optimizer" if verbose: widgets[-1] = error else: print(error) break # Otherwise, printing value of loss function else: temp_model = copy.deepcopy(model) loss_values.append(loss_value) if verbose: widgets[-1] = "Loss: {:6.2f}".format(loss_value) # Updating the progressbar if verbose: bar.update(epoch + 1) # Terminating the progressbar if verbose: bar.finish() # Finilazing the model if temp_model is not None: temp_model = temp_model.eval() model = copy.deepcopy(temp_model) else: raise ValueError(error) return model, loss_values
def main(): parser = argparse.ArgumentParser(description='Interactive input') parser.add_argument('-n', default=1, type=checkSize, help='Size of a file(default: 1Mb)') parser.add_argument('--k', nargs='+', type=checkTuple, default=(10, 100), help='Amount of words in a string(default: (10, 100))') parser.add_argument('--l', nargs='+', type=checkTuple, default=(3, 10), help='Length of a word(default: (3, 10))') parser.add_argument('--name', default='output', help='Give a name to output file.') args = parser.parse_args() k = tuple(args.k) rand_for_str = [] for i in range(k[0], k[1] + 1): rand_for_str.append(i) checkDimension(k) l = tuple(args.l) rand_for_word = [] for i in range(l[0], l[1] + 1): rand_for_word.append(i) checkDimension(l) file_name = args.name file = open(file_name + ".txt", "w") alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' n = args.n length = random.choice(rand_for_word) string_len = random.choice(rand_for_str) spaces = 0 size_in_bytes = 0 str_help = 0 bar = progressbar.ProgressBar(maxval=int(1024**2 * n + 1), widgets=[ 'Working...: ', progressbar.Bar(left='[', marker='*', right=']'), progressbar.Percentage(), ]).start() start_time = time.time() while size_in_bytes < int(1024**2 * n): if size_in_bytes != 0 and spaces != 0 and (spaces % (string_len) == 0): spaces = 0 file.write('\n') size_in_bytes += 2 string_len = random.choice(rand_for_str) file.write(random.choice(alphabet)) size_in_bytes += 1 str_help += 1 if str_help % length == 0: spaces += 1 file.write(' ') size_in_bytes += 1 length = random.choice(rand_for_word) str_help = 0 bar.update(size_in_bytes) bar.finish() file.close() print("Total time complexity", (time.time() - start_time))
def run(template_traces: np.ndarray, template_keys: np.ndarray, template_plaintext: np.ndarray, attack_traces: np.ndarray, attack_keys: np.ndarray, attack_plain: np.ndarray, pooled: bool, num_points_of_interest: int, spacing_points_of_interest: int, subkey: int, gpu: bool = False, leakage_model: bool = True, debug_mode_enabled: bool = False, feature_select: int = 0) -> np.array: """ Method used to select correct version of Template Attack. :param template_traces: the traces to use :param template_keys: the keys to use :param template_plaintext: the plaintexts to use :param attack_traces: the traces to use for attacking :param attack_keys: the keys to use for attacking :param attack_plain: the plaintexts to use for attacking :param pooled: whether to do a pooled attack :param num_points_of_interest: number of points of interest to use :param spacing_points_of_interest: spacing between the points of interest :param subkey: the subkey index to analyze. Must be in the range [0-16]. 16 signals the full key. :param gpu: whether or not to use gpu for this attack :param leakage_model: the leakage model to use :param debug_mode_enabled: whether to enable debug mode :param feature_select: which feature selection method to use, see main for which number is which. :return: array containing the calculated key """ # Init progress bar with rough amount of iterations num_subkeys = 16 if subkey == 16 else 1 # Attack takes roughly equal time per subkey so * 16 for whole key max_value = len(attack_traces) * 256 * num_subkeys bar = progressbar.ProgressBar( max_value=max_value, widgets=progress_bar_util.get_widgets(debug_mode_enabled)) ta = TA(template_traces, template_keys, template_plaintext, attack_traces, attack_keys, attack_plain) indices = [] if feature_select > 0: print("Feature selection is being calculated...") if num_subkeys == 16: for i in range(16): temp = DataPartitioner.select_features( template_traces, template_keys, template_plaintext, i, feature_select, num_points_of_interest, 1, 0, leakage_model) for j in temp: indices.append(j) else: # Select at least 10 features num_features = max(num_points_of_interest, 10) indices = DataPartitioner.select_features( template_traces, template_keys, template_plaintext, subkey, feature_select, num_features, 1, 0, leakage_model) ta.template_traces, ta.attack_traces = template_traces[:, indices], attack_traces[:, indices] if pooled: result = ta.run_pooled(num_points_of_interest, spacing_points_of_interest, subkey, bar, gpu, leakage_model, debug_mode_enabled) else: result = ta.run_normal(num_points_of_interest, spacing_points_of_interest, subkey, bar, gpu, leakage_model, debug_mode_enabled) bar.finish() print('The final key is: ', result) return result
def create_benchmark_archive(blk_hash): blk = json.loads( subprocess.check_output([ZELCASH_CLI, 'getblock', blk_hash])) print 'Height: %d' % blk['height'] print 'Transactions: %d' % len(blk['tx']) os.mkdir('benchmark') with open('benchmark/block-%d.dat' % blk['height'], 'wb') as f: f.write( binascii.unhexlify( subprocess.check_output( [ZELCASH_CLI, 'getblock', blk_hash, 'false']).strip())) txs = [ json.loads( subprocess.check_output( [ZELCASH_CLI, 'getrawtransaction', tx, '1'])) for tx in blk['tx'] ] js_txs = len([tx for tx in txs if len(tx['vjoinsplit']) > 0]) if js_txs: print 'Block contains %d JoinSplit-containing transactions' % js_txs return inputs = [(x['txid'], x['vout']) for tx in txs for x in tx['vin'] if x.has_key('txid')] print 'Total inputs: %d' % len(inputs) unique_inputs = {} for i in sorted(inputs): if unique_inputs.has_key(i[0]): unique_inputs[i[0]].append(i[1]) else: unique_inputs[i[0]] = [i[1]] print 'Unique input transactions: %d' % len(unique_inputs) db_path = 'benchmark/block-%d-inputs' % blk['height'] db = plyvel.DB(db_path, create_if_missing=True) wb = db.write_batch() bar = progressbar.ProgressBar(redirect_stdout=True) print 'Collecting input coins for block' for tx in bar(unique_inputs.keys()): rawtx = json.loads( subprocess.check_output( [ZELCASH_CLI, 'getrawtransaction', tx, '1'])) mask_size = 0 mask_code = 0 b = 0 while 2 + b * 8 < len(rawtx['vout']): zero = True i = 0 while i < 8 and 2 + b * 8 + i < len(rawtx['vout']): if 2 + b * 8 + i in unique_inputs[tx]: zero = False i += 1 if not zero: mask_size = b + 1 mask_code += 1 b += 1 coinbase = len(rawtx['vin']) == 1 and 'coinbase' in rawtx['vin'][0] first = len(rawtx['vout']) > 0 and 0 in unique_inputs[tx] second = len(rawtx['vout']) > 1 and 1 in unique_inputs[tx] code = 8*(mask_code - (0 if first or second else 1)) + \ (1 if coinbase else 0) + \ (2 if first else 0) + \ (4 if second else 0) coins = bytearray() # Serialized format: # - VARINT(nVersion) coins.extend(encode_varint(rawtx['version'])) # - VARINT(nCode) coins.extend(encode_varint(code)) # - unspentness bitvector, for vout[2] and further; least significant byte first for b in range(mask_size): avail = 0 i = 0 while i < 8 and 2 + b * 8 + i < len(rawtx['vout']): if 2 + b * 8 + i in unique_inputs[tx]: avail |= (1 << i) i += 1 coins.append(avail) # - the non-spent CTxOuts (via CTxOutCompressor) for i in range(len(rawtx['vout'])): if i in unique_inputs[tx]: coins.extend( encode_varint( compress_amount(int(rawtx['vout'][i]['valueZat'])))) coins.extend( compress_script( binascii.unhexlify( rawtx['vout'][i]['scriptPubKey']['hex']))) # - VARINT(nHeight) coins.extend( encode_varint( json.loads( subprocess.check_output( [ZELCASH_CLI, 'getblockheader', rawtx['blockhash']]))['height'])) db_key = b'c' + bytes(binascii.unhexlify(tx)[::-1]) db_val = bytes(coins) wb.put(db_key, db_val) wb.write() db.close() # Make reproducible archive os.remove('%s/LOG' % db_path) files = subprocess.check_output(['find', 'benchmark']).strip().split('\n') archive_name = 'block-%d.tar' % blk['height'] tar = tarfile.open(archive_name, 'w') for name in sorted(files): tar.add(name, recursive=False, filter=deterministic_filter) tar.close() subprocess.check_call(['xz', '-6', archive_name]) print 'Created archive %s.xz' % archive_name subprocess.call(['rm', '-r', 'benchmark'])
def new_data() -> None: """ Function for generating images and XML markup and storing this data in the directories specified in the config file :return: None """ # array with full parameters for each class item = [] random.seed() # getting class names and their number from a file with open(config.file_with_classnames, 'r') as file: for line in file: str_ = line.split(' ') name = str_[0] count = str_[1][:-1] item.append(it.Item(name, count)) # function for converting the source instances of each class to the initial dimensions change_size(item) # current class number index = 0 # creating a variable responsible for the file number _count_ = 1 print('\n*****Create image and xml for pix: *****') number_classes = len(item) # the list of accounting for the number of elements of each class list_count_of_each_class = [0] * number_classes progress_count = 0 # counter of the total number of objects in the future dataset count_of_image = 0 name_to_xml = str(_count_) # cycle through all class parameters for x in item: count_of_image += int(x.count) bar = progressbar.ProgressBar(maxval=count_of_image, widgets=config.widgets).start() # flag that is responsible for stopping the formation of the dataset flag = True # cycle for creating a dataset while flag: # a tuple that stores the size of an image size_ = (config.image_size['width'], config.image_size['height']) # creating an xml file root = Element("annotation") file = SubElement(root, "filename") file.text = str(name_to_xml + '.jpg') path = SubElement(root, "path") path.text = str( f'{config.dir_output_data}/{config.dir_result_img}/{name_to_xml}.jpg' ) size_tree = SubElement(root, "size") width = SubElement(size_tree, "width") width.text = str(int(size_[0])) height = SubElement(size_tree, "height") height.text = str(int(size_[1])) depth = SubElement(size_tree, "depth") depth.text = "3" # creating auxiliary arrays object_, name, xmin_, xmax_, ymax_, ymin_, bndbox = [], [], [], [], [], [], [] # creating a new image with the specified dimensions and a black background img = Image.new( "RGB", (config.image_size['width'], config.image_size['height']), '#191919') # flag that controls the fullness of the dataset flag_end = True # variable responsible for the number of files created end_file = 0 # auxiliary variable for accessing arrays for xml markup number = 0 # creating a matrix for the location of class objects matrix = create_matrix() # checking for boundary conditions while flag_end: end = 0 if index >= number_classes: index = 0 if list_count_of_each_class[index] >= int(item[index].count): while list_count_of_each_class[index] >= int( item[index].count): index += 1 end += 1 if index == number_classes: index = 0 if end == number_classes: flag = False break if not flag: break coeff_orientation = None # orientation check to generate the orientation coefficient if item[index].orientation == 0: # original coeff_orientation = 1 # elif(item[index].orientation == 1): # horizontal # coeff_orientation = random.randrange(70, 80, 1) / 100 # elif(item[index].orientation == 2): # vertical # coeff_orientation = 100 / random.randrange(70, 80, 1) # if (item[index].current_size[1] + item[index].config.step_change_size <= item[index].size_finish[1]): # item[index].orientation = (item[index].orientation + 1 if (item[index].orientation < 2) else 0) # the number of occupied cells in the matrix by width count_of_x_boxes_item = int(( item[index].current_size[1] // config.step['width'] + 1) if item[index].current_size[1] % config.step['width'] > 0 else int(item[index].current_size[1] // config.step['width'])) # the number of occupied cells in the matrix by height count_of_y_boxes_item = int(( item[index].current_size[0] // config.step['height'] + 1) if item[index].current_size[0] % config.step['height'] > 0 else int(item[index].current_size[0] // config.step['height'])) # the path to the prepared image of the class object full_file_name = os.path.join(config.work_dir, config.dir_output_data, config.dir_prepare_boxes, f'{item[index].name}.png') # the path to the prepared image of the class object opening the image and converting to RGBA button = Image.open(full_file_name).convert("RGBA") # flag for determining whether an object of the class will be added to the image done_item_box = False # cycle through the matrix in height for i in range(4): global j # cycle through the matrix in width for j in range(4): # flag for adding a class object to an image is_fit = True # checking that an object of the class with the new dimensions is placed on the image if (count_of_x_boxes_item + j < 5) and (count_of_y_boxes_item + i < 5): # cycle through the allowed cells in height for y in range(i, i + count_of_y_boxes_item): # cycle through the allowed cells in width for x in range(j, j + count_of_x_boxes_item): # checking that the cells of the matrix in which we want to place the object are free if matrix[y, x] > 0: # we transfer the flag to the stage of the impossibility of adding it is_fit = False break if not is_fit: break # checking whether an object of the class can be added to the image if is_fit: # cycle through the allowed cells in height for y in range(i, i + count_of_y_boxes_item): # cycle through the allowed cells in width for x in range(j, j + count_of_x_boxes_item): # note in the matrix that these cells are filled in matrix[y, x] = 1 # setting the flag for adding a class object to the true state done_item_box = True break # checking whether an object of the class can be added to the image if done_item_box: # resetting the variable number of filled files end_file = 0 # the width of the area required for the class object width = (j + count_of_x_boxes_item) * config.step['width'] # the height of the area required for the class object height = (i + count_of_y_boxes_item) * config.step['height'] # offset from the edge random_indent = random.randrange( 5, 15, 5) if ((item[index].current_size[1] == 100) or (item[index].current_size[1] == 120) ) else random.randrange(20, 40, 5) width_min, height_min, width_max, height_max = None, None, None, None # checking that the class object should be shifted to the upper-left corner relative to the start # cell if item[index].current_position == 1: width_min = width - count_of_x_boxes_item * config.step[ 'width'] + random_indent height_min = height - count_of_y_boxes_item * config.step[ 'height'] + random_indent width_max = width - count_of_x_boxes_item * config.step[ 'width'] + item[index].current_size[ 1] + random_indent height_max = height - count_of_y_boxes_item * config.step[ 'height'] + item[index].current_size[ 0] + random_indent item[index].current_position += 1 # checking that the class object should be shifted to the upper-right corner relative to the start # cell elif item[index].current_position == 2: width_min = width - item[index].current_size[ 1] - random_indent height_min = height - count_of_y_boxes_item * config.step[ 'height'] + random_indent width_max = width - random_indent height_max = height - count_of_y_boxes_item * config.step[ 'height'] + item[index].current_size[ 0] + random_indent item[index].current_position += 1 # checking that the class object should be shifted to the lower-left corner relative to the start # cell elif item[index].current_position == 3: width_min = width - count_of_x_boxes_item * config.step[ 'width'] + random_indent height_min = height - item[index].current_size[ 0] - random_indent width_max = width - count_of_x_boxes_item * config.step[ 'width'] + item[index].current_size[ 1] + random_indent height_max = height - random_indent item[index].current_position += 1 # checking that the class object should be shifted to the lower-right corner relative to the start # cell elif item[index].current_position == 4: width_min = width - item[index].current_size[ 1] - random_indent height_min = height - item[index].current_size[ 0] - random_indent width_max = width - random_indent height_max = height - random_indent item[index].current_position += 1 # checking that the class object should be shifted to the center part relative to the start cell elif item[index].current_position == 5: width_min = width - count_of_x_boxes_item * config.step['width'] / 2 - \ item[index].current_size[1] / 2 height_min = height - count_of_y_boxes_item * config.step['height'] / 2 - \ item[index].current_size[0] / 2 width_max = width - count_of_x_boxes_item * config.step['width'] / 2 + \ item[index].current_size[1] / 2 height_max = height - count_of_y_boxes_item * config.step['height'] / 2 + \ item[index].current_size[0] / 2 item[index].current_position = 1 progress_count += 1 bar.update(progress_count) # filter for resizing an image resample = Image.ANTIALIAS button.load() # splitting an image into strips (R, G, B, A) bands = button.split() # getting new object sizes dim = (int(width_max - width_min), int(height_max - height_min)) # converting each color-light strip of the image to a new size bands = [b.resize(dim, resample) for b in bands] # combining all the stripes into a single image button = Image.merge('RGBA', bands) # inserting the resulting class object into the image img.paste(button, (int(width_min), int(height_min)), button) # adding a class name to xml object_.append(SubElement(root, "object_")) name.append(SubElement(object_[number], "name")) name[number].text = str(item[index].name) bndbox.append(SubElement(object_[number], "bndbox")) xmin_.append(SubElement(bndbox[number], "xmin")) xmin_[number].text = str(int(width_min)) ymin_.append(SubElement(bndbox[number], "ymin")) ymin_[number].text = str(int(height_min)) xmax_.append(SubElement(bndbox[number], "xmax")) xmax_[number].text = str(int(width_max)) ymax_.append(SubElement(bndbox[number], "ymax")) ymax_[number].text = str(int(height_max)) # increasing the number of added objects for the current class list_count_of_each_class[index] += 1 # increasing the auxiliary variable by 1 number += 1 # getting a new object height height_current = (item[index].current_size[1] + item[index].step_change_size) * \ item[index].current_size[0] / item[index].current_size[1] # getting a new object width item[index].current_size[1] = ( item[index].current_size[1] + item[index].step_change_size if item[index].current_size[1] + item[index].step_change_size <= item[index].size_finish[1] else item[index].size_start[1]) # checking that the current width is equal to the initial width if item[index].current_size[1] == item[index].size_start[ 1]: height_current = item[index].size_start[0] # getting the height item[index].current_size[ 0] = height_current * coeff_orientation break # checking that the class object did not fit into the file if not done_item_box: end_file += 1 # checking that the number of filled files == the number of classes if end_file == number_classes: flag_end = False # moving the class number variable to the next class index += 1 # saving the image img.save( os.path.join(config.work_dir, config.dir_output_data, config.dir_result_img, f'{str(_count_)}.jpg')) # path to the xml file name_ = os.path.join(config.work_dir, config.dir_output_data, config.dir_result_xml, f'{str(_count_)}.xml') # calling the function that saves the xml file save_xml(name_, root) # increasing the file number by 1 _count_ += 1 bar.finish()
logfile.write(str(getattr(args, arg))) logfile.write("\n") cmdfile = open("3_cmds", 'w') bcin = open(bcfile, 'r') bc = [] for bcinline in bcin: bc.append(bcinline) bclength = len(bc) print "Number of entries: ", bclength print "\nWorking..." bar = progressbar.ProgressBar() for i in bar(range(bclength)): bcline = bc[i] bccols = bcline.split() sample = bccols[1] output = wd + "/" + sample if seqprep_output_in_output: seqprep_output = output + "/" + seqprep_output_orig else: seqprep_output = seqprep_output_orig so_s = seqprep_output + "/" + sample mo_s = metadir + "/" + sample alllist = [] if not os.path.isfile(so_s + ".all.SP.fq"):
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout): # Currently only 'marathon' instances are supported for wait_for_deployment because they # are the only thing that are worth waiting on. service_configs = PaastaServiceConfig(service=service, soa_dir=soa_dir, load_deployments=False) total_instances = 0 clusters_data = [] api_endpoints = load_system_paasta_config().get_api_endpoints() for cluster in service_configs.clusters: if cluster not in api_endpoints: paasta_print( PaastaColors.red( 'Cluster %s is NOT in paasta-api endpoints config.' % cluster, )) raise NoSuchCluster instances_queue = Queue() for instance_config in service_configs.instance_configs( cluster=cluster, instance_type='marathon'): if instance_config.get_deploy_group() == deploy_group: instances_queue.put(instance_config) total_instances += 1 if not instances_queue.empty(): clusters_data.append( ClusterData( cluster=cluster, service=service, git_sha=git_sha, instances_queue=instances_queue, )) if not clusters_data: _log( service=service, component='deploy', line= ("Couldn't find any marathon instances for service {} in deploy group {}. Exiting." .format(service, deploy_group)), level='event', ) return paasta_print("Waiting for deployment of {} for '{}' to complete...".format( git_sha, deploy_group)) deadline = time.time() + timeout green_light = Event() green_light.set() with progressbar.ProgressBar(maxval=total_instances) as bar: while time.time() < deadline: _query_clusters(clusters_data, green_light) if not green_light.is_set(): raise KeyboardInterrupt bar.update(total_instances - sum((c.instances_queue.qsize() for c in clusters_data))) if all((cluster.instances_queue.empty() for cluster in clusters_data)): sys.stdout.flush() return 0 else: time.sleep(min(60, timeout)) sys.stdout.flush() _log( service=service, component='deploy', line=compose_timeout_message(clusters_data, timeout, deploy_group, service, git_sha), level='event', ) raise TimeoutError
import pickle import math import progressbar loc = input('Enter the path to the file you want to prune: ') data = None try: data = open(loc, 'rb') except Exception as e: raise Exception("Could not read file") menance = pickle.load(data) done = 0 length = len(menance.moves) pbartmp = progressbar.ProgressBar(maxval=1).default_widgets() pbar = progressbar.ProgressBar(widgets=pbartmp[:], maxval=length) del pbartmp for key in menance.moves: done += 1 state = menance.moves[key].state if len(state) > 0: unique = list(set(state)) unique_count = [state.count(num) for num in unique] mmin = min(unique_count) new_count = [math.ceil(count / mmin) for count in unique_count] tmp = [] for i in range(len(new_count)): orig = unique[i] for j in range(new_count[i]): tmp.append(orig)
def _consume_multi_display_multi_pbar(progress_queue, num_workers, num_simulations, max_width, fd): # In order to display multiple progress bars, we need to manipulate the # terminal/console to move up lines. Colorama is used to wrap stderr such # that ANSI escape sequences are mapped to equivalent win32 API calls. fd = colorama.AnsiToWin32(fd).stream def ansi_up(n): return b'\x1b[{}A'.decode('latin1').format(n) ansi_bold = b'\x1b[1m'.decode('latin1') ansi_norm = b'\x1b[0m'.decode('latin1') overall_pbar = _get_overall_pbar(num_simulations, max_width, fd) try: worker_progress = OrderedDict() completed = set() while len(completed) < num_simulations: sim_index, now, t_stop, timescale = progress_queue.get() if now == t_stop: completed.add(sim_index) if worker_progress: print(ansi_up(len(worker_progress)), end='', file=fd) if sim_index in worker_progress: for pindex, pbar in worker_progress.items(): if sim_index == pindex and pbar: if now == t_stop: pbar.finish() worker_progress[sim_index] = None else: if t_stop and pbar.max_value != t_stop: pbar.max_value = t_stop pbar.widgets = _get_progressbar_widgets( sim_index, timescale, know_stop_time=True) pbar.update(now) print(file=fd) else: print(file=fd) else: for pindex, pbar in worker_progress.items(): if pbar is None: worker_progress.pop(pindex) break print('\n' * len(worker_progress), file=fd) pbar = progressbar.ProgressBar( fd=fd, term_width=overall_pbar.term_width, min_value=0, max_value=(progressbar.UnknownLength if t_stop is None else t_stop), widgets=_get_progressbar_widgets(sim_index, timescale, know_stop_time=t_stop is not None)) worker_progress[sim_index] = pbar print(ansi_bold, end='', file=fd) overall_pbar.update(len(completed)) print(ansi_norm, end='', file=fd) finally: print(ansi_bold, end='', file=fd) overall_pbar.finish() print(ansi_norm, end='', file=fd)
def compute(self, networkinfo, elon, elat, edep, vp=6.5, vs=3.5, nnst=6, procdelay=False, target=None, nmaps=500, resultsfn='p_wave_tt_6_stations.npz', datadir='./data', latencies=None, vp3d=False): # If given a target location (lon,lat) it will also compute # the lead time from any potential epicenter to the target site if target is not None: tln, tlt = target # get the station locations networks = networkinfo.get_networks() # get the delay measurement distributions: # pkdel = pick delays # envdel = envelope delays # ascdel = associator/locator delays # magdel = magnitude delays # pkdefault = pick default delay # envdefault = envelope default delay # Note pkdel and envdel have to be Python dictionaries that have # a Numpy array of delay measurements for every station name key # ascdel and magdel are simply Numpy arrays containing delay # observations # pkdefault and envdefault are Python dictionaries that contain a # default delay value for every network name key. These default values # are used if a station is in the list of stations returned by # networkinfo but does not have delay observations in pkdel and envdel if procdelay and latencies is not None: pkdel = latencies.get_pick_delays() envdel = latencies.get_envelope_delays() ascdel = latencies.get_associator_delays() magdel = latencies.get_magnitude_delays() pkdefault = latencies.get_pick_default() envdefault = latencies.get_envelope_default() # do we have a grid or a set of points? if elon.shape != elat.shape or elon.shape != edep.shape: raise DelayException( "elon, elat, and edep must have the same shape!") outshape = [nmaps] for _s in elon.shape: outshape.append(_s) lat = elat.ravel() lon = elon.ravel() dep = edep.ravel() ngp = lon.size ttP = np.zeros(outshape) ttPtmp = np.zeros(ngp) tstarget = ttPtmp.copy() g = pyproj.Geod(ellps='sphere') # Setup progressbar widgets = ['tt: ', pg.Percentage(), ' ', pg.Bar('#'), ' ', pg.ETA()] pbar = pg.ProgressBar(widgets=widgets, maxval=nmaps * ngp).start() no_env_dl = [] no_pk_dl = [] # loop over samples (Monte-Carlo simulation) for nm in range(nmaps): idx = 0 # loop over earthquake locations for _lat, _lon, _dep in zip(lat, lon, dep): pbar.update(nm * ngp + idx) min_dt = 1.e38 # loop over networks; all stations within a network are # considered for delay computations but only the fastest alert # is stored for net in networks.keys(): if len(networks[net]['lat']) < 1: continue stlat = networks[net]['lat'] stlon = networks[net]['lon'] nwcode = np.array(networks[net]['nw']) names = np.array(networks[net]['nm']) maxnstat = min(10, len(stlat)) nnst = min(nnst, len(stlat)) # An option to used traveltimes from a 3D model instead # of a homogeneous halfspace; only works on my computer # at the moment if vp3d: datadir = './data/ttime_ch/' fin = os.path.join(datadir, "%.4f_%.4f_%.1f.npy" % \ (_lat, _lon, _dep)) if not os.path.isfile(fin): raise DelayException('File %s does not exist!' % fin) a = np.load(fin) pt = a['ttime'][:, 0] dt = max(pt[0:nnst]) stat_names = a['name'] else: # Find the <nnst> nearest stations lats = np.ones((len(stlat), )) * _lat lons = np.ones((len(stlon), )) * _lon az, baz, dist = g.inv(lons, lats, stlon, stlat) dist_sorted = np.sort(dist) stat_names = names[np.argsort(dist)] dz = np.ones((maxnstat, )) * _dep pt = np.sqrt(dz * dz + dist_sorted[0:maxnstat] / 1000. * dist_sorted[0:maxnstat] / 1000.) / vp dt = max(pt[0:nnst]) if target is not None: azt, bazt, distt = g.inv(_lon, _lat, tln, tlt) distt /= 1000. tstarget[idx] = np.sqrt(distt * distt + _dep * _dep) / vs # Add delays # Note: if a close by station has very long delays, the # algorithm will pick one of the next further stations # if they can send data before the closer station if procdelay: pk_delays = [] env_delays = [] for stat in stat_names[0:maxnstat]: if stat in pkdel.keys(): pk_delays.append(pkdel[stat][np.random.randint( 0, len(pkdel[stat]))]) else: if stat not in no_pk_dl: no_pk_dl.append(stat) pk_delays.append(pkdefault[net]) if stat in envdel.keys(): env_delays.append( envdel[stat][np.random.randint( 0, len(envdel[stat]))]) else: if stat not in no_env_dl: no_env_dl.append(stat) env_delays.append(envdefault[net]) if len(pk_delays) != maxnstat or len( env_delays) != maxnstat: raise DelayException( 'Number of delays is not equal to %d' % nnst) temp = np.sort(pt + np.array(pk_delays)) + ascdel[ np.random.randint(0, len(ascdel))] origin_delay = max(temp[0:nnst]) waveform_delay = min(pt + np.array(env_delays)) dt = max(origin_delay, waveform_delay) + magdel[np.random.randint( 0, len(magdel))] if dt < min_dt: min_dt = dt ttPtmp[idx] = min_dt idx += 1 ttP[nm] = ttPtmp.reshape(elon.shape) tstarget = tstarget.reshape(elon.shape) # store the results for plotting if resultsfn is not None: np.savez(resultsfn, ttP=ttP, tstarget=tstarget, lat=elat, lon=elon, dep=edep) pbar.finish() print "No envelope delay data available for the following stations:" print ' '.join(no_env_dl) print "No pick delay info available for the following stations:" print ' '.join(no_pk_dl) return elat, elon, edep, ttP, tstarget
def generate(self): """ Gradient descent based PCF matching for point synthesis """ const = self.edge_correction() sigma = self.choose_sigma() g0, g_init, D, D_old, weight, G_init_local, X_init = self.initial_calculation( ) max_r = 2 * self.r_1 r = np.linspace(0.0001, max_r, self.len_r) g = g_init """ Initialize parameters """ j = 1 X = dc(X_init) G_local_old = (G_init_local) error_func = [] error = np.square(np.subtract(g, g0)).mean() error_func.append(error) losses = [] alpha = self.lr beta1 = 0.9 beta2 = 0.999 eps = 1 / np.power(10, 8) m_t = np.zeros((self.N, self.d)) v_t = np.zeros((self.N, self.d)) m_hat = np.zeros((self.N, self.d)) v_hat = np.zeros((self.N, self.d)) """ Gradient Descent Loop """ with progressbar.ProgressBar(max_value=self.maxGD) as bar: while (error > self.MSE and j < self.maxGD): j = j + 1 for i in range(1, self.N + 1): G = np.divide(np.multiply(g - g0, const), weight) U = X[i - 1, :] - X N1 = np.linalg.norm(U, axis=1) + 0.0001 D_old[i - 1] = dc(N1) D_old[i - 1][D_old[i - 1] == 0] = float('inf') Y_old = (np.mat(D_old[i - 1])) - (np.transpose( np.matrix(r))) Lo_old = self.G_kern(Y_old, sigma) N1[i - 1] = 1 N2 = np.tile(np.transpose(np.matrix(N1)), (1, self.d)) U = np.divide(U, N2) Y_old[np.isinf(Y_old)] = 0 Z = np.multiply( np.multiply(np.transpose(np.tile(G, (self.N, 1))), Y_old), Lo_old) W = Z.sum(axis=0) M = np.multiply(U, np.transpose(np.tile(W, (self.d, 1)))) grad = -(M.sum(axis=0) / np.linalg.norm(M.sum(axis=0) + 0.0001)) """ ADAM gradient update """ m_t[i - 1, :] = beta1 * m_t[i - 1, :] + (1 - beta1) * grad v_t[i - 1, :] = beta2 * v_t[i - 1, :] + ( 1 - beta2) * np.square(grad) m_hat[i - 1, :] = m_t[i - 1, :] / (1 - beta1) v_hat[i - 1, :] = v_t[i - 1, :] / (1 - beta2) adam_grad = alpha * m_hat[i - 1, :] / ( np.sqrt(v_hat[i - 1, :]) + eps) X[i - 1, :] = np.subtract(X[i - 1, :], adam_grad) X[np.isnan(X)] = 0 """ Points should be in the bounded region """ for jj in range(self.d): if np.mod(i, 3) == 0: if X[i - 1, jj] > 1: X[i - 1, jj] = np.random.rand(1) if X[i - 1, jj] < 0: X[i - 1, jj] = np.random.rand(1) else: if X[i - 1, jj] > 1: X[i - 1, jj] = .999 if X[i - 1, jj] < 0: X[i - 1, jj] = 0.001 U = X[i - 1, :] - X D[i - 1] = np.linalg.norm(U, axis=1) D[i - 1][D[i - 1] == 0] = float('inf') Y = (np.mat(D[i - 1])) - (np.transpose(np.matrix(r))) Lo = self.G_kern(Y, sigma) Ks = Lo.sum(axis=1) G_local_new = np.multiply( np.array(const).flatten(), np.array(Ks).flatten()) G_local_old[i - 1, :] = G_local_new """ Update local PCF of all the point """ g = G_local_old.sum(axis=0) g[g < 0] = 0 Loss = np.mean(np.divide(np.multiply(g - g0, const), weight)) losses.append(Loss) error = np.square(np.subtract(g, g0)).mean() error_func.append(error) # print ( 'epoch :'+str(j)+' MSE error:'+str(error) +' Loss:'+str(Loss)) bar.update(j) return X, g, g0, error_func, losses
def main(): # 默认路径 # CONFIG_FILE = 'configs/mask_rcnn/mask_rcnn_r50_fpn_1x_coco.py' # 模型的配置文件 # CONFIG_FILE = 'configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco.py' # 模型的配置文件 # CONFIG_FILE = 'configs/yolo/yolov3_d53_mstrain-608_273e_coco.py' # 模型的配置文件 CONFIG_FILE = 'configs/ssd/ssd300_coco.py' # 模型的配置文件 # CONFIG_FILE = 'configs/retinanet/retinanet_r50_fpn_1x_coco.py' # 模型的配置文件 CHECKPOINT_FILE = 'work_dirs/ssd300_coco/epoch_12.pth' # 训练好的模型权重 # IMG_PATH = '/home/watson/Documents/mask_THzDatasets/val' IMG_PATH = '/home/watson/Documents/mask_THzDatasets/test2/imgs' RESULT_ROOT = "/home/watson/Documents/mmdet_result/{}".format( os.path.splitext(os.path.basename(CONFIG_FILE))[0]) # 解析参数 args = parse_args() if args.config is not None: CONFIG_FILE = args.config if args.checkpoint is not None: CHECKPOINT_FILE = args.checkpoint if args.img_input is not None: IMG_PATH = args.img_input if args.img_output is not None: RESULT_ROOT = args.img_output RESULT_IMG_PATH = os.path.join(RESULT_ROOT, "imgs") RESULT_XML_PATH = os.path.join(RESULT_ROOT, "xmls") if not os.path.exists(RESULT_IMG_PATH): os.makedirs(RESULT_IMG_PATH) if not os.path.exists(RESULT_XML_PATH): os.makedirs(RESULT_XML_PATH) # 初始化模型 print('=== initialing detector ===') model = init_detector(CONFIG_FILE, CHECKPOINT_FILE) # 推断图片 print('inference start!') img_list = os.listdir(IMG_PATH) img_cnt = 0 img_totalNum = len(img_list) bar = progressbar.ProgressBar(max_value=img_totalNum).start() for img_fileName in img_list: img_cnt += 1 bar.update(img_cnt) img_fullFileName = os.path.join(IMG_PATH, img_fileName) result_img_fullName = os.path.join(RESULT_IMG_PATH, img_fileName) result_xml_fullName = os.path.join( RESULT_XML_PATH, img_fileName.split('.')[0] + "_result.xml") result = inference_detector(model, img_fullFileName) img = model.show_result(img_fullFileName, result, score_thr=0.3, show=False, out_file=result_img_fullName) # show_result_pyplot(model, img_fullFileName, result, score_thr=0.3) detections = resolve_result(result, class_names=model.CLASSES, score_thr=0.3) store_voc_result(img_fullFileName, detections, result_xml_fullName) bar.finish() print('inference finished!')
return finale # create custEmails dataframe with regular customers' ids and emails (below) custEmails = pd.DataFrame(orig_data[orig_data['Customer'].str.startswith('CC')] ['Customer'].drop_duplicates()) #save count to var howmany (only regular customers) howmany = len(custEmails) counter = 0 #counter for While loop emailarray = [] #empty array for loop print "Creating email addresses..." prebar = progressbar.ProgressBar(maxval=int(howmany)) for i in prebar(range(howmany)): while counter < howmany: emailarray.append(str(makeEmail())) counter = counter + 1 prebar.update(i) print "Email creation completed." bar = progressbar.ProgressBar(maxval=int(howmany)) custEmails['Email'] = emailarray # save regular customer Ids and Emails in custEmails.csv custEmails.to_csv('../custEmails.csv', index=None, header=False)
""" Command line options """ parser.add_argument( "--download_folder", type=str, default=download_folder, help="Folder to download weights to. Default is " + download_folder, ) FLAGS = parser.parse_args() url = "https://pjreddie.com/media/files/yolov3.weights" r = requests.get(url, stream=True) f = open(os.path.join(download_folder, "yolov3.weights"), "wb") file_size = int(r.headers.get("content-length")) chunk = 100 num_bars = file_size // chunk bar = progressbar.ProgressBar(maxval=num_bars).start() i = 0 for chunk in r.iter_content(chunk): f.write(chunk) bar.update(i) i += 1 f.close() call_string = "python convert.py yolov3.cfg yolov3.weights yolo.h5" subprocess.call(call_string, shell=True, cwd=download_folder)
def main(): workspace_path = os.environ.get('AE_WORKSPACE_PATH') if workspace_path is None: print('Please define a workspace path:\n') print('export AE_WORKSPACE_PATH=/path/to/workspace\n') exit(-1) gentle_stop = np.array((1, ), dtype=np.bool) gentle_stop[0] = False def on_ctrl_c(signal, frame): gentle_stop[0] = True signal.signal(signal.SIGINT, on_ctrl_c) parser = argparse.ArgumentParser() parser.add_argument("experiment_name") parser.add_argument("-d", action='store_true', default=False) parser.add_argument("-gen", action='store_true', default=False) arguments = parser.parse_args() full_name = arguments.experiment_name.split('/') experiment_name = full_name.pop() experiment_group = full_name.pop() if len(full_name) > 0 else '' debug_mode = arguments.d generate_data = arguments.gen cfg_file_path = u.get_config_file_path(workspace_path, experiment_name, experiment_group) log_dir = u.get_log_dir(workspace_path, experiment_name, experiment_group) checkpoint_file = u.get_checkpoint_basefilename(log_dir) ckpt_dir = u.get_checkpoint_dir(log_dir) train_fig_dir = u.get_train_fig_dir(log_dir) dataset_path = u.get_dataset_path(workspace_path) if not os.path.exists(cfg_file_path): print('Could not find config file:\n') print('{}\n'.format(cfg_file_path)) exit(-1) if not os.path.exists(ckpt_dir): os.makedirs(ckpt_dir) if not os.path.exists(train_fig_dir): os.makedirs(train_fig_dir) if not os.path.exists(dataset_path): os.makedirs(dataset_path) args = configparser.ConfigParser() args.read(cfg_file_path) shutil.copy2(cfg_file_path, log_dir) with tf.variable_scope(experiment_name): dataset = factory.build_dataset(dataset_path, args) queue = factory.build_queue(dataset, args) encoder = factory.build_encoder(queue.x, args, is_training=True) decoder = factory.build_decoder(queue.y, encoder, args, is_training=True) ae = factory.build_ae(encoder, decoder, args) codebook = factory.build_codebook(encoder, dataset, args) train_op = factory.build_train_op(ae, args) saver = tf.train.Saver(save_relative_paths=True) num_iter = args.getint('Training', 'NUM_ITER') if not debug_mode else 100000 save_interval = args.getint('Training', 'SAVE_INTERVAL') model_type = args.get('Dataset', 'MODEL') print("there will be %d iterations" % num_iter) if model_type == 'dsprites': dataset.get_sprite_training_images(args) else: dataset.get_training_images(dataset_path, args) dataset.load_bg_images(dataset_path) if generate_data: print('finished generating synthetic training data for ' + experiment_name) print('exiting...') exit() widgets = [ 'Training: ', progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.Counter(), ' / %s' % num_iter, ' ', progressbar.ETA(), ' ' ] bar = progressbar.ProgressBar(maxval=num_iter, widgets=widgets) gpu_options = tf.GPUOptions(allow_growth=True, per_process_gpu_memory_fraction=0.9) config = tf.ConfigProto(gpu_options=gpu_options) with tf.Session(config=config) as sess: chkpt = tf.train.get_checkpoint_state(ckpt_dir) if chkpt and chkpt.model_checkpoint_path: saver.restore(sess, chkpt.model_checkpoint_path) else: sess.run(tf.global_variables_initializer()) merged_loss_summary = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(ckpt_dir, sess.graph) if not debug_mode: print('Training with %s model' % args.get('Dataset', 'MODEL'), os.path.basename(args.get('Paths', 'MODEL_PATH'))) bar.start() queue.start(sess) for i in range(ae.global_step.eval(), num_iter): print("step %d" % i) if not debug_mode: sess.run(train_op) if i % 10 == 0: loss = sess.run(merged_loss_summary) summary_writer.add_summary(loss, i) bar.update(i) if (i + 1) % save_interval == 0: saver.save(sess, checkpoint_file, global_step=ae.global_step) this_x, this_y = sess.run([queue.x, queue.y]) reconstr_train = sess.run(decoder.x, feed_dict={queue.x: this_x}) train_imgs = np.hstack( (u.tiles(this_x, 4, 4), u.tiles(reconstr_train, 4, 4), u.tiles(this_y, 4, 4))) cv2.imwrite( os.path.join(train_fig_dir, 'training_images_%s.png' % i), train_imgs * 255) else: this_x, this_y = sess.run([queue.x, queue.y]) reconstr_train = sess.run(decoder.x, feed_dict={queue.x: this_x}) cv2.imshow( 'sample batch', np.hstack((u.tiles(this_x, 3, 3), u.tiles(reconstr_train, 3, 3), u.tiles(this_y, 3, 3)))) k = cv2.waitKey(0) if k == 27: break if gentle_stop[0]: break queue.stop(sess) if not debug_mode: bar.finish() if not gentle_stop[0] and not debug_mode: print('To create the embedding run:\n') print('ae_embed {}\n'.format(full_name))