def eval(self, query_numbers_, queries_, passage_): ''' Function to run the model and store the results in file_path Args: query_numbers_: IDs for corresponding queries, needed to store the results queries_: numpy array for queries passage_: numpy array for passages ''' if self.is_training: raise SystemExit("config setup for training:", self.is_training) dm = DatasetManager(query_numbers_, queries_, passage_) # load the model self.sess = tf.Session() self.saver = tf.train.Saver() self.saver.restore(self.sess, self.save_folder) # results list qn_list = [] results = [] # iterate and get results num_batches = dm.get_num_batches(self.batch_size) print('[!] Number of pairs processed:', num_batches * self.batch_size) for i in tqdm(range(num_batches)): b_qn, b_query, b_passage = dm.get_batch(query_numbers_, queries_, passage_, self.batch_size) # perform padding try: b_query = add_padding(b_query, self.pad_id, self.seqlen) b_passage = add_padding(b_passage, self.pad_id, self.seqlen) except Exception as e: print(b_query) print(b_passage) print(b_query.shape) print(b_passage.shape) # get predictions preds = self.sess.run(self.net_op, { self.query_input: b_query, self.passage_input: b_passage }) results.append(preds) qn_list.append(b_qn) # return the results return qn_list, results
def __encrypt_block(self, block: str) -> str: """Encrypt the text block.""" block_ints = chars_to_ints(block) self._validate_input(block_ints) block_ints = add_padding(block_ints, self.block_bytes) cipher_ints = self._encrypt_block_ints(block_ints) cipher = ''.join(ints_to_hexes(cipher_ints)) return cipher
def write_note(note, next_note_start_tick): # TODO FIX ERROR WHEN WRITING NOTE WHOSE HEX NEXT NOTE VALUE IS 3 DIGIT, NEEDS POST NOTE PADDING velocity = add_padding(str(hex(note.velocity))[2:], 2) next_note = next_note_start_tick - note.start_tick if len(str(hex(next_note))[2:]) == 1: next_note = '0' + str(next_note) else: next_note = str(hex(next_note))[2:] pad_code, bank_switch = gen_pad_code_bank_switch(note.bank, note.pad) encoding = f'{next_note}{pad_code}0{bank_switch}00{velocity}40{get_hex_length(note.length)}\n' return encoding
def gen_pad_bank(pad_code, bank_switch): real_code = int(pad_code, 16) - constants.pad_offset_magic_number bank = int(real_code / 12) pad = real_code % 12 if pad == 0: pad = 12 bank -= 1 if bank_switch == 1: bank += 5 ascii_bank = chr(bank + constants.ascii_character_offset) pd = add_padding(str(pad), 2) return pad, ascii_bank.upper()
def preview_pattern(pattern, bpm): bpm = int(bpm) MpT = milliseconds_per_tick(bpm) pattern_audio = AudioSegment.silent(duration=constants.ticks_per_bar * pattern.length * MpT) for track in pattern.tracks: for note in track.notes: pd = add_padding(str(note.pad), 2) path = f"{constants.path_to_samples}{note.bank.upper()}00000{pd}.WAV" sample_audio = AudioSegment.from_file(path) pattern_audio = pattern_audio.overlay(sample_audio[:note.length * MpT], position=note.start_tick) play(pattern_audio)
def test(): model = keras.models.load_model("tf/model.h5") word_index = modify_word_index() with open('tf/test_review.txt', encoding='utf-8 ') as file: review = filter_text(file.read()) print(review) encode = add_padding([encode_review(review, word_index)], value=word_index['<PAD>'], maxlen=250) print(decode_review(encode_review(review, word_index), word_index)) predict = model.predict(encode) print("\n\n") print("Original Review link: " "https://timesofindia.indiatimes.com/" "entertainment/marathi/movie-reviews/" "basta/movie-review/80670252.cms") print("Rating on Site: 2.5/5") print("Prediction: ", predict[0])
def load_synthetic_data_set(name, target_model): """ Method that loads the local data set stored in the directory called 'name' :param name: name of the directory where data set is stored :param target_model: decides the format in which to return the data set :return: if for patchy_san return all graphs and labels in the data set else return all attributes and labels in the data set """ all_graphs = list() all_labels = list() dataset_directory = get_directory() + '/Data_Sets/Provenance_Graphs/' + name number_of_classes, graphs_per_class = DataLoader.__load_data_property_file(dataset_directory + '/property_file') for index in range(1, number_of_classes + 1): class_directory = dataset_directory + '/Class_' + str(index) class_graphs = DataLoader.__load_graphs(class_directory, graphs_per_class[index - 1]) for graph in class_graphs: all_graphs.append(graph) all_labels.append(index) all_labels = np.array(all_labels) all_graphs = np.array(all_graphs) if target_model == 'patchy_san': return all_graphs, all_labels, number_of_classes elif target_model == 'baselines': all_values = list() for graph in all_graphs: all_values.append(merge_splits(graph.values())) all_values = add_padding(all_values, 0) all_values = np.array(all_values) return all_values, all_labels, number_of_classes
time_delta = ds.getTimeDelta(delta=True, format='days') ds.setDotyFlag(True) dotys, dotys_sin_cos = ds.getDayOfTheYear() ds.dotyReplicateSamples(sample_n=1) # Reconstruct the image print("Reconstructing the labes and predictions...") patch_size = 32 add_padding_flag = False if add_padding_flag == True: full_ims_test, stride, step_row, step_col, overlap = seq_add_padding( full_ims_test, patch_size, 0) #full_label_test, _, _, _, _ = seq_add_padding(full_label_test,32,0) mask_pad, _, _, _, _ = add_padding(mask, patch_size, 0) else: mask_pad = mask.copy() stride = patch_size overlap = 0 print(full_ims_test.shape) print(full_label_test.shape) print(np.unique(full_label_test, return_counts=True)) sequence_len, row, col, bands = full_ims_test.shape #pdb.set_trace() prediction_rebuilt = np.ones((sequence_len, row, col)).astype(np.uint8) * 255 print("stride", stride) print(len(range(patch_size // 2, row - patch_size // 2, stride))) print(len(range(patch_size // 2, col - patch_size // 2, stride)))
def train(self, queries_, passage_, label_, num_epochs=50, val_split=0.1, display_results_after=1000, jitter=0.05, smooth_thresh_upper=0.8, smooth_thresh_lower=0.15): ''' This is the function used to train the model. Args: queries_: numpy array for queries passage_: numpy array for passages label_: numpy array for labels num_epochs: number of epochs for training ''' if not self.is_training: raise Exception("Config not setup for training,", self.is_training) # make the dataset manager to handle the datasets dm = DatasetManager(queries_, passage_, label_) # establish the saver self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) self.saver = tf.train.Saver() # train logs train_loss = [] train_acc = [] # value thresh self.smooth_thresh_lower = smooth_thresh_lower self.smooth_thresh_upper = smooth_thresh_upper self.s_ll = smooth_thresh_lower - jitter # smooth lower - lower self.s_lu = smooth_thresh_upper + jitter # smooth lower - upper self.s_ul = smooth_thresh_upper - jitter # smooth upper - lower self.s_uu = smooth_thresh_upper + jitter # smooth upper - upper # counter global_step_prev = 0 for ep in range(num_epochs): batch_loss = [] batch_accuracy = [] # for display counters display_loss = [] display_accuracy = [] d_pos = [] d_neg = [] # iterate over all the batches num_batches = dm.get_num_batches(self.batch_size) time_start = time() for batch_num in tqdm(range(num_batches)): # for each epoch, go over the entire dataset once b_query, b_passage, b_label = dm.get_batch( queries_, passage_, label_, self.batch_size) # pad the sequences try: b_query = add_padding(b_query, self.pad_id, self.seqlen) b_passage = add_padding(b_passage, self.pad_id, self.seqlen) except Exception as e: print(b_query) print(b_passage) print(b_query.shape) print(b_passage.shape) # reshape b_label = np.reshape(b_label, [-1, 1]) # print stats if if ep == 0 and batch_num == 0: print('[*] Batch Shapes:') print('b_query:', b_query.shape) print('b_passage:', b_passage.shape) print('b_label:', b_label.shape) # operate b_ops = [self._loss, self._train_step, self.pred] feed_dict = { self.query_input: b_query, self.passage_input: b_passage, self.target_input: b_label } b_loss, _, b_pred = self.sess.run(b_ops, feed_dict) batch_loss.append(b_loss) b_acc, b_pos, b_neg = self.calculate_accuracy(target=b_label, pred=b_pred) batch_accuracy.append((b_acc, b_pos, b_neg)) display_loss.append(b_loss) display_accuracy.append(b_acc) d_pos.append(b_pos) d_neg.append(b_neg) if self.global_step != 0 and self.global_step % self.save_freq == 0: self.save_model() if self.global_step != 0 and self.global_step % display_results_after == 0 or batch_num == num_batches - 1: d_mean_loss = np.mean(display_loss) d_mean_acc = np.mean(display_accuracy) d_mean_pos = np.mean(d_pos) d_mean_neg = np.mean(d_neg) time_end = time() print('[#] Global Step: {0}, Epoch: {1}'.format( self.global_step, ep)) print( 'mean loss: {0}, mean accuracy: {1} [true positive:{2}, true negative: {3})]' .format(d_mean_loss, d_mean_acc * 100, d_mean_pos * 100, d_mean_neg * 100)) print('Time taken for {0} examples: {1} seconds\n'.format( self.global_step - global_step_prev, time_end - time_start)) time_start = time() global_step_prev = self.global_step # reset display_accuracy = [] display_loss = [] # update the global step self.global_step += 1 # once all the batches are done train_loss.append(batch_loss) train_acc.append(batch_accuracy) ''' == Add to Tensorboard output == Add the stats to tensorboard output. Note that there alead is a function self.merge_all but I don't know how to use this. ''' ''' # add validation support # get validation values val_acc, val_loss = self.validation() ''' b_acc = np.sum(np.array(train_loss[-1]).T[0]) print('\n[!] Epoch: {0}, train_loss: {1}, accuracy: {2}\n'.format( ep, np.mean(train_loss[-1]), b_acc)) # save the model one last time self.save_model() self.sess.close() print('... Done! Training complete exiting the model') return train_loss, train_acc
def _set_key(self, key: str, is_encrypt: bool) -> None: """Validate and set the key.""" key_ints = self._preprocess_key(key) key_ints = add_padding(key_ints, self.key_bytes) self.__key_schedule = self.__key_expansion(key_ints)
def draw_line(line, outpath, spacing='normal', gtpath=None): global allmatch if spacing == 'normal': font_space = 0 elif spacing == "condensed": font_space = -5000 size = "40" w = 3000 h = 200 nof = 0 # spacing = 'condensed' # this doesn't seem to work anyway # for spacing in ['normal', 'condensed']: for fontname in fontnames: surf = cairo.ImageSurface(cairo.FORMAT_ARGB32, w, h) context = cairo.Context(surf) #draw a background rectangle: context.rectangle(0, 0, w, h) context.set_source_rgb(1, 1, 1) context.fill() # font_map = pangocairo.cairo_font_map_get_default() pangocairo_context = pangocairo.CairoContext(context) pangocairo_context.set_antialias(cairo.ANTIALIAS_SUBPIXEL) layout = pangocairo_context.create_layout() # font_params = [fontname,'normal', 'normal', spacing, size] font_params = [fontname, size] font = pango.FontDescription(" ".join(font_params)) attr = pango.AttrLetterSpacing(font_space, 0, -1) # attr = pango.AttrLetterSpacing(0, 0, -1) attrlist = pango.AttrList() attrlist.change(attr) # font.set_stretch(pango.STRETCH_CONDENSED) # font.set_stretch(pango.STRETCH_ULTRA_CONDENSED) # print font.get_stretch() layout.set_font_description(font) layout.set_attributes(attrlist) layout.set_text(line) context.set_source_rgb(0, 0, 0) pangocairo_context.update_layout(layout) pangocairo_context.show_layout(layout) # fname = "/tmp/%s%d.png" % (fontname, randint(0,20000000)) # fname = "/tmp/%s.png" % ('-'.join(font_params)) fname = outpath + fontname + '-' + spacing + '.png' # if os.path.exists(fname) and os.path.exists(outpath + fontname + '.gt.txt'): # allmatch += 1 # if os.path.exists(fname) and not os.path.exists(gtpath + fontname + '.gt.txt'): # print fname, gtpath + fontname + '.gt.txt' # codecs.open(gtpath + fontname + '.gt.txt', 'w', 'utf-8').write(line) # else: # nof += 1 # continue with open(fname, "wb") as image_file: surf.write_to_png(image_file) im = Image.open(fname) im = im.convert('L') a = np.asarray(im) a = trim(a) / 255 a = add_padding(a, padding=2) # os.remove(fname) # Image.fromarray(a*255).show() Image.fromarray(a * 255).save(fname) codecs.open(gtpath + fontname + '.gt.txt', 'w', 'utf-8').write(line) print allmatch
else: scaler_filename = os.path.join(args.model_dir, "scaler.save") scaler = joblib.load(scaler_filename) image = image.reshape(row * col, bands) image = scaler.transform(image) image = image.reshape(row, col, bands) if args.mode == "Test": params.ovrl_test = 0 params.ovrl = params.ovrl_test #if args.plot_histogram == True: # plot_histogram(image) image_tr, stride, step_row, step_col, overlap = add_padding( image, params.patch_size, params.ovrl) labels_tr, _, _, _, _ = add_padding(labels, params.patch_size, params.ovrl) ## labels_val, _, _, _, _ = add_padding(labels_dev, params.patch_size, params.ovrl) # get coords coords_tr = extract_patches_coord(labels_tr, params.patch_size, stride, train=True) ## coords_val = extract_patches_coord(labels_val, params.patch_size, stride, train = False) coords_tr = np.rollaxis(coords_tr, 1, 0) print(coords_tr.shape) ## pdb.set_trace()
def get_spectrum(self, ns=30, smin=3, smax=100, p_corr=0., nt=1): ''' Calculate the isotropic power spectral density (PDS) for an image of irregular shape. Args: ns (int): number of length-scales at which the PDS is calculated smin (int): minimum length-scale in pixels (>=2) smax (int): maximum length-scale in pixels p_corr: spectral slope if known beforehand (to correct normalization) nt: number of threads for multiprocessing Returns: kr: radial wavenumbers in px^(-1) pds: PDS measured at kr ''' print('calculate the PDS...') lx, ly = self.data.shape # subtract the mean component self.data[self.mask == 1.] -= self.data[self.mask == 1.].mean() # replace NaNs by zeros, add padding dp = add_padding(self.data, width=0.5) mp = add_padding(self.mask, width=0.5) # Fourier transform of the padded image and mask dpf = fftpack.fftn(dp) / np.sqrt(lx * ly) mpf = fftpack.fftn(mp) / np.sqrt(lx * ly) #pds = mp.Array('d', np.zeros(N)) pds = np.zeros(ns) # set the range of radial wave numbers lx, ly = dp.shape krmin = 1. / smax if smax <= 0.5 * max(lx, ly) else 2. / max( lx, ly) # mind zero padding krmax = 1. / smin # 1/2 is the Nyquist frequency fk = (krmax / krmin)**(1. / (ns - 1)) kr = krmin * np.power(fk, np.arange(ns)) eps = 1e-3 # Split calculation for different wave numbers between threads params = [{ 'i': i, 'kri': kr[i], 'datf': dpf, 'mskf': mpf, 'msk': mp, 'eps': eps, 'p_corr': p_corr } for i in range(ns)] if nt > 1: with multiprocessing.Pool(nt) as pool: pds = pool.map(calc_pds_single, params) pool.join() else: pds = map(calc_pds_single, params) if np.any(np.isnan(pds)): print('Error: NaN in the PDS') self.kr = kr self.pds = pds np.savetxt(self.output_paths['pds'], np.asarray([self.kr, self.pds])) print('PDS calculated and saved as ' + self.output_paths['pds']) return (self.kr, self.pds)
def get_pad_code(bank_letter, pad_number): bank = ord(bank_letter.lower()) - constants.ascii_character_offset code = str(hex((bank * constants.pads_per_bank) + int(pad_number)))[2:] return add_padding(code, 3)
def get_hex_length(length): length = str(hex(length))[2:] length = add_padding(length, 4) return length
# make list of results all_preds = [] num_batches = 0 proc_batches = 0 for _ in get_batch(idx_, n = 512): num_batches += 1 print('[!] number of batches:', num_batches) # iterate for x in get_batch(idx_, n = 512): proc_batches += 1 q = add_padding(queries_[x], model.pad_id, 80) p = add_padding(passages_[x], model.pad_id, 80) feed_dict = {model.query_input: q, model.passage_input: p} preds = model.sess.run(model.net_op, feed_dict = feed_dict) all_preds.append(preds) ''' Step 4: convert to proper list and write the answers.tsv file ''' all_pred_ = np.reshape(all_preds, [-1]) all_pred_ = all_pred_[:len(qn_)] all_pred_ = np.reshape(all_pred_, [-1, 10]) # converting to the required dump format res_path = '../final1_saves/res1/final_26.tsv'
def draw_fonts(args): label = args[0] char = args[1] output = [] for cycle in range(1): for fontname in fontnames: surf = cairo.ImageSurface(cairo.FORMAT_ARGB32, 600, 600) context = cairo.Context(surf) #draw a background rectangle: context.rectangle(0, 0, 600, 600) context.set_source_rgb(1, 1, 1) context.fill() #get font families: font_map = pangocairo.cairo_font_map_get_default() # context.translate(0,0) pangocairo_context = pangocairo.CairoContext(context) pangocairo_context.set_antialias(cairo.ANTIALIAS_SUBPIXEL) layout = pangocairo_context.create_layout() #fontname = sys.argv[1] if len(sys.argv) >= 2 else "Sans" # font = pango.FontDescription(fontname + " 200") if cycle == 0: font = pango.FontDescription(fontname + " 200") else: font = pango.FontDescription(fontname + " bold 200") layout.set_font_description(font) layout.set_text(char) context.set_source_rgb(0, 0, 0) pangocairo_context.update_layout(layout) pangocairo_context.show_layout(layout) fname = "/tmp/%s%d.png" % (fontname, randint(0, 20000000)) with open(fname, "wb") as image_file: surf.write_to_png(image_file) im = Image.open(fname) os.remove(fname) im = im.convert('L') a = np.asarray(im) a = trim(a) a = add_padding(a, padding=2) #####experimental normalization h, w = a.shape h = float(h) w = float(w) L = 32 sm = np.argmin([h, w]) bg = np.argmax([h, w]) R1 = [h, w][sm] / [h, w][bg] # R2 = np.sqrt(np.sin((np.pi/2.0)*R1)) # R2 = pow(R1, (1/3)) R2 = np.sqrt(R1) # R2 = R1 # if R1 < .5: # R2 = 1.5*R1 + .25 # else: # R2 = 1 if sm == 0: H2 = L * R2 W2 = L else: H2 = L W2 = L * R2 alpha = W2 / w beta = H2 / h a = resize(a, (0, 0), fy=beta, fx=alpha, interpolation=INTER_AREA) smn = a.shape[sm] offset = int(np.floor((L - smn) / 2.)) c = np.ones((L, L), dtype=np.uint8) * 255 # print a # print a.shape if (L - smn) % 2 == 1: start = offset + 1 end = offset else: start = end = offset if sm == 0: # print c[start:L-end, :].shape, a.shape c[start:L - end, :] = a else: # print c[:,start:L-end].shape, a.shape c[:, start:L - end] = a #########classic approach # im = Image.fromarray(a) # im.thumbnail((16,32), Image.ANTIALIAS) # im = np.asarray(im) # a = np.ones((32,16), dtype=np.uint8)*255 # a[0:im.shape[0],0:im.shape[1]] = im ########### #### USE FOLLOWING IF WANT TO SAVE NEW TRAINING DATA ################## a = c a[np.where(a < 120)] = 0 a[np.where(a >= 120)] = 1 # a = degrade(a) output.append( str(label) + ',' + ','.join(str(i) for i in a.flatten())) # # return output
def get_pad_code(bank_letter, pad_number): bank = ord(bank_letter.lower()) - 97 code = str(hex((bank * 12) + int(pad_number)))[2:] return add_padding(code, 3)