def generate_keys(self): p = maurer_generator(key_length) g = generator(p) x = random.randint(1, p - 1) y = pow(g, x, p) self.public_key = (y, g, p) self.private_key = x
def make_gif(dirpath="../samples/exp0", save2path="../demo/test_imageio.gif"): import imageio import cv2 dirname = "/".join(save2path.split("/")[:-1]) if not os.path.exists(dirname): os.makedirs(dirname) collections = [] for file in generator(root_dir=dirpath, file_type='png', file_label_fun=None, stop_after=None, verbose=1): collections.append(file) # print(file) collections.sort() collections = sorted(collections, key=lambda x: int(x.split("/")[-1].split(".")[0])) # print(collections) print("Reading images...") collections = [cv2.imread(x) for x in collections[100:-100]] # collections = list(map(lambda file:cv2.imread(file), collections)) print("+ Done.") print("Making Gif...") imageio.mimsave(save2path, collections) print("+ Done.")
def get_accuracy(model, test_loader): num_samples = 0 num_correct = 0 for (labels, outputs) in generator(model, test_loader): outputs = outputs > 0 num_samples += torch.numel(labels) num_correct += (labels == outputs).sum().item() return num_correct / num_samples
def _input_fn(): if partition == "train": dataset = tf.data.Dataset.from_generator( generator(x_train, y_train), (tf.float32, tf.int32), ((28 * 28), ())) else: dataset = tf.data.Dataset.from_generator(generator(x_test, y_test), (tf.float32, tf.int32), ((28 * 28), ())) if training: dataset = dataset.shuffle(10 * batch_size, seed=RANDOM_SEED).repeat() dataset = dataset.map(preprocess_image).batch(batch_size) iterator = dataset.make_one_shot_iterator() features, labels = iterator.get_next() return features, labels
def train_model(model, train, valid, batch_size): model.compile(loss='mse', optimizer='adam') # using adam optimizer # saving model at each epoch checkpoint = ModelCheckpoint('model-{epoch:03d}.h5', monitor='val_loss', verbose=0, save_best_only=True, mode='auto') model.fit_generator(generator(train, batch_size), steps_per_epoch=math.ceil(len(train) / batch_size), validation_data=generator(valid, batch_size), validation_steps=math.ceil(len(valid) / batch_size), epochs=1, verbose=1, callbacks=[checkpoint]) model.save('model.h5')
def __init__(self, *args, **kwargs): xbmcgui.WindowXML.__init__(self, *args, **kwargs) #xbmcgui.WindowXMLDialog.__init__(self, *args, **kwargs) #<--- what's the difference? self.subreddits_file = kwargs.get("subreddits_file") self.listing = kwargs.get("listing") self.main_control_id = kwargs.get("id") #self.gui_listbox.addItems(self.listing) listing_generator=generator(self.listing) tlc_id=0 #create id's for top-level-comments self.child_lists[:] = [] #a collection of child comments (non-tlc) tlc_children tlc_children=[] #for listing in self.listing: for listing in listing_generator: depth=int(listing.getProperty('comment_depth')) #add root comments and links on the listbox if not listing.getProperty('link_url'): if depth==0: tlc_id+=1 listing.setProperty('tlc_id',str(tlc_id)) #assign an id to this top-level-comment self.items_for_listbox.append(listing) #this post will be on the listbox #log('tlc: '+listing.getProperty('plot')) #save the set of comments from previous top level comment self.child_lists.append(tlc_children) #begin a new list of child comments(this does not clear the old refernces) tlc_children=[] tlc_children.append( self.get_post_text_tuple(listing) ) #save the post_text of the top level comment else: #collect the child comments. when depth=0 again, this list is reset. child_comment=listing #log(' : '+child_comment.getProperty('plot')) tlc_children.append( self.get_post_text_tuple(child_comment) ) else: #link in tlc if depth>0: listing.setProperty('tlc_id',str(tlc_id)) listing.setProperty('non_tlc_link','true') listing.setProperty('tlc_id',str(tlc_id)) self.items_for_listbox.append(listing) #don't forget to add the children of the last tlc self.child_lists.append(tlc_children) #log(pprint.pformat(self.child_lists)) self.exit_monitor = ExitMonitor(self.close_gui)#monitors for abortRequested and calls close on the gui #can't dynamically create an auto-height textbox inside a grouplist # so we make x of them in the xml and hope they're enough # these are their id's self.x_controls=[x for x in range(1000, 1071)]
def nga_spider(id): url = f'https://bbs.nga.cn/read.php?tid={id}' try: options = webdriver.FirefoxOptions() options.add_argument('-headless') browser = webdriver.Firefox(options=options) except Exception: options = webdriver.chrome.options() options.add_argument('-headless') browser = webdriver.Chrome(options=options) # 打开隐藏部分 click_button = """ document.querySelectorAll(".collapse_btn button").forEach(function(each){each.click()}) """ browser.get(url) sleep(15) # 让 NGA 自己渲染 bbcode browser.execute_script(click_button) content = browser.execute_script('return document.querySelector("html").innerHTML;') sleep(5) # 防止报错:Failed to decode response from marionette browser.close() generator('NGA', get_meta(content, url), get_posts(content), get_date(content))
def fit(self, X, y, M=None, epochs=1, updates_per_batch=1, samples=30, callback=None): """Trains the network with the given in X and y data. epochs: The iteration count over the whole dataset M: The size of the batch that should be itertated over for optimization updates_per_batch: The count of consecetive interations over the same batch samples: samples drawn from the mdoels for calucating grading descent callback: a function to be called every 1000 epochs while training the model """ latent_vars = {} N = y.shape[0] for var, q_var in zip(self.priorWs, self.qWs): latent_vars[var] = q_var for var, q_var in zip(self.priorBs, self.qBs): latent_vars[var] = q_var if M is None: M = N n_batch = int(N / M) n_epoch = epochs data = ut.generator([X, y], M) inference = ed.KLqp(latent_vars, data={self.y: self.y_ph}) inference.initialize(n_iter=n_epoch * n_batch * updates_per_batch, n_samples=samples, scale={self.y: N / M}) tf.global_variables_initializer().run() print("Total iterations: " + str(inference.n_iter)) for i in range(n_epoch): total_loss = 0 for _ in range(inference.n_iter // updates_per_batch // n_epoch): X_batch, y_batch = next(data) for _ in range(updates_per_batch): info_dict = inference.update({ self.y_ph: y_batch, self.X: X_batch }) total_loss += info_dict['loss'] print("Epoch " + str(i) + " complete. Total loss: " + str(total_loss)) if i % 1000 == 0 and callback is not None: callback(self, i)
def gans(ginputs, dinputs, choice, freeze_D): gout = generator(ginputs).get_G() gout = tf.cond( freeze_D, lambda: gout, lambda: tf.stop_gradient(gout)) din = tf.cond( choice, lambda: tf.concat([gout, ginputs], axis = -1), lambda: tf.concat([dinputs, ginputs], axis = -1) ) dout = discriminator(din, freeze_D).get_D() return gout, dout
def get_public_params(len_p): aa = 2**len_p bb = 2**(len_p + 1) #while True: p = utils.get_big_prime(aa, bb) # if sympy.isprime((p - 1) // 2): # break print('genned p') g = utils.generator(p) # print('Сгенерированы параметры \n g: {} \n p: {}'.format(g, p)) with open('log.log', 'a') as f: f.write('Сгенерированы параметры \n\t g: {} \n\t p: {}\n'.format(g, p)) return g, p
def define_network(self): # Generators # This one is used to generate fake data self.gen_b_fake = generator(self.X_shoes, self.initializer, scope_name="generator_sb") self.gen_s_fake = generator(self.X_bags, self.initializer, scope_name="generator_bs") # Reconstruction Generators # Note that parameters are being used from previous layers self.gen_recon_s = generator(self.gen_b_fake, self.initializer, scope_name="generator_sb", reuse=True) self.gen_recon_b = generator(self.gen_s_fake, self.initializer, scope_name="generator_bs", reuse=True) # Discriminator for Shoes self.disc_s_real = discriminator(self.X_shoes, self.initializer, scope_name="discriminator_s") self.disc_s_fake = discriminator(self.gen_s_fake, self.initializer, scope_name="discriminator_s", reuse=True) # Discriminator for Bags self.disc_b_real = discriminator(self.X_bags, self.initializer, scope_name="discriminator_b") self.disc_b_fake = discriminator(self.gen_b_fake, self.initializer, reuse=True, scope_name="discriminator_b")
def onInit(self): xbmc.executebuiltin( "Dialog.Close(busydialog)" ) #important to reset the listbox. when control comes back to this GUI(after calling another gui). # kodi will "onInit" this GUI again. we end up adding items in gui_listbox #self.gui_listbox.reset() #self.exit_monitor = ExitMonitor(self.close_gui)#monitors for abortRequested and calls close on the gui if self.title_bar_text: self.ctl_title_bar = self.getControl(1) self.ctl_title_bar.setText(self.title_bar_text) if self.poster: #self.getControl(2).setImage(self.poster) self.getControl(3).setImage(self.poster) listing_generator=generator(self.listing) for control_id in self.x_controls: tx_control=self.getControl(control_id) bn_control=self.getControl(control_id+1000) img_control=self.getControl(control_id+2000) try: #gether the listitem properties and prepare to put them on the button li=listing_generator.next() link_url=li.getProperty('link_url') label=li.getLabel() label=label.replace(link_url, "") #remove the http:... part to avoid it looking duplicated because it is already put as a label in button. thumb=li.getArt('thumb') log('item_type='+repr(li.getProperty('item_type')) ) log('onClick_action='+repr(li.getProperty('onClick_action')) ) except StopIteration: li=label=link_url=None if label: tx_control.setText(label) #xbmcgui.ControlButton object has no attribute 'setProperty' if link_url=='http://blank.padding': #the regex generating this list is not perfect. a padding was added to make it work. we ignore it here. bn_control.setVisible(False) else: bn_control.setLabel(label=link_url) else: tx_control.setText(None) tx_control.setVisible(False) bn_control.setVisible(False) if thumb: img_control.setImage(thumb) else: img_control.setVisible(False) #hide the unused controls so they don't occupy 'space' in the grouplist
def train(training_data: list, model, model_name: str): full = [] for td in training_data: with open(td, 'rb') as file: full += pickle.load(file) track1_df = pd.DataFrame([(i, a[0]) for i, a in full], columns=['Image', 'Angle']) train_samples, valid_samples = train_test_split(track1_df, test_size=0.2) train_generator = generator(train_samples) valid_generator = generator(valid_samples) es = EarlyStopping(monitor='val_loss', mode='auto', verbose=1, patience=5) cp = ModelCheckpoint(model_name + '.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='auto') history = model.fit(train_generator, steps_per_epoch=ceil(len(train_samples)/32), validation_data=valid_generator, validation_steps=ceil(len(valid_samples)/32), epochs=60, callbacks=[es, cp]) plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('model mean squared error loss') plt.ylabel('mean squared error loss') plt.xlabel('epoch') plt.legend(['training', 'validation'], loc='upper right') plt.savefig(model_name + '-plot')
def run_epoch(model, data_loader, criterion, optimizer=None, scheduler=None): if optimizer is None: assert ( scheduler is None ), "If `scheduler` is provided, you must also specify an `optimizer`" total_loss = 0 for (labels, outputs) in generator(model, data_loader): loss = criterion(outputs, labels) if optimizer: optimizer.zero_grad() loss.backward() optimizer.step() scheduler.step() total_loss += loss.item() return total_loss / len(data_loader)
def get_prime_and_generator(key_length): while True: p = maurer_generator(key_length - 1) q = p * 2 + 1 if millerrabin(q, 1): break print(p) print(q) a = set() while True: g = generator(q, a) a.add(g) print(g) if is_generator(p, q, g): break return q, g
def train_step(video_real, video_wrong, text): num_clips, t, h, w, c = video_real.shape word, sentence = bert([text]) word, sentence = tf.convert_to_tensor(word), tf.convert_to_tensor(sentence) word = tf.squeeze(word) z = tf.random.normal(shape=(1, 100)) with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape: video_fake = generator(video_real, word, z) # All frames put together with bs = 1 video_real = tf.reshape(video_real, (1, num_clips * t, h, w, c)) video_wrong = tf.reshape(video_wrong, (1, num_clips * t, h, w, c)) video_fake = tf.reshape(video_fake, (1, num_clips * t, h, w, c)) # Discriminator out disc_video_real, disc_frame_real, disc_motion_real = discriminator( video_real, sentence) disc_video_wrong, disc_frame_wrong, disc_motion_wrong = discriminator( video_wrong, sentence) disc_video_fake, disc_frame_fake, disc_motion_fake = discriminator( video_fake, sentence) # Losses total_video_loss = video_loss(disc_video_real, disc_video_wrong, disc_video_fake) total_frame_loss = frame_loss(disc_frame_real, disc_frame_wrong, disc_frame_fake) total_motion_loss = motion_loss(disc_motion_real, disc_motion_wrong, disc_motion_fake) disc_loss = discriminator_loss(total_video_loss, total_frame_loss, total_motion_loss) gen_loss = generator_loss(disc_video_fake, disc_frame_fake, disc_motion_fake) gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables) gradients_of_discriminator = disc_tape.gradient( disc_loss, discriminator.trainable_variables) generator_optimizer.apply_gradients( zip(gradients_of_generator, generator.trainable_variables)) discriminator_optimizer.apply_gradients( zip(gradients_of_discriminator, discriminator.trainable_variables))
def train(self, train_data, operations, iter): loss = 0 evaluator = self.getEvaluator() start_time = time.time() for x_train in utils.generator(train_data, operations.m_op, self.config, train=True): _, val, predicted_ner, actual_ner, predicted_rel, actual_rel, _, m_train = self.sess.run( [ operations.train_step, operations.obj, operations.predicted_op_ner, operations.actual_op_ner, operations.predicted_op_rel, operations.actual_op_rel, operations.score_op_rel, operations.m_op ], feed_dict=x_train ) # sess.run(embedding_init, feed_dict={embedding_placeholder: wordvectors}) if self.config.evaluation_method == "relaxed": evaluator.add(predicted_ner, actual_ner, predicted_rel, actual_rel, m_train['BIO']) else: evaluator.add(predicted_ner, actual_ner, predicted_rel, actual_rel) loss += val print('****iter %d****' % (iter)) print('-------Train-------') print('loss: %f ' % (loss)) if self.config.evaluation_method == "relaxed": evaluator.computeInfoMacro() else: evaluator.printInfo() elapsed_time = time.time() - start_time print("Elapsed train time in sec:" + str(elapsed_time)) print()
def evaluate(self, eval_data, operations, set): print('-------Evaluate on ' + set + '-------') evaluator = self.getEvaluator() for x_dev in utils.generator(eval_data, operations.m_op, self.config, train=False): predicted_ner, actual_ner, predicted_rel, actual_rel, _, m_eval = self.sess.run( [ operations.predicted_op_ner, operations.actual_op_ner, operations.predicted_op_rel, operations.actual_op_rel, operations.score_op_rel, operations.m_op ], feed_dict=x_dev) if self.config.evaluation_method == "relaxed": evaluator.add(predicted_ner, actual_ner, predicted_rel, actual_rel, m_eval['BIO']) else: evaluator.add(predicted_ner, actual_ner, predicted_rel, actual_rel) if self.config.evaluation_method == "relaxed": if set == 'test': evaluator.computeInfoMacro(printScores=True) else: evaluator.computeInfoMacro(printScores=True) if "other" in [ x.lower() for x in self.config.dataset_set_ec_tags ]: # if other class exists report score without "Other" class, see previous work on the CoNLL04 return evaluator.getMacroF1scoresNoOtherClass()[2] else: return evaluator.getMacroF1scores()[2] else: evaluator.printInfo(printScores=True) return evaluator.getChunkedOverallAvgF1()
def main(args): if not os.path.exists(args.outdir): os.makedirs(args.outdir) # Prepare output directory # output directory counter out_counter = len([None for out in os.listdir(args.outdir) if args.algo in out]) out = os.path.join(args.outdir, args.algo + '_' + str(out_counter + 1)) if not os.path.exists(out): os.makedirs(out) config = {'data': args.data_path, 'algo': args.algo, 'batch_size': args.batch_size, 'obs_shape': args.obs_size, 'iteration': args.iteration, 'D_step': args.D_step, 'G_step': args.G_step, 'g_optimizer': args.g_optimizer, 'd_optimizer': args.d_optimizer, 'gamma': args.gamma, 'initia_learning_rate': args.initial_lr, 'lr_schedules': args.lr_schedules, 'c_vf': args.c_vf, 'c_entropy': args.c_entropy, 'c_l1': args.c_l1, 'vf_clip': args.vf_clip, 'leaky': args.leaky, 'g_sn': args.g_sn, 'd_sn': args.d_sn} with open(os.path.join(out, 'config.json'), 'w') as f: f.write(json.dumps(config, indent=4)) # Load moving mnist data = np.load(args.data_path) obs_shape = [3, args.obs_size, args.obs_size, 1] # Start generator gen = generator(data, batch_size=args.batch_size, img_size=args.obs_size)
def predict(self, eval_data, operations, set): print('-------Prediction on ' + set + '-------') evaluator = self.getEvaluator() for x_dev in utils.generator(eval_data, operations.m_op, self.config, train=False): predicted_ner, actual_ner, predicted_rel, actual_rel, _, m_eval = self.sess.run( [ operations.predicted_op_ner, operations.actual_op_ner, operations.predicted_op_rel, operations.actual_op_rel, operations.score_op_rel, operations.m_op ], feed_dict=x_dev) if self.config.evaluation_method == "relaxed": evaluator.add_test(predicted_ner, actual_ner, predicted_rel, actual_rel, m_eval['BIO']) else: evaluator.add_test(predicted_ner, actual_ner, predicted_rel, actual_rel)
def populate_tlc_children(self,tlc_id): #controls_generator=generator(controls) child_comments_tuple_generator=generator(self.child_lists[tlc_id]) for control_id in self.x_controls: control=self.getControl(control_id) try: post_text,author,depth=child_comments_tuple_generator.next() except StopIteration: post_text,author,depth=None,None,0 if post_text: #control.setText( ("[B]"+repr(control_id-1000)+"[/B] " + post_text) if post_text else None) #control.setText(post_text+' '+author) #log(('.'*depth)+repr(post_text)) control.setText(post_text) else: control.setText(None) #use animation to stagger the comments according to how deep they are control.setAnimations( [ animation_format(0,100,'slide', 0, (20*depth), 'sine', 'in' ) ] ) #either there's no more child comments or we run out of controls return
def build_model(self): noise_dim = self.config.noise_dim image_dim = self.config.image_dim learning_rate = self.config.learning_rate #create placeholders self.gen_input = tf.placeholder(tf.float32, shape=[None, noise_dim], name="input_noise") self.disc_input = tf.placeholder(tf.float32, shape=[None, image_dim], name="disc_input") self.parameters = give_me_parameters(self.config) #build generator network with tf.name_scope('generator') as scope: self.gen_sample = generator(self.gen_input, self.parameters) with tf.name_scope('discriminator') as scope: disc_real = discriminator(self.disc_input, self.parameters) disc_fake = discriminator(self.gen_sample, self.parameters) with tf.name_scope('loss') as scope: #make losses self.gen_loss = -tf.reduce_mean(tf.log(disc_fake)) self.disc_loss = -tf.reduce_mean(tf.log(disc_real) + tf.log(1. - disc_fake)) with tf.name_scope('summaries') as scope: gen_images = tf.reshape(self.gen_sample, [-1, 28, 28, 1]) tf.summary.scalar('Generative Loss', self.gen_loss) tf.summary.scalar('Discriminator Loss', self.disc_loss) #tf.summary.image("Generated image", gen_images) tf.summary.image("Generated image", gen_images, 100, family='generated_images') self.merged_summary = tf.summary.merge_all() with tf.name_scope('optimizers') as scope: #create Optimizers optimizer_gen = tf.train.AdamOptimizer(learning_rate=learning_rate) optimizer_disc = tf.train.AdamOptimizer(learning_rate=learning_rate) gen_vars = give_gen_vars(self.parameters) disc_vars = give_disc_vars(self.parameters) self.train_gen = optimizer_gen.minimize(self.gen_loss, var_list=gen_vars) self.train_disc = optimizer_disc.minimize(self.disc_loss, var_list=disc_vars)
def train(gpu_num, epochs=15, load_path=None, if_test=False): """网络训练函数 gpu_num:使用的gpu个数(>1) epochs: load_path:不为None时,将从该路径导入模型参数,在该参数基础再训练 if_test:为True时,只进行测试。此时load_path不能为None """ batch_size = 3 * gpu_num # 3 * gpu_num learing_rate = 1e-3 print('load data>>>>') config = tf.ConfigProto() # config.gpu_options.allow_growth = True sess = tf.Session(graph=tf.get_default_graph(), config=config) # (z,y,x) # window_size=(72,80,80), moving_size=(20,24,24),test_moving_size=(40,48,48) data_loader = LobeData(window_size=(32, 224, 224), moving_size=(8, 56, 56), test_moving_size=(16, 112, 112), train_path='./lobe_data2/train/', test_path='./lobe_data2/test/') if not if_test: train_images, train_labels, train_edges, train_coors = data_loader.load_train_data() #train_images, train_labels, train_dists,train_coors, val_images, val_labels, val_dists,val_coors = data_loader.load_train_data(validation = True) test_images, test_labels, test_edges, test_coors, test_names = data_loader.load_test_data() train_coor_num = 0 if not if_test: for temp_coors in train_coors: train_coor_num += len(temp_coors) print ("train_coor_num:", train_coor_num) # val_coor_num = 0 # for temp_coors in val_coors: # val_coor_num += len(temp_coors) # print ("val_coor_num:", val_coor_num) print('data loading complete!') print('model loaded>>>>') print('fitting model>>>>') K.set_session(sess) sess.run(tf.global_variables_initializer()) result_folder=time.strftime("%Y%m%d%H%M", time.localtime()) save_path=os.path.join('./results', 'unet_res_aug_data2_edge2_FA_GP_fissureAtt_fintune_attpredict' + result_folder) log_path = os.path.join(save_path, 'logs') mid_path = os.path.join(save_path, 'mid_res') if not os.path.exists(log_path): os.makedirs(log_path) if not os.path.exists(mid_path): os.mkdir(mid_path) tb = TensorBoard(log_path) predict_callback = PredictCases(test_images, test_labels, test_coors, test_names, batch_size, save_path=mid_path, run_epoch=[0, 2, 5, 8, 10, 12, epochs-1]) #stop = EarlyStopping(patience=4) if if_test: model = unet_residual_fissureAtt(input_shape=(32, 224, 224, 1)) model.load_weights(load_path) predict_cases(model, test_images, test_labels, test_coors, test_names, epoch=0, batch_size=batch_size, save_path=save_path) # predict_cases(model, test_images, test_labels, test_coors, test_names, epoch=0, batch_size=batch_size, save_path=save_path, save_only=True) return with tf.device('/cpu:0'): model = unet_residual_fissureAtt(input_shape=(32, 224, 224, 1)) #(32, 224, 224, 1) if load_path != None: model.load_weights(load_path, by_name=True) model.summary() checkpoint = MyCbk(model, path=save_path) if gpu_num == 1: with tf.device('/gpu:0'): model = unet_residual_fissureAtt(input_shape=(32, 224, 224, 1)) if load_path != None: model.load_weights(load_path, by_name=True) checkpoint = MyCbk(model, path=save_path) parallel_model = model else: parallel_model = multi_gpu_model(model, gpus=gpu_num) f_loss1 = focal_loss(num_classes = 6) f_loss2 = focal_loss(num_classes = 2) dice_loss1 = dice_coef_loss([1, 1, 1, 1, 1, 1]) # [0.606, 0.0831, 0.0346, 0.0944, 0.0955, 0.0862] dice_loss2 = dice_coef_loss(None, classes_w_t2=[0., 1.]) loss_weights = [0.7, 0.3] # [0.7, 0.3] [1.0, 0.0] print("save_path:",save_path) print("load_path:",load_path) print("learing_rate:", learing_rate) print("loss_weights:", loss_weights) print("dice_loss1:", "dice_loss2") parallel_model.compile(optimizer=Adam(lr=learing_rate), loss=[dice_loss1, dice_loss2], loss_weights=loss_weights, metrics=[dice_coef]) #lr=1e-3 # model.fit(x=image_train,y=label_train,batch_size=2,epochs=20,validation_split=0.1,callbacks=[tb]) parallel_model.fit_generator(generator=generator(train_images, train_labels, train_edges, train_coors, batch_size=batch_size), steps_per_epoch=train_coor_num / batch_size, epochs=epochs, #validation_data=generator(val_images, val_labels, val_dists,val_coors, batch_size = batch_size), #validation_steps=val_coor_num / batch_size, verbose=1, max_queue_size=20, # callbacks=[tb, checkpoint, predict_callback] callbacks=[tb, checkpoint]) model.save_weights(os.path.join(save_path, 'weight.h5')) del train_images, train_labels, train_edges, train_coors, model gc.collect() model = unet_residual_fissureAtt(input_shape=(32, 224, 224, 1)) # batch_size = gpu_num * 2 for ep in [0, 2, 5, 8, 10, 12, epochs-1]: model_path = os.path.join(save_path, "model_at_epoch_%d.h5"%ep) model.load_weights(model_path) predict_cases(model, test_images, test_labels, test_coors, test_names, epoch=ep, batch_size=batch_size, save_path=mid_path)
def get_x_generator(): netG = utils.generator(params.uncon_shape[0], params.nz, params.uncon_ngf, params.uncon_shape[2]) state_dict = torch.load(params.uncon_cp_path) netG.load_state_dict(state_dict) return netG
def hello(): form = SpeachForm() filename = '' if request.method == "POST": if form.validate_on_submit(): filename = generator() #wit_response = client.message(str(form.text.data)) c = conversation(form.text.data) response = nlu(form.text.data) #print(response) asd = json.loads(response) if (asd['entities'][0]['disambiguation']['subtype'][0]) == 'City': if asd['concepts'][0]['text'] == 'Weather' and asd['concepts'][ 0]['relevance'] >= .9: #print(asd['entities'][0]['text']) r = requests.get( 'https://{}:{}@twcservice.eu-gb.mybluemix.net/api/weather/v3/location/search?query={}&language=en-US' .format(USERNAME, PASSWORD, asd['entities'][0]['text'])) res = r.json() #print(res['location'], res['location']['latitude'], res['location']['longitude']) #print(res['location']['latitude'][0], res['location']['longitude'][0]) g = geocoder.google([ res['location']['latitude'][0], res['location']['longitude'][0] ], method='reverse') #print(g.json) #r = requests.get('https://{}:{}@twcservice.eu-gb.mybluemix.net/api/weather/v1/geocode/{}/{}/observations.json'.format(USERNAME, PASSWORD, res['location']['latitude'][0], res['location']['longitude'][0])) r = requests.get( 'https://{}:{}@twcservice.eu-gb.mybluemix.net/api/weather/v1/geocode/{}/{}/forecast/daily/7day.json' .format(USERNAME, PASSWORD, res['location']['latitude'][0], res['location']['longitude'][0])) res = r.json() fields = [ 'temp', 'pop', 'uv_index', 'narrative', 'phrase_12char', 'phrase_22char', 'phrase_32char' ] print(res['forecasts'][0]['narrative']) #print(res['forecasts'][0]['night']) print(res['forecasts'][0]['night']['narrative']) text_to_speach( 'by day: ' + str(res['forecasts'][0]['narrative']) + ', at night: ' + str(res['forecasts'][0]['night']['narrative']), filename) #for i in res['metadata']: # print('METADATA : ' + str(i)) #r = requests.get('https://{}:{}@dda5b6bb-a242-4438-b1c9-9ed7edc4f2a6:[email protected]:443/'.format(USERNAME, PASSWORD)) else: answer = json.loads(c) answer = answer['output']['text'][0] #flash('I heard you say: ' + str(form.text.data)) #flash('Yay, got Wit.ai response: ' + str(wit_response)) #flash('I understand: ' + response) #flash('Answer from bot: ' + str(c)) flash('You: ' + str(form.text.data)) flash('Julia: ' + str(answer)) #text_to_speach(answer, filename) return render_template('index.html', form=form, play=True, filename=filename) return render_template('index.html', form=form, play=True, filename=filename)
def get_hamming_accuracy(model, test_loader): scores = [] for (labels, outputs) in generator(model, test_loader): score = _hamming_accuracy(labels, outputs) scores.append(score) return sum(scores) / len(scores)
def train(self, train_data, operations, iter): self.is_train = True loss = 0 evaluator = self.getEvaluator() if self.config.ner_classes == "BIO": tagset = self.config.dataset_set_bio_tags tag2id = {k: v for v, k in enumerate(tagset)} id2tag = {v: k for v, k in enumerate(tagset)} relset = self.config.dataset_set_bio_relation_ners rel2id = {k: v for v, k in enumerate(relset)} id2rel = {v: k for v, k in enumerate(relset)} start_time = time.time() for x_train in utils.generator(train_data, operations.m_op, self.config, train=True): _, val, m_train, transition_params1, entity1Scores, predEntity1, transition_params2, entity2Scores, predEntity2 = self.sess.run( [ operations.train_step, operations.obj, operations.m_op, operations.transition_params1, operations.entity1Scores, operations.predEntity1, operations.transition_params2, operations.entity2Scores, operations.predEntity2 ], feed_dict=x_train) for i in range(len(predEntity1)): trueNer, trueRel, predNer, predRel = [], [], [], [] trueSeq = m_train['entity1_tags_ids'][i] predSeq = predEntity1[i] k1 = m_train['k1'][i] k2 = m_train['k2'][i] trueNer += utils.ConvertToEntity(trueSeq, tag2id) predNer += utils.ConvertToEntity(predSeq, tag2id) tners1 = utils.ConvertToEntity(trueSeq[k1:k2 + 1], tag2id) pners1 = utils.ConvertToEntity(predSeq[k1:k2 + 1], tag2id) trueSeq2 = m_train['entity2_tags_ids'][i] predSeq2 = predEntity2[i] tners2, trels = utils.getRel(trueSeq2, rel2id, id2rel) pners2, prels = utils.getRel(predSeq2, rel2id, id2rel) trueNer += tners2 predNer += pners2 trueRel = utils.collectNerAndRel(tners1, tners2, trels) predRel = utils.collectNerAndRel(pners1, pners2, prels) evaluator.add(predNer, trueNer, predRel, trueRel) loss += val print('****iter %d****' % (iter)) print('-------Train-------') print('loss: %f ' % (loss)) # if self.config.evaluation_method == "relaxed": # evaluator.computeInfoMacro() # else: evaluator.printInfo() elapsed_time = time.time() - start_time print("Elapsed train time in sec:" + str(elapsed_time)) return loss
from skimage.measure import label,regionprops np.random.seed(1) CLF_SIZE=256 parent_path="E:/Kaggle/RSNA Pneumonia Detection Challenge/" model_dir=os.path.join(parent_path,"seg_models/COORD_ASPP_UNET/") models=["model_0.h5","model_1.h5","model_2.h5"] data_list=get_opacity_list() chosed_idxs=np.random.choice(len(data_list),5,replace=False) val_list=data_list[chosed_idxs] avg_preds=np.zeros((len(val_list),CLF_SIZE,CLF_SIZE,1)) gen=generator(len(val_list),val_list,CLF_SIZE) val_x,val_y=gen.__next__() for model_p in models: model_path=os.path.join(model_dir,model_p) model=MODEL(CLF_SIZE) model.load_weights(model_path) preds=model.predict(val_x,verbose=1) avg_preds+=preds avg_preds/=len(models) avg_preds=avg_preds>0.2
def evaluate(self, eval_data, operations, set): import tensorflow as tf self.is_train = False print('-------Evaluate on ' + set + '-------') evaluator = self.getEvaluator() if self.config.ner_classes == "BIO": tagset = self.config.dataset_set_bio_tags tag2id = {k: v for v, k in enumerate(tagset)} id2tag = {v: k for v, k in enumerate(tagset)} relset = self.config.dataset_set_bio_relation_ners rel2id = {k: v for v, k in enumerate(relset)} id2rel = {v: k for v, k in enumerate(relset)} # lstm_out = self.lstm(reuse=True) for x_dev, doc in utils.generator(eval_data, operations.m_op, self.config, train=False): # print(doc.docId) trueNer, trueRel = utils.getNerAndRel(doc) predNer, predRel = [], [] entity1Scores, predEntity1s = self.sess.run( [operations.entity1Scores, operations.predEntity1], feed_dict=x_dev) # lstm_out = tf.convert_to_tensor(lstm_out[0], dtype=tf.float32) for k in range(len(predEntity1s)): predEntity1 = predEntity1s[k] if self.config.ner_loss == "crf": i = 0 while i < len(predEntity1): tagid = predEntity1[i] default = tag2id['O'] if tagid != default: ner1 = utils.getNer(predEntity1, i, id2tag) if ner1[0] == 'Other' or ner1[0] == 'other': end = ner1[2] i = end + 1 continue predNer.append(ner1) end = ner1[2] k1 = np.asarray([i]) k2 = np.asarray([end]) x_dev[operations.m_op['k1']] = k1 x_dev[operations.m_op['k2']] = k2 predEntity2s = self.sess.run( operations.predEntity2, feed_dict=x_dev) predEntity2 = predEntity2s[0] # print(len(predEntity2)) ners2, rels = utils.getRel(predEntity2, rel2id, id2rel) predNer += ners2 assert len(ners2) == len(rels) for j in range(len(rels)): r = rels[j] ner2 = ners2[j] e1 = (ner1[1], ner1[2]) e2 = (ner2[1], ner2[2]) predRel.append((e1, r, e2)) i = end + 1 else: i += 1 # elif self.config.ner_loss == "softmax": # _, entity1Scores, label_matrix = self.sess.run([self.model1(lstm_out)]) # predEntity1 = self.sess.run([tf.cast(tf.arg_max(entity1Scores, 2), tf.int32)]) # predEntity1 = predEntity1[0] # i = 0 # while i < len(predEntity1): # tagid = predEntity1[i] # default = tag2id['O'] # if tagid != default: # ner1 = utils.getNer(predEntity1, i, id2tag) # predNer.append(ner1) # end = ner[2] # fd = {m_op['k1']:np.asarray([i]), m_op['k2']:np.asarray([end])} # entity2Scores = self.sess.run([self.model2(lstm_out, None, entity1Scores, label_matrix)], feed_dict=fd) # predEntity2 = tf.cast(tf.arg_max(entity2Scores, 2), tf.int32) # ners2, rels = utils.getRel(predEntity2, rel2id, id2rel) # print(ners2) # predNer += ners2 # assert len(ners2) == len(rels) # for j in range(len(rels)): # r = rels[i] # ner2 = ners2[i] # e1 = (ner1[1], ner1[2]) # e2 = (ner2[1], ner2[2]) # predRel.append((e1, r, e2)) # i = end + 1 # else: # i += 1 # print(predNer) evaluator.add(predNer, trueNer, predRel, trueRel) evaluator.printInfo() return evaluator.getChunkedOverallAvgF1()
import tensorflow as tf import os import utils batch_size = 100 iterations = 10000 im_size = 64 layer = 3 z_size = 100 z_in = tf.placeholder(shape=[batch_size,z_size], dtype=tf.float32) real_in = tf.placeholder(shape=[batch_size,im_size,im_size,layers], g = utils.generator(z_in,im_size, layers,batch_size) d = utils.discriminator(real_in,batch_size d_fake = utils.discriminator(g, batch_size, reuse=True) d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(d, tf.ones_like(d))) d_loss = d_loss_fake
def __init__(self,batch_size=10,im_size=64,channels=3,dtype=tf.float32,analytics=True): self.analytics = analytics self.batch_size = batch_size self.x_a = tf.placeholder(dtype,[None,im_size,im_size,channels],name='xa') self.x_b = tf.placeholder(dtype,[None,im_size,im_size,channels],name='xb') #Generator Networks self.g_ab = utils.generator(self.x_a,name="gen_AB",im_size=im_size) self.g_ba = utils.generator(self.x_b,name="gen_BA",im_size=im_size) #Secondary generator networks, reusing params of previous two self.g_aba = utils.generator(self.g_ab,name="gen_BA",im_size=im_size,reuse=True) self.g_bab = utils.generator(self.g_ba,name="gen_AB",im_size=im_size,reuse=True) #Discriminator for input a self.disc_a_real = utils.discriminator(self.x_a,name="disc_a",im_size=im_size) self.disc_a_fake = utils.discriminator(self.g_ba,name="disc_a",im_size=im_size,reuse=True) #Discriminator for input b self.disc_b_real = utils.discriminator(self.x_b,name="disc_b") self.disc_b_fake = utils.discriminator(self.g_ab,name="disc_b",reuse=True) #Reconstruction loss for generators self.l_const_a = tf.reduce_mean(utils.huber_loss(self.g_aba,self.x_a)) self.l_const_b = tf.reduce_mean(utils.huber_loss(self.g_bab,self.x_b)) #Generation loss for generators self.l_gan_a = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.disc_a_fake,labels=tf.ones_like(self.disc_a_fake))) self.l_gan_b = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.disc_b_fake,labels=tf.ones_like(self.disc_b_fake))) #Real example loss for discriminators self.l_disc_a_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.disc_a_real,labels=tf.ones_like(self.disc_a_real))) self.l_disc_b_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.disc_b_real,labels=tf.ones_like(self.disc_b_real))) #Fake example loss for discriminators self.l_disc_a_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.disc_a_fake,labels=tf.zeros_like(self.disc_a_fake))) self.l_disc_b_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.disc_b_fake,labels=tf.zeros_like(self.disc_b_fake))) #Combined loss for individual discriminators self.l_disc_a = self.l_disc_a_real + self.l_disc_a_fake self.l_disc_b = self.l_disc_b_real + self.l_disc_b_fake #Total discriminator loss self.l_disc = self.l_disc_a + self.l_disc_b #Combined loss for individual generators self.l_ga = self.l_gan_a + self.l_const_b self.l_gb = self.l_gan_b + self.l_const_a #Total GAN loss self.l_g = self.l_ga + self.l_gb #Parameter Lists self.disc_params = [] self.gen_params = [] for v in tf.trainable_variables(): if 'disc' in v.name: self.disc_params.append(v) if 'gen' in v.name: self.gen_params.append(v) if self.analytics: self.init_analytics() self.gen_a_dir = 'generator a->b' self.gen_b_dir = 'generator b->a' self.rec_a_dir = 'reconstruct a' self.rec_b_dir = 'reconstruct b' self.model_directory = "models" if not os.path.exists(self.gen_a_dir): os.makedirs(self.gen_a_dir) if not os.path.exists(self.gen_b_dir): os.makedirs(self.gen_b_dir) if not os.path.exists(self.rec_b_dir): os.makedirs(self.rec_b_dir) if not os.path.exists(self.rec_a_dir): os.makedirs(self.rec_a_dir) self.sess = tf.Session() self.saver = tf.train.Saver()
def hupu_spider(id): url = f'https://bbs.hupu.com/{id}.html' generator('虎扑', get_meta(url), get_posts(url), get_date(url))
def tieba_spider(id): url = f'https://tieba.baidu.com/p/{id}?see_lz=1' generator('贴吧', get_meta(url), get_posts(url), get_date(url))