def send_email(send_to, templaterich, templateplain, subject, **kwargs): """ Sends an email to the target email with two types 1) HTML 2) Plain We will try the template with .htm for rich and .txt for plain. Both will rendered with Jinja2 """ mailer = Mailer(dict( transport=dict(use='smtp', host=config.EMAIL_SMTP_SERVER, debug=config.EMAIL_DEBUG), manager=dict())) mailer.start() message = mailer.new() message.author = config.EMAIL_SENDER message.to = send_to message.subject = subject template_rich = env.get_template(templaterich) template_plain = env.get_template(templateplain) message.rich = template_rich.render(**kwargs) message.plain = template_plain.render(**kwargs) logger.info('Sent an email to ' + send_to) message.send() mailer.stop()
def dataHandleCoroutine(self): length = self.factory.dataprotocl.getHeadlength() while True: data = yield self.buff += data while len(self.buff) >= length: try: self.factory.dataprotocl.unpack(self.buff[:length]) except Exception: logger.info('receive illegal header package') self.transport.loseConnection() break rlength = self.factory.dataprotocl.datalen command = self.factory.dataprotocl.command cmdid = self.factory.dataprotocl.cmdid request = self.buff[length:length + rlength] if len(request) < rlength: logger.info('some data lost') break self.buff = self.buff[length + rlength:] d = self.factory.doDataReceived(self, command, request) if not d: continue d.addCallback(self.safeToWriteData, cmdid) d.addErrback(DefferedErrorHandle)
def remote_takeProxy(self, name, transport): '''添加node节点 ''' logger.info('node [%s] connect' % name) node = Node(name, name) self.nodesmanager.addNode(node) node.setTransport(transport) self.doNodeConnect(name, transport)
def storage(self, data): if not self.parse_audience(data['audience']): logger.info('audience %s not found' % data['audience']) return ErrorPage(ErrNo.NO_MATCHED_OBJ) msg = self.parse_nofification(data['app_key'], data['notification'], data.get('options', None)) # send msg async threads.deferToThread(self.send_to_router, data['audience'], msg) return SuccessPage(msg.to_dict(exclude=('app_key', 'generator', 'title', 'body', 'expires')))
def dropNodeSessionId(self, session_id): '''删除子节点记录''' node = self.nodesmanager.getNodeBySessionId(session_id) if not node: return logger.info('node [%s] lost' % node._name) node_id = node._id self.doNodeLostConnect(node_id) self.nodesmanager.dropNodeByID(node_id)
def _push(connID, did, data): msg = PushMessage() msg.id = data['id'] msg.sendno = data['sendno'] msg.generator = data['generator'] msg.title = data['title'] msg.body = data['body'] msg.timestamp = data['timestamp'] logger.info('push message(id:%d, did:%s)' % (msg.id, did)) factory.connmanager.pushObject(DataPackProtoc.CMD_PUAH, msg.SerializeToString(), [connID])
def getAvgTXPwrByDistance(self): avgtxs = [] uniqueDistances = list(set(self.distances)) uniqueDistances.sort() logger.info("Found %d unique distances" % len(uniqueDistances)) logger.info("Calculating Average TXPwr for each distance") for distance in uniqueDistances: es = self.findEventByDistance(distance) avgtxs.append(es.avgtxpwr) return uniqueDistances, np.array(avgtxs)
def train(self, counter=1, gen_dirs=()): if self.conf.need_to_load: self.load(self.conf.checkpoint_dir, step=counter) data = self.data logger.info('Total amount of images: %s' % len(data)) # np.random.shuffle(data) tf.initialize_all_variables().run() # counter = 1 start_time = time.time() batch_idxs = min(len(data), self.conf.train_size) / self.conf.batch_size stego_accuracy = 0 accuracies = [] accuracies_steps = [] logger.debug('Starting updating') for epoch in range(self.conf.epoch): losses = [] np.random.shuffle(data) logger.info('Starting epoch %s' % epoch) for idx in range(0, int(batch_idxs)): batch_files = data[idx * self.conf.batch_size:(idx + 1) * self.conf.batch_size] batch = [get_image(batch_file, self.conf.image_size) for batch_file in batch_files] batch_images = np.array(batch).astype(np.float32) batch_targets = self.get_targets(batch_files) self.sess.run(self.optimize, feed_dict={self.images: batch_images, self.target: batch_targets}) loss = self.loss.eval({self.images: batch_images, self.target: batch_targets}) losses.append(loss) # logger.debug("[ITERATION] Epoch [%2d], iteration [%4d/%4d] time: %4.4f, Loss: %8f, accuracy: %8f" % # (epoch, idx, batch_idxs, time.time() - start_time, loss, stego_accuracy)) counter += 1 if counter % 300 == 0: logger.info('------') stego_accuracy = self.accuracy(n_files=-1, test_dir=self.test_dir) logger.info('[TEST] Epoch {:2d} accuracy: {:3.1f}%'.format(epoch + 1, 100 * stego_accuracy)) for gen_dir in gen_dirs: gen_accuracy = self.accuracy(n_files=-1, test_dir=gen_dir) logger.info('[GEN_TEST] Folder {}, accuracy: {:3.1f}%'.format(gen_dir, 100 * gen_accuracy))
def drawTXRegion(self,pos,size): draw = ImageDraw.Draw(self.cimg) x,y = pos x = np.int16(np.floor(x)) y = np.int16(np.floor(y)) logger.info("x:%f,y:%f" % (x,y)) draw.rectangle((x-size,y-size,x+size,y+size), outline=(255,255,0), fill=(255,0,255)) draw.text((x-size/2,y-size/2), "Rep") return np.asarray(self._cimg)
def getAvgPathlossByDistance(self): avgloss = [] uniqueDistances = list(set(self.distances)) uniqueDistances.sort() logger.info("Found %d unique distances" % len(uniqueDistances)) logger.info("Calculating Average Pathloss for each distance") for distance in uniqueDistances: es = self.findEventByDistance(distance) avgloss.append(es.avgpathloss) return uniqueDistances, np.array(avgloss)
def plot(self): logger.info("Plotting Data") self.fig1 = pyplot.figure(1) self.fig1.show() rxpwrs = self.events.rxpwrs pyplot.scatter(self.events.distances, self.events.rxpwrs) pyplot.plot(self.dps, self.rxp, c='r') pyplot.scatter(self.dps,self.rxp,c='r') ds = np.arange(100,1300,5) estrx = ChannelModel.estPwr(self.kappa,self.eta,ds) pyplot.plot(ds,estrx,c='g', linewidth=3)
def getAppData(key): logger.info("Hitting URL %s", request.url) error, response = None, None response = AppDataCore.getAppDataForKey(key) if response == None: error = "Invalid data requested" logger.debug("Error: %s", error) logger.debug("Response: %s", response) return jsonify(response=response, error=error)
def save(self, checkpoint_dir, step): model_dir = "%s_%s" % (self.conf.model_name, self.conf.batch_size) checkpoint_dir = os.path.join(checkpoint_dir, model_dir) if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) ckpt_name = "%s_%s.ckpt" % (self.conf.model_name, step) logger.info("[SAVING] step: %s, name: %s" % (step, ckpt_name)) self.saver.save(self.sess, os.path.join(checkpoint_dir, ckpt_name), global_step=step)
def readfile(): logger.info("Hitting URL %s", request.url) rawContent = None filePath = request.args.get("file_path") error, response = FileIO.readFile(filePath) if error: response = None logger.debug("Error: %s", error) logger.debug("Response: %s", response) return jsonify(response=response, error=error)
def component_proxy(message): """ Proxy between email component and sms interpreter. Email component event returns an email.Message object and the interpreter takes 2 arguments: text and a mobile number. """ response_dictionary = sms.send_sms(str(message["To"].split('@')[0]), message.get_payload()) if response_dictionary["status"] == 200: logger.info("SMS Sent out successfully to: " + str(message["To"].split('@')[0])) else: logger.info("SMS Sent out to " + str(message["To"].split('@')[0]) + " with the status code: " + str( response_dictionary["status"]))
def getProfileInfo(profileName): logger.info("Hitting URL %s", request.url) error, response = None, None appData = AppDataCore.getAppData() response = appData["profile"].get(profileName, None) if response == None: error = "Profile %s does not exist." % (profileName) logger.debug("Error: %s", error) logger.debug("Response: %s", response) return jsonify(response=response, error=error)
def load(self, checkpoint_dir, step): model_dir = "%s_%s" % (self.conf.model_name, self.conf.batch_size) checkpoint_dir = os.path.join(checkpoint_dir, model_dir) try: ckpt_name = "%s_%s.ckpt-%s" % (self.conf.model_name, step, step) logger.info("[LOADING] step: %s, name: %s" % (step, ckpt_name)) self.saver.restore(self.sess, os.path.join(checkpoint_dir, ckpt_name)) except Exception as e: logger.debug(e) ckpt_name = "StegoDCGAN-%s" % (step) logger.info("[LOADING] step: %s, name: %s" % (step, ckpt_name)) self.saver.restore(self.sess, os.path.join(checkpoint_dir, ckpt_name))
def __init__(self, events, estkappa=-35.0, esteta=2.0, maxloop=1000): self.estkappa = estkappa self.esteta = esteta self.eta = self.esteta self.kappa = self.estkappa self.events = events self.maxloop = maxloop self.fp = lambda v,x,y: ChannelModel.estDistance(v[0],v[1],x,y) self.err = lambda v,x,y,z: (self.fp(v,x,y)-z) self.v0 = [self.estkappa, self.esteta] logger.info("Sorting By Distance") self._sortByDistance() logger.info("Calculating Optimized Parameters") self._optimizeParams()
def check_acking_queue(): current_time = int(time.time()) rets = msgQueue.get_from_acking() if len(rets) == 0: return for item in rets: did = item[0] msg = item[1] expires = int(msg['expires']) if current_time > expires: logger.info('message(id:%d, did:%s) expired, drop it' % (int(msg['id']), did)) msgQueue.rem_from_acking(did, msg['id']) msgQueue.add_to_dead(did, msg['id'], MessageStatus.EXPIRED) else: # TODO: resend??? pass
def addProfile(profile): logger.info("Hitting URL %s", request.url) error, response = None, None action = request.form.get("action") if action == "add": error, response = TagCore.addProfile(profile) elif action == "delete": error, response = TagCore.removeProfile(profile) if error: response = None logger.debug("Error: %s", error) logger.debug("Response: %s", response) return jsonify(response=response, error=error)
def send_to_router(): current_time = int(time.time()) rets = msgQueue.get_from_sending() if len(rets) == 0: return for item in rets: did = item[0] msg = item[1] expires = int(msg.pop('expires')) msg['sendno'] = int(msg['sendno']) msg['timestamp'] = int(msg['timestamp']) msg['id'] = int(msg['id']) if current_time > expires: logger.info('message(id:%d, did:%s) not send and expired, drop it' % (msg['id'], did)) msgQueue.rem_from_sending(did, msg['id']) msgQueue.add_to_dead(did, msg['id'], MessageStatus.NOT_SEND) else: defer = remote.callRemote('is_device_online', did) defer.addCallback(_send_callback, did, msg)
def accuracy(self, test_dir='test', abs=False, n_files=2 ** 12): logger.info('[TEST], test data folder: %s, n_files: %s' % (test_dir, 2 * n_files)) X_test = self.get_images_names('%s/*.%s' % (test_dir, self.conf.img_format), abs=abs)[:n_files] accuracies = [] batch_idxs = min(len(X_test), self.conf.train_size) / self.conf.batch_size # logger.debug('Starting iteration') for idx in range(0, int(batch_idxs)): batch_files_stego = X_test[idx * self.conf.batch_size:(idx + 1) * self.conf.batch_size] batch = [get_image(batch_file, self.conf.image_size) for batch_file in batch_files_stego] batch_images = np.array(batch).astype(np.float32) batch_targets = self.get_targets(batch_files_stego) accuracies.append(self.get_accuracy(batch_images, batch_targets)) return np.mean(accuracies)
def main(): logger.info('Started.') gmail = build('gmail', 'v1', http=make_google()) gmail_send = gmail.users().messages().send email_messages = {} with open(CONFIG_FILE, 'r') as file: users = json.load(file) logger.debug('User loaded') for endpoint in ENDPOINTS: logger.info('Processing endpoint: %s', endpoint.name) endpoint.request() if not make_cache(endpoint.name, endpoint._text): logger.debug('Content is same with cached. Skipping.') continue for user in users: logger.debug('Processing user: %s', user['name']) if user['name'] in endpoint: logger.debug('User in endpoint.') message = make_message('match', regexp=user['name'], name=endpoint.name, url=endpoint.url) email_messages.setdefault(user['email'], []).append(message) logger.info('Done fetching endpoints. Now drafting email.') queue = [] for recepient, messages in email_messages.items(): message_text = make_message('frame', matches='\n'.join(messages)) message = MIMEText(message_text) message['to'] = recepient # message['from'] = sender message['subject'] = MAIL_SUBJECT # The byte/str drama, you know. raw = base64.b64encode(message.as_string().encode()) queue.append({'raw': raw.decode()}) logger.info('%d email(s) have been drafted. Sending.', len(queue)) consume_message_queue(gmail_send, queue) logger.info('Done.')
def run(self, stop_on_failure=False): fail_cnt = 0 metadata = None getter = self.iter_get() try: while True: try: # Some getters require metadata from the last response to figure out the # pagination of the next one data_str = getter.send(metadata) logger.debug('Got: %s' % data_str) logger.info('Got records') data = self.parse(data_str) logger.debug('Parsed into: %s' % str(data)) metadata, records = self.normalize(data) logger.debug('Normalized: \nMetadata: %s \nRecords: %s' % \ (str(metadata), str(records))) logger.info('Normalized %d records' % len(records)) logger.info('Storing') self.store(metadata, records) except Exception, e: if isinstance(e, StopIteration): raise if stop_on_failure or fail_cnt > 100: raise else: fail_cnt += 1 sys.stderr.write("Failed: %s" % str(e)) continue except StopIteration: pass
def main(_): logger.info('====================================================') logger.info('===================NEW EXPERIMENT===================') logger.info('====================================================') logger.info(flags.FLAGS.__flags) if not os.path.exists(FLAGS.checkpoint_dir): os.makedirs(FLAGS.checkpoint_dir) if not os.path.exists(FLAGS.sample_dir): os.makedirs(FLAGS.sample_dir) with tf.Session() as sess: steganalisys = Steganalyzer(config=FLAGS, sess=sess, stego_algorithm=LSBMatching, stego_name='lsb_matching') if FLAGS.is_train: steganalisys.train(counter=1, gen_dirs=['gen_test_seed_666', 'gen_test_more_train']) else: # steganalisys.load(FLAGS.checkpoint_dir, step=28481) # LSB matching clf steganalisys.load(FLAGS.checkpoint_dir, step=122114) print('ACCURACY:::::::::%s' % steganalisys.accuracy(test_dir='gen_test_more_train', n_files=-1))
def normalize_metadata(self, data): logger.info('Original Structure:\n%s' % get_structure(data)) records = [d['data'] for d in data['data']['children']] return {'after': data['data']['after']}, records
def normalize_metadata(self, data): data = fix_single_elems(data) logger.info('Original Structure:\n%s' % get_structure(data)) return {}, [data['result']['project']]
# +++++++++++++++++++++++++++++++++++++++++++++++++++ import time import numpy as np from data.load_data import load_data from utils.args import args from utils.logger import logger from utils.calc_hamming_ranking import calc_hamming_ranking from dlfh_algo import dlfh_algo from hash_func import linear_hash if __name__ == "__main__": np.random.seed(args.seed) '''load dataset''' logger.info('load dataset: {}'.format(args.dataname)) test_text, test_image, test_label, database_text, database_image, database_label = load_data( ) '''learning procedure''' logger.info('start training procedure') start_t = time.time() db_image_codes, db_text_codes = dlfh_algo(train_labels=database_label) '''out-of-sample extension''' wx = linear_hash(database_image, db_image_codes) wy = linear_hash(database_text, db_text_codes) end_t = time.time() - start_t '''start encoding''' test_image_codes = np.sign(test_image.dot(wx)) test_text_codes = np.sign(test_text.dot(wy))
def fastai_prediction(train_df, test_df, test_folder, column_list, path, topic): result_dict = {} for column in column_list: df_trn = train_df[['image_path', column]].copy() df_trn.columns = ['name', 'label'] df_trn = df_trn[~pd.isnull(df_trn['label'])] df_trn['label'] = df_trn['label'].astype(str) tfms = get_transforms() data = ImageDataBunch.from_df(path, df=df_trn, size=224, num_workers=6, ds_tfms=tfms, valid_pct=0.0001) data.normalize() learn = create_cnn(data, models.resnet34, metrics=accuracy) learn.fit(1) learn.unfreeze() learn.fit(5) train_metric = learn.validate(learn.data.train_dl) val_metric = learn.validate(learn.data.valid_dl) train_loss = train_metric[0] val_loss = val_metric[0] train_acc = train_metric[1].item() val_acc = val_metric[1].item() logger.info('Train Loss on Topic {} : {}'.format(column, train_loss)) logger.info('Validation Loss on Topic {} : {}'.format( column, val_loss)) logger.info('Train Accuracy on Topic {} : {}'.format( column, train_acc)) logger.info('Validation Accuracy on Topic {} : {}'.format( column, val_acc)) learn.export(fname='{}_{}_export.pkl'.format(topic, column)) test = ImageList.from_folder(path / test_folder) final_learn = load_learner(path, test=test, fname='{}_{}_export.pkl'.format( topic, column)) test_items = final_learn.data.label_list.test.items test_preds = final_learn.get_preds(ds_type=DatasetType.Test) pred_df = pd.DataFrame(to_np(test_preds[0]), columns=learn.data.classes) pred_df['image_path'] = test_items pred_df['image_path'] = pred_df['image_path'].apply( lambda y: str(y).split('/')[-1]) test_df['image_path'] = test_df['image_path'].apply( lambda y: str(y).split('/')[-1]) merge_df = test_df[['image_path']].merge(pred_df, on='image_path', how='left', validate='1:1') merge_df = merge_df.drop(columns=['image_path']) result_dict[column] = merge_df return result_dict
def stop(self): for idx, proc in enumerate(self.procs): proc.join() if self.verbose: logger.info("process: {} done".format(idx))
return "Password Mismatch," + "NOT MINING" if consts.NO_MINING else "MINING" @app.route("/<url:re:.+>") @error(403) @error(404) @error(505) def error_handle(url="url", error="404"): log_ip(request, inspect.stack()[0][3]) return template("error.html") if __name__ == "__main__": try: if consts.NEW_BLOCKCHAIN: logger.info("FullNode: Starting New Chain from Genesis") BLOCKCHAIN.add_block(genesis_block) else: # Restore Blockchain logger.info("FullNode: Restoring Existing Chain") header_list = read_header_list_from_db() BLOCKCHAIN.build_from_header_list(header_list) # Sync with all my peers sync_with_peers() # Start mining Thread Thread(target=start_mining_thread, daemon=True).start() if consts.NO_MINING: logger.info("FullNode: Not Mining")
def save_pt_key(pin, ptkey): Jd.update( pin=pin, ptkey=ptkey, ).where(Jd.pin == pin).execute() logger.info("更新京东用户ptkey【{}】信息".format(pin))
def fit_generator(self): # 保存超参数 self.parameters['model_env_parameters'][ 'is_training'] = False # 预测时候这些设为False self.parameters['model_env_parameters']['trainable'] = False save_json(jsons=self.i2l, json_path=self.index2label_path) save_json(jsons=self.parameters, json_path=self.path_parameters) model_code = self.model_code # DataGenerator只是一种为了节约内存的数据方式 class DataGenerator: def __init__(self, data, l2i, tokenizer, categories, maxlen=128, batch_size=32, shuffle=True): self.data = data self.l2i = l2i self.batch_size = batch_size self.categories = categories self.maxlen = maxlen self.tokenizer = tokenizer self.shuffle = shuffle self.steps = len(self.data) // self.batch_size if len(self.data) % self.batch_size != 0: self.steps += 1 def __len__(self): return self.steps def __iter__(self): while True: idxs = list(range(len(self.data))) if self.shuffle: np.random.shuffle(idxs) X, Y = [], [] for i in idxs: d = self.data[i] text = d[1][:self.maxlen].replace(' ', '') x = self.tokenizer.encode( text, algo_code=model_code) # token_ids # print(text) # print(x) y = self.l2i.get(str(d[0])) X.append(x) Y.append(y) if len(X) == self.batch_size or i == idxs[-1]: X = seq_padding(X, 0, self.maxlen) Y = np.array(to_categorical(Y, self.categories)) # print("*"*10,X.shape) yield (X, Y) X, Y = [], [] train_D = MyDataGenerator(self.train_data, self.l2i, self.tokenizer, self.categories, self.max_len, self.batch_size, shuffle=True) valid_D = MyDataGenerator(self.valid_data, self.l2i, self.tokenizer, self.categories, self.max_len, self.batch_size, shuffle=True) # test_D = DataGenerator(self.test_data, self.l2i,self.tokenizer, self.categories, self.max_len, self.batch_size, # shuffle=True) # 模型训练 history = self.model.fit_generator( train_D.__iter__(), steps_per_epoch=len(train_D), epochs=self.epoch, validation_data=valid_D.__iter__(), validation_steps=len(valid_D), callbacks=self.callback(), ) epoch = history.epoch[-1] + 1 acc = history.history['acc'][-1] val_acc = history.history['val_acc'][-1] logger.info("model:{} last_epoch:{} train_acc{} val_acc{}".format( self.model_code, epoch, acc, val_acc))
def operation(self, flag, index, section, buttonName, capabilityNamelist=[]): """ :param flag,index,section,buttonName,capabilityNamelist=[] :return: """ if self.is_usm_page() == True: if buttonName == "New" or (section == "Roles" and buttonName == "Edit"): self.ctrl_all(UsmEntity().get_input_textbox("name")) self.type(UsmEntity().get_input_textbox("name"), "UI automation" + nowTime) self.ctrl_all(UsmEntity().get_input_textbox("description")) self.type(UsmEntity().get_input_textbox("description"), BasePage(self.driver).randomData("string", 6)) self.sleep(2) permission_list = [ "Read", "Update", "Create", "Delete", "MassUpdate" ] # capability_list = self.find_elements(UsmEntity.capabilitity_list) capability_item = None capability_index = [] for i in capabilityNamelist: self.ctrl_all(UsmEntity.filter) self.type(UsmEntity.filter, i) capability_list = self.find_elements( UsmEntity.capabilitity_list) for j, item in enumerate(capability_list): if i == item.text: capability_item = j capability_index.append(capability_item) for b in permission_list: self.click(UsmEntity().get_capability( b, capability_index[0])) if capability_item is None: logger.info(msg="capabilityName %s not found!" % i) # for a in capability_index: # for b in permission_list: # self.click(UsmEntity().get_capability(b, a)) elif section == "Users": self.ctrl_all(UsmEntity().get_input_textbox("userName")) self.type(UsmEntity().get_input_textbox("userName"), BasePage(self.driver).randomData("string", 6)) self.ctrl_all(UsmEntity().get_input_textbox("firstName")) self.type(UsmEntity().get_input_textbox("firstName"), BasePage(self.driver).randomData("string", 6)) self.ctrl_all(UsmEntity().get_input_textbox("lastName")) self.type(UsmEntity().get_input_textbox("lastName"), BasePage(self.driver).randomData("string", 6)) elif buttonName == "Inactivate": self.click(UsmEntity.confirm_delete) if buttonName == "New" or buttonName == "Edit": if index == 1: if section == "Roles": self.click(UsmEntity().get_button("Next")) list = "Users" else: list = "Roles" if flag == "add": self.type( UsmEntity.filter, self.find_element(UsmEntity().get_list_value( "available" + list + "Selected", "1")).text) Select( self.find_element(UsmEntity().get_two_list( "available" + list + "Selected"))).select_by_index(0) self.sleep(1) self.click(UsmEntity().get_button("Add")) elif flag == "remove": Select( self.find_element(UsmEntity().get_two_list( "assigned" + list + "Selected"))).select_by_index(0) self.sleep(1) self.click(UsmEntity().get_button("Remove")) else: if section == "Roles": self.click(UsmEntity().get_switch_tab("Users")) list = "Users" else: list = "Roles" if flag == "add": self.ctrl_multiSelect( UsmEntity().get_list_value( "available" + list + "Selected", "1"), UsmEntity().get_list_value( "available" + list + "Selected", "2")) self.sleep(1) self.click(UsmEntity().get_button("Add")) elif flag == "remove": self.ctrl_multiSelect( UsmEntity().get_list_value( "assigned" + list + "Selected", "1"), UsmEntity().get_list_value( "assigned" + list + "Selected", "2")) self.sleep(1) self.click(UsmEntity().get_button("Remove")) self.click(UsmEntity().get_button("Finish")) self.sleep(2) if "successfully" in CustomerRecordPage( self.driver).get_tips_msg(): return True else: return False
def doFinish(self): logger.info('Face count: %s' % self.face_count) logger.info('Frame with face: %s' % self.detected_frame_count)
def search(self, key_word, only_need_first=True, needed_pages=50): """ 搜索 :param key_word: 关键字 :param only_need_first: 只需要第一条 :param needed_pages: 需要多少页 :return: """ # Todo 不需要详情页和评论,只需要首页搜索 不需要cookie assert isinstance(key_word, str) assert key_word != None or key_word.strip() != '' if self.custom_search_url != '': key_word = self.custom_search_url logger.info('开始搜索:' + key_word) # header = self.get_header() for i in tqdm(range(1, needed_pages + 1), desc='页数'): # 针对只需要收条的情况,跳出页数循环 if only_need_first is True and i != 1: break url = 'http://www.dianping.com/search/keyword/' + str( self.location_id) + '/' + str( self.channel_id) + '_' + str(key_word) + '/p' + str(i) # 第一页不是用页码标识,会出发验证码 if i == 1: url = 'http://www.dianping.com/search/keyword/' + str( self.location_id) + '/' + str( self.channel_id) + '_' + str(key_word) # 替换url,并且过滤掉p1(p1会触发验证码) if self.custom_search_url != '': url = self.custom_search_url + str(i) if i == 1: url = self.custom_search_url[:-1] r = requests_util.get_requests(url, request_type='search') # r = requests.get(url, headers=header) text = r.text # 获取加密文件 file_map = get_search_map_file(text) # 替换加密文件 text = requests_util.replace_search_html(text, file_map) # 网页解析 html = BeautifulSoup(text, 'lxml') shop_all_list = html.select('.shop-list')[0].select('li') search_res = [] for shop in shop_all_list: try: image_path = shop.select('.pic')[0].select('a')[0].select( 'img')[0]['src'] except: image_path = '-' try: shop_id = shop.select('.txt')[0].select('.tit')[0].select( 'a')[0]['data-shopid'] except: shop_id = '-' try: detail_url = shop.select('.txt')[0].select( '.tit')[0].select('a')[0]['href'] except: detail_url = '-' try: name = shop.select('.txt')[0].select('.tit')[0].select( 'a')[0].text.strip() except: name = '-' # 两个star方式,有的页面显示详细star分数,有的显示icon # 解析icon try: star_point = \ shop.select('.txt')[0].select('.comment')[0].select('.star_icon')[0].select('span')[0]['class'][ 1].split('_')[1] star_point = float(star_point) / 10 star_point = str(star_point) except: star_point = '-' # 解析详细star try: star_point = \ shop.select('.txt')[0].select('.comment')[0].select('.star_score')[0].text star_point = float(star_point) star_point = str(star_point) except: pass try: review_number = shop.select('.txt')[0].select( '.comment')[0].select('.review-num')[0].text.replace( '\n', '') except: review_number = '-' try: mean_price = shop.select('.txt')[0].select('.comment')[ 0].select('.mean-price')[0].select('b')[0].text except: mean_price = '¥0' try: tags = shop.select('.txt')[0].select( '.tag-addr')[0].select('.tag') tag1 = tags[0].text.replace('\n', ' ').strip() tag2 = tags[1].text.replace('\n', ' ').strip() except: tag1 = '-' tag2 = '-' try: addr = shop.select('.txt')[0].select( '.tag-addr')[0].select('.addr')[0].text.replace( '\n', ' ').strip() except: addr = '-' try: recommend = shop.select('.recommend')[0].text.replace( '\n', ' ').strip() except: recommend = '-' try: commend_list = shop.select( '.comment-list')[0].text.replace('\n', ' ').strip() except: commend_list = '-' one_step_search_res = [ shop_id, name, star_point, review_number, mean_price, tag1, tag2, addr, recommend, commend_list, image_path, detail_url, '1', '1' ] # 最后两位是搜索标记 # 这个数据结构暂时没用 search_res.append(one_step_search_res) # 只要首条,跳出 if only_need_first is True: break # 解析详情页 if self.need_detail == '1': try: detail = Detail().get_detail(shop_id) print('\n' + ','.join(detail) + '\n') self.saver.save_data([detail], 'detail') except: # 设置标记 one_step_search_res[-2] = 0 logger.warning('详情信息获取失败,失败id:' + shop_id) print('\n' + ','.join(one_step_search_res) + '\n') if self.jump_wait is False: print( '检查浏览器,处理验证码,输入y程序继续运行,输入n跳过检查', 'http://www.dianping.com/shop/' + str(shop_id)) if input() == 'y': continue elif input() == 'n': self.jump_wait = True else: print('\n' + ','.join(one_step_search_res) + '\n') # 解析评论页 if self.need_comment == '1': try: review = Review().get_review(shop_id) print('获取', name, '评论', len(review), '条') self.saver.save_data(review, 'review') except: # 设置标记 one_step_search_res[-1] = 0 logger.warning('评论获取失败,失败id:' + shop_id) # 保存数据 self.saver.save_data([one_step_search_res], 'search') logger.info('解析完成:' + key_word)
# testing contexts ctx = [mx.gpu(int(i)) for i in cfg.gpus] ctx = ctx if ctx else [mx.cpu()] num_anchors = len(cfg.anchor_scales) * len(cfg.anchor_ratios) test_dataset = get_dataset(cfg.dataset, cfg.dataset_path) test_data = get_dataloader(test_dataset, cfg) cfg.dataset_size = len(test_dataset) cfg.num_classes = len(test_dataset.classes) cfg.classes = test_dataset.classes net = FasterRCNN(network=cfg.network, pretrained_base=False, batch_size=cfg.batch_size, num_classes=cfg.num_classes, scales=cfg.anchor_scales, ratios=cfg.anchor_ratios, feature_stride=cfg.feature_stride, allowed_border=cfg.allowed_border, rpn_batch_size=cfg.rpn_batch_size, rpn_fg_fraction=cfg.rpn_fg_fraction, rpn_positive_threshold=cfg.rpn_positive_threshold, rpn_negative_threshold=cfg.rpn_negative_threshold, rpn_pre_nms_top_n=cfg.rpn_test_pre_nms_top_n, rpn_post_nms_top_n=cfg.rpn_test_post_nms_top_n, rpn_nms_threshold=cfg.rpn_nms_threshold, rpn_min_size=cfg.rpn_min_size, roi_batch_size=cfg.roi_batch_size, roi_fg_fraction=cfg.roi_fg_fraction, roi_fg_threshold=cfg.roi_fg_threshold, roi_bg_threshold_hi=cfg.roi_bg_threshold_hi, roi_bg_threshold_lo=cfg.roi_bg_threshold_lo, bbox_nms_threshold=cfg.bbox_nms_threshold, bbox_nms_top_n=cfg.bbox_nms_top_n, bbox_mean=cfg.bbox_mean, bbox_std=cfg.bbox_std) net.load_parameters(cfg.model_params.strip(), ctx=ctx) map_name, mean_ap = test_faster_rcnn(net, test_data, cfg) result_msg = '\n'.join(['%s=%f' % (k, v) for k, v in zip(map_name, mean_ap)]) logger.info('[Done] Test Results: \n%s' % result_msg)
def _print(self): logger.info('Dataset name: {}'.format(self.dataset_name)) logger.info('Number of training samples: {:,}'.format( self.num_train_samples)) logger.info('Number of validation samples: {:,}'.format( self.num_val_samples))
def training(self): self.model.train() num_img_tr = len(self.train_loader) logger.info('\nTraining') max_iter = self.args.training.max_iter it = self.start_it # support multiple optimizers, but only one # optimizer is used here, i.e., names = ['match'] names = self.args.training.optimizer.keys() while it < max_iter: for samples in self.train_loader: samples = to_cuda(samples) # validation val_iter = self.args.training.get('val_iter', -1) if val_iter > 0 and it % val_iter == 0 and it >= self.args.training.get('start_eval_it', 15000): self.validation(it, 'val') self.model.train() if it % 100 == 0: logger.info('\n===> Iteration %d/%d' % (it, max_iter)) # update class weights if it >= 500 and self.args.training.get('weight_update_iter', -1) > 0 and it % self.args.training.get('weight_update_iter', -1) == 0: self.model.update_hard() logger.info('\nUpdate hard ID: %.3f'%self.model.center.ratio) self.writer.add_scalar('train/data_ratio', self.model.center.ratio, it) for name in names: losses = dict() self.optimizer[name].zero_grad() outputs = self.model(samples, type=name) losses = self.criterion(outputs, name) loss = losses['loss'] loss.backward() self.optimizer[name].step() losses.update(losses) # log training loss if it % 100 == 0: loss_log_str = '=>%s loss: %.4f'%(name, loss.item()) for loss_name in losses.keys(): if loss_name != 'loss': loss_log_str += ' %s: %.4f'%(loss_name, losses[loss_name]) self.writer.add_scalar('train/%s_iter'%loss_name, losses[loss_name], it) logger.info(loss_log_str) self.writer.add_scalar('train/total_loss_iter_%s'%name, loss.item(), it) # adjust learning rate lr_decay_iter = self.args.training.optimizer[name].get('lr_decay_iter', None) if lr_decay_iter is not None: for i in range(len(lr_decay_iter)): if it == lr_decay_iter[i]: lr = self.args.training.optimizer[name].lr * (self.args.training.optimizer[name].lr_decay ** (i+1)) logger.info('\nReduce lr to %.6f\n'%(lr)) for param_group in self.optimizer[name].param_groups: param_group["lr"] = lr break it += 1 # save model and optimizer if it % self.args.training.save_iter == 0 or it == max_iter or it == 1: logger.info('\nSaving checkpoint ......') optimizer_to_save = dict() for i in self.optimizer.keys(): optimizer_to_save[i] = self.optimizer[i].state_dict() self.saver.save_checkpoint({ 'start_it': it, 'stage': self.stage, 'state_dict': self.model.state_dict(), 'optimizer': optimizer_to_save, }, filename='ckp_%06d.pth.tar'%it) logger.info('Done.')
def train_classification(data_loader, model, criterion, optimizer, scheduler, num_epochs, device, data_type): """ - training for image classification - One epoch's training. :param data_loader: :param model: :param criterion: :param optimizer: :param scheduler: :param num_epochs: :param device: :return: """ print_freq = 200 # print training or validation status every __ batches model.train() # training mode enables dropout batch_time = AverageMeter() # forward prop. + back prop. time data_time = AverageMeter() # data loading time losses = AverageMeter() # loss start = time.time() # Batches for i, (images, labels) in enumerate(data_loader): data_time.update(time.time() - start) # Move to default device images = images.to(dtype=data_type) images = images.to(device) # (batch_size (N), 3, 300, 300) labels = labels.to(device) # Forward prop. outputs = model(images) # Loss loss = criterion(outputs, labels) # Backward prop. optimizer.zero_grad() loss.backward() # # Clip gradients, if necessary # if grad_clip is not None: # clip_gradient(optimizer, grad_clip) # Update models optimizer.step() losses.update(loss.item(), images.size(0)) batch_time.update(time.time() - start) start = time.time() # Print status if i % print_freq == 0: logger.info( 'Epoch: [{0}][{1}/{2}]\t' 'Batch Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data Time {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format( num_epochs, i, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses))
defer.addCallback(lambda ret: dev.SerializeToString() if ret == RetNo.SUCCESS else None) return defer d.addCallback(callback) return d @gatewayServiceHandle def push_ack(conn, data): try: ack = PushMessageAck() ack.ParseFromString(data) ids = json.loads(ack.ids) except Exception, e: logger.error(e) return None logger.info('ack message ids:%s, did:%s' % (ids, conn.device_id)) remote.callRemote('messages_ack', conn.device_id, ids) @routerServiceHanlde def push(did, data): connID = ConnectionMapping.get(did, None) if connID is not None: _push(connID, did, data) def _push(connID, did, data): msg = PushMessage() msg.id = data['id'] msg.sendno = data['sendno'] msg.generator = data['generator']
break except RobotError, e: logger.error("{0} robot Event:{1}".format(robot.get_local_time(), e)) finally: # 断开服务器链接 if robot.connected: # 关闭机械臂 robot.robot_shutdown() # 断开机械臂链接 robot.disconnect() print("Disconnected") # 释放库资源 WuziRobot.uninitialize() logger.info("{0} test completed.".format(WuziRobot.get_local_time())) # Stop streaming pipeline.stop() import time time.sleep(1) robot_first = True controller = Controller(True) if robot_first: controller.action(None) new_chess = controller.wait_human() while 1:
if __name__ == '__main__': # trainLabbeledPath = os.path.join(lmdb_root_path, 'train-labelled') # trainUnLabelledPath = os.path.join(lmdb_root_path, 'train-unlabelled') # testCleanPath = os.path.join(lmdb_root_path, 'test-clean') # testOtherPath = os.path.join(lmdb_root_path, 'test-other') trainAirtelPaymentsPath = os.path.join(lmdb_airtel_payments_root_path, 'train-labelled-en') testAirtelPaymentsPath = os.path.join(lmdb_airtel_payments_root_path, 'test-labelled-en') # data_train = lmdbMultiDataset(roots=[trainLabbeledPath, trainUnLabelledPath, trainAirtelPaymentsPath], transform=image_train_transform) # data_test = lmdbMultiDataset(roots=[testAirtelPaymentsPath, testCleanPath, testOtherPath], transform=image_val_transform) data_train = lmdbMultiDataset(roots=[trainAirtelPaymentsPath], transform=image_train_transform) data_test = lmdbMultiDataset(roots=[testAirtelPaymentsPath], transform=image_val_transform) logger.info(f'Number of examples in train is {len(data_train)}') logger.info(f'Number of examples in test is {len(data_test)}') for epoch in range(1): data_loader_train = torch.utils.data.DataLoader( data_train, batch_size=train_batch_size, shuffle=True, num_workers=workers, pin_memory=True, collate_fn=allign_collate) data_loader_test = torch.utils.data.DataLoader( data_test, batch_size=train_batch_size, shuffle=True, num_workers=workers, pin_memory=True,
logger.info('dev mode activated... overriding parameters:') print(' --debug_mode = True,') print(' --use_samples = True,') print(' --max_nsents = 25,') print(' --intrasentsim_samplesize = 10,') print(' --isotropycorrection_samplesize = 25') print(' --selfsim_samplesize = 1') opts.debug_mode = True opts.max_nsents = 25 #opts.use_samples=False opts.intrasentsim_samplesize = 10 opts.isotropycorrection_samplesize = 25 opts.selfsim_samplesize = 3 #opts.save_results = True #opts.huggingface_models = 'bert-base-uncased' if __name__ == '__main__': parser = opts.get_parser() opt = parser.parse_args() if opt.dev_params: update_opts_to_devmode(opt) if opt.debug_mode: logger.info('HUOM! debug mode activated... ipdb will be launched on exeption ... ') with ipdb.launch_ipdb_on_exception(): main(opt) else: main(opt)
id_bar = driver.find_element(By.CSS_SELECTOR, 'input[name="txtMRNo"]') id_bar.send_keys(patient['id']) time.sleep(300) return except Exception as e: print(f'execute {patient} failed, {e}') logger.exception(f'execute {patient} failed, {e}') return if __name__ == "__main__": patient_list = [] # read file print('start register') logger.info('start register') try: dir_path = os.path.dirname(os.path.realpath(__file__)) print('show dir path %s' % dir_path) # TODO check OS # if os = mac, encoding = utf-8 # else if windows, endcoding = utf-8-sig with open('patients.json', 'r+') as patient_file: patient_list = json.load(patient_file) print(f'patient list {patient_list}') logger.info(f'patient list {patient_list}') except Exception as e: logger.exception(f'failed to read file, {e}') print(f'failed to read file, {e}') patient_list = []
def compute_similarity_metrics(w2s, modname, embeddings): N_LAYERS, _ , HDIM = embeddings[0].shape nsents= len(embeddings) // 2 if w2s['target'] else len(embeddings) # BASELINES [FOR ANISOTROPY CORRECTION] logger.info(' computing baselines for anisotropy correction ') b1, b3 = Sim.get_baselines(embeddings, w2s, N_LAYERS) # SELF-SIMILARITY & MAXIMUM EXPLAINABLE VARIANCE logger.info(' computing self-similarity and max explainable variance ') selfsim = torch.zeros((len(w2s['source'].w2sdict), N_LAYERS)) mev = torch.zeros((len(w2s['source'].w2sdict), N_LAYERS )) for wid, occurrences in tqdm(enumerate(w2s['source'].w2sdict.values())): for layer in range(N_LAYERS): embs4thisword = torch.zeros((len(occurrences), HDIM)) for i, idx_tuple in enumerate(occurrences): sentence_id, word_id = idx_tuple embs4thisword[i,:] = embeddings[sentence_id][layer, word_id,:] selfsim[wid, layer] = Sim.self_similarity(embs4thisword).item() #- b1[layer] mev[wid, layer] = Sim.max_expl_var(embs4thisword) #- b3[layer] if w2s['target']: N_LAYERS_dec, _ , HDIM = embeddings[-1].shape selfsim_dec = torch.zeros((len(w2s['target'].w2sdict), N_LAYERS_dec)) mev_dec = torch.zeros((len(w2s['target'].w2sdict), N_LAYERS_dec )) for wid, occurrences in tqdm(enumerate(w2s['target'].w2sdict.values())): for layer in range(N_LAYERS_dec): embs4thisword = torch.zeros((len(occurrences), HDIM)) for i, idx_tuple in enumerate(occurrences): sentence_id, word_id = idx_tuple embs4thisword[i,:] = embeddings[sentence_id][layer, word_id,:] selfsim_dec[wid, layer] = Sim.self_similarity(embs4thisword).item() #- b1[layer] mev_dec[wid, layer] = Sim.max_expl_var(embs4thisword) #- b3[layer] # INTRA-SENTENCE SIMILARITY logger.info(' computing intra-sentence similarity ') insentsim = torch.zeros((nsents, N_LAYERS)) for layer in range(N_LAYERS): logger.info(f' layer: {layer}') for sid in range(nsents): insentsim[sid,layer] = Sim.intra_similarity(embeddings[sid][layer,:,:]) #- b2[layer] if w2s['target']: insentsim_dec = torch.zeros((nsents, N_LAYERS_dec)) for layer in range(N_LAYERS_dec): logger.info(f' layer: {layer+N_LAYERS}') for sid in range(nsents): insentsim_dec[sid,layer] = Sim.intra_similarity(embeddings[nsents+sid][layer,:,:]) #- b2[layer] logger.info(' intra sentence similarity finished') #b2 = insentsim.mean(dim=0) #b3bis = mev.mean(dim=0)# INTERPRETATION 2: baseline_metric3 <- this is the WRONG one metricsdict = {'selfsim':selfsim, 'selfsim_isotropic': selfsim - b1[:N_LAYERS], 'intrasentsim': insentsim, 'intrasentsim_isotropic': insentsim - b1[:N_LAYERS], 'mev': mev, 'mev_isotropic': mev - b3[:N_LAYERS], 'baseline1': b1, 'baseline3': b3 } if w2s['target']: metricsdict.update( selfsim_decoder=selfsim_dec, selfsim_decoder_isotropic=selfsim_dec - b1[-N_LAYERS_dec:], mev_decoder=mev_dec, mev_decoder_isotropic=mev_dec - b3[-N_LAYERS_dec:], insentsim_decoder=insentsim_dec, insentsim_decoder_isotropic=insentsim_dec - b1[-N_LAYERS_dec:] ) return {k:v.numpy() for k,v in metricsdict.items()}
def show_parameters(model): trainable_param_counter = defaultdict(float) logger.info('Trainable Parameters:') for name, param in model.named_parameters(): if param.requires_grad: prefix = name.split('.')[0] trainable_param_counter[prefix] += param.nelement() logger.info('{}-{}-{}-{}'.format(name, param.shape, param.dtype, param.device)) logger.info('-------------') trainable_sum = 0 for key in trainable_param_counter.keys(): logger.info('[PARAMS-COUNTING] #%s:%.2fM' % (key, trainable_param_counter[key] / 1e6)) trainable_sum += trainable_param_counter[key] logger.info('[PARAMS-SUM] #%s:%.2fM' % ('Trainable', trainable_sum / 1e6)) non_trainable_param_counter = defaultdict(float) logger.info('###########') logger.info('Non-Trainable Parameters:') for name, param in model.named_parameters(): if param.requires_grad is False: prefix = name.split('.')[0] non_trainable_param_counter[prefix] += param.nelement() logger.info('{}-{}-{}-{}'.format(name, param.shape, param.dtype, param.device)) logger.info('-------------') non_trainable_sum = 0 for key in non_trainable_param_counter.keys(): logger.info('[PARAMS-COUNTING] #%s:%.2fM' % (key, non_trainable_param_counter[key] / 1e6)) non_trainable_sum += non_trainable_param_counter[key] logger.info('[PARAMS-SUM] #%s:%.2fM' % ('Non-Trainable', non_trainable_sum / 1e6)) logger.info('-------------') logger.info('[PARAMS-SUM] #%s:%.2fM' % ('Total', (trainable_sum + non_trainable_sum) / 1e6))
def main(opt): if not opt.load_embeddings_path: samples = load_samples(opt) toks_and_embeddings_paths = Emb.compute_embeddings(opt, samples) else: toks_and_embeddings_paths = Emb.emul_compute_embeddings(opt) if opt.only_save_embs: print('Saved embeddings ... finishing job') raise SystemExit logger.info('Loading embeddings & tokenized sentences...') metrics={} for modelname, paths in toks_and_embeddings_paths.items(): tokdsents, embs = Emb.load_embeddings(paths, opt.max_nsents) logger.info(f'[*] MODEL: {modelname} ') w2s=make_indexer(opt=opt,sents=tokdsents,modname=modelname) logger.info('Computing metrics') metrics[modelname] = compute_similarity_metrics(w2s, modelname, embs) logger.info('Saving metrics') dump_similarity_metrics(opt, metrics[modelname],modelname) if opt.plot_results: Plt.makeplot(metrics,opt.outdir) else: logger.info('finishing ... to make the plots call `utils/ploting.py path/to/savedMetrics.pkl` ') logger.info(' or use option --plot_results')
def train(self): if self.conf.need_to_load: self.load(self.conf.checkpoint_dir) data = glob(os.path.join(self.conf.data, "*.%s" % self.conf.img_format)) logger.info('Total amount of images: %s' % len(data)) # np.random.shuffle(data) d_fr_optim = tf.train.AdamOptimizer(self.conf.learning_rate, beta1=self.conf.beta1) d_fr_optim = d_fr_optim.minimize(self.d_fr_loss, var_list=self.d_fr_vars) d_s_n_optim = tf.train.AdamOptimizer(self.conf.learning_rate, beta1=self.conf.beta1) d_s_n_optim = d_s_n_optim.minimize(self.d_stego_loss_total, var_list=self.d_s_n_vars) g_optim_fake = tf.train.AdamOptimizer(self.conf.learning_rate, beta1=self.conf.beta1) g_optim_fake = g_optim_fake.minimize(self.g_loss, var_list=self.g_vars) # g_optim_stego = tf.train.AdamOptimizer(0.000005, beta1=0.9) # g_optim_stego = g_optim_stego.minimize(self.g_loss_stego, var_list=self.g_vars) merged = tf.merge_all_summaries() train_writer = tf.train.SummaryWriter('./logs_sgan', self.sess.graph) tf.initialize_all_variables().run() sample_z = np.random.uniform(-1, 1, size=(self.sample_size, self.z_dim)) sample_files = data[0:self.sample_size] sample = [get_image(sample_file, self.image_size, need_transform=True) for sample_file in sample_files] sample_images = np.array(sample).astype(np.float32) counter = 1 start_time = time.time() batch_idxs = min(len(data), self.conf.train_size) / self.conf.batch_size logger.debug('Starting updating') for epoch in range(self.conf.epoch): stego_losses, fake_real_losses, generator_losses = [], [], [] logger.info('Starting epoch %s' % epoch) for idx in range(0, int(batch_idxs)): batch_files = data[idx * self.conf.batch_size:(idx + 1) * self.conf.batch_size] batch = [get_image(batch_file, self.image_size, need_transform=True) for batch_file in batch_files] batch_images = np.array(batch).astype(np.float32) batch_z = np.random.uniform(-1, 1, [self.conf.batch_size, self.z_dim]).astype(np.float32) self.sess.run(d_fr_optim, feed_dict={self.images: batch_images, self.z: batch_z}) self.sess.run(d_s_n_optim, feed_dict={self.images: batch_images, self.z: batch_z}) self.sess.run(g_optim_fake, feed_dict={self.z: batch_z}) self.sess.run(g_optim_fake, feed_dict={self.z: batch_z}) # # if epoch > 5: # self.sess.run(g_optim_stego, feed_dict={self.z: batch_z}) # errD_fake = self.d_loss_fake.eval({self.z: batch_z}) # errD_real = self.d_loss_real.eval({self.images: batch_images}) # # errD_stego = self.d_loss_stego.eval({self.z: batch_z}) # errD_n_stego = self.d_loss_nonstego.eval({self.z: batch_z}) # # errG = self.g_loss.eval({self.z: batch_z}) # # fake_real_losses.append(errD_fake + errD_stego) # stego_losses.append(errD_stego + errD_n_stego) # generator_losses.append(errG) # logger.debug("[ITERATION] Epoch [%2d], iteration [%4d/%4d] time: %4.4f" % (epoch, idx, batch_idxs, time.time() - start_time)) # logger.debug('[LOSS] Real/Fake: %.8f' % (errD_fake + errD_real)) # logger.debug('[LOSS] Stego/Non-Stego: %.8f' % (errD_stego + errD_n_stego)) # logger.debug('[LOSS] Generator: %.8f' % errG) counter += 1 if np.mod(counter, 1000) == 0: self.save(self.conf.checkpoint_dir, counter) if np.mod(counter, 300) == 0: logger.info('Save samples') samples, d_loss, g_loss = self.sess.run( [self.sampler, self.d_fr_loss, self.g_loss_fake, ], feed_dict={self.z: sample_z, self.images: sample_images} ) save_images_to_one(samples, [8, 8], './samples/train_%s_%s.png' % (epoch, idx))
def run(self, _start_time, _end_time): """ 只计算聚类 """ # 记录job开始日志 cluster_job_log = ClusterJobLog() job_id = cluster_job_log.save_auto_job(self.is_manual, self.get_group_id, self.language_type, self.save_group_id, self.week_fag) # 开始执行 # exe_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 从mysql中读取数据 corpus = [] base_data_view = BaseDataView() if self.is_manual == 2: # 涉华 corpus = base_data_view.get_involved_china_corpus(_start_time, _end_time, self.language_type, self.get_group_id) if self.is_manual == 0: corpus = base_data_view.get_corpus(_start_time, _end_time, self.language_type, self.get_group_id) corpus_length = len(corpus) if corpus_length == 0: logger.info('returning run, because len(lise_size) == 0, {start_time: %s, end_time: %s}.' % ( _start_time, _end_time)) # 更新job日志 cluster_job_log.update_job_log(job_id, 0) return # 聚类 if self.week_fag == 1: # 周数据太多, 层次聚类 O(n^3), 太慢; 更换为 dbscan聚类 cluster_results = dbscan_twice_cluster.perform_cluster(self.is_manual, self.cluster_type, self.manual_id, self.subtopic_id, self.language_type, corpus, self.save_group_id) cluster_dao = ClusterResultWeek() else: if self.language_type == 1: # 英文聚类 if self.is_manual == 2: # 涉华 min_sample = 1 else: min_sample = 2 cluster_results = hierarchical_cluster.perform_cluster(self.is_manual, self.cluster_type, self.manual_id, self.subtopic_id, self.language_type, corpus, min_sample, self.save_group_id) cluster_dao = MergerClusterResult() else: # 中文采用DBScan聚类算法 cluster_results = dbscan_twice_cluster.perform_cluster(self.is_manual, self.cluster_type, self.manual_id, self.subtopic_id, self.language_type, corpus, self.save_group_id) cluster_dao = ClusterResult() # 将结果保存到mysql中 exe_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') logger.info('time to mysql: %s; cluster_results. cluster num: %s' % (exe_time, len(cluster_results))) cluster_dao.save_cluster_results(cluster_results) # 清空上一批数据 cluster_dao.delete_cluster_results(exe_time, self.is_manual, self.cluster_type, self.manual_id, self.subtopic_id, self.save_group_id) # 更新job日志 cluster_job_log.update_job_log(job_id, len(cluster_results)) logger.info('end run.')
from datetime import datetime from utils.logger import logger from resource.token import Amil from flask import Flask from utils.cache import cache from flask_restful import Api app = Flask(__name__) api = Api(app) api.add_resource(Amil, '/amil') if __name__ == '__main__': logger.info("Starting app %s", datetime.now()) cache.init_app(app) app.run(debug=True)
def process_request(self, request): try: logger.info( '************************************************* 下面是新的一条日志 ***************************************************' ) logger.info('拦截请求的地址:%s;请求的方法:%s' % (request.path, request.method)) logger.info( '==================================== headers 头信息 ====================================================' ) for key in request.META: if key[:5] == 'HTTP_': logger.debug('%s %s' % (str(key), str(request.META[key]))) logger.info('代理IP:%s' % request.META.get('REMOTE_ADDR')) logger.info( '真实IP:%s' % request.META.get('HTTP_X_FORWARDED_FOR')) # HTTP_X_REAL_IP logger.info( '==================================== request body信息 ==================================================' ) logger.info('params参数:%s' % request.GET) if request.path == '/uploadfile/': logger.info('body参数:文件类型') else: logger.info('body参数:%s' % request.body.decode()) # if 'application/x-www-form-urlencoded' in request.META['CONTENT_TYPE']: # print('body参数:', urllib.parse.unquote(request.body.decode())) logger.info( '================================== View视图函数内部信息 ================================================' ) except Exception as e: logger.error('发生错误:已预知的是上传文件导致,非预知错误见下:') logger.error('未知错误:%s' % str(e)) return JsonResponse({ "message": "出现了无法预料的错误:%s" % e, "errorCode": 1, "data": {} })
def train_objectdetection(data_loader, model, criterion, optimizer, num_epochs, device): """ One epoch's training. :param train_loader: DataLoader for training data :param model: model :param criterion: MultiBox loss :param optimizer: optimizer :param epoch: epoch number """ print_freq = 200 # print training or validation status every __ batches model.train() # training mode enables dropout batch_time = AverageMeter() # forward prop. + back prop. time data_time = AverageMeter() # data loading time losses = AverageMeter() # loss start = time.time() # Batches for i, (images, boxes, labels, _) in enumerate(data_loader): data_time.update(time.time() - start) # Move to default device images = images.to(device) # (batch_size (N), 3, 300, 300) boxes = [b.to(device) for b in boxes] labels = [l.to(device) for l in labels] # Forward prop. predicted_locs, predicted_scores = model( images) # (N, 8732, 4), (N, 8732, n_classes) # Loss # for i in range(len(boxes)): # boxes[i] = boxes[i].to('cpu') # labels[i] = labels[i].to('cpu') # print (predicted_locs, predicted_scores) # print (predicted_locs.shape, predicted_scores.shape) # print (len(boxes), len(labels)) # print (boxes[1], labels[1]) predicted_locs = predicted_locs.to(device) predicted_scores = predicted_scores.to(device) loss = criterion(predicted_locs, predicted_scores, boxes, labels) # scalar # Backward prop. optimizer.zero_grad() loss.backward() # # Clip gradients, if necessary # if grad_clip is not None: # clip_gradient(optimizer, grad_clip) # Update model optimizer.step() losses.update(loss.item(), images.size(0)) batch_time.update(time.time() - start) start = time.time() # Print status if i % print_freq == 0: logger.info( 'Epoch: [{0}][{1}/{2}]\t' 'Batch Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data Time {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format( num_epochs, i, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses)) return losses.avg
def clean_cache(self, lib): lib_path = os.path.join(self.cache_libs_path, f'{lib}.zip') logger.info(f'Clean cache of {lib} at {lib_path}') FileUtils.remove_file(lib_path)
def messages_ack(did, msg_ids): for msg_id in msg_ids: logger.info('ack message(id:%d, did:%s)' % (msg_id, did)) msgQueue.rem_from_acking(did, msg_id)
def auto_backup_job(): for backup in BACKUP_DIR.glob("Auto*.zip"): backup.unlink() export_db(tag="Auto", template=None) logger.info("Auto Backup Called")
def go_to_av(self): self.find(self._audio_study).click() logger.info('进入视听学习页') return AVPage()
def create_enriched_data_views(db_url): engine = create_engine(db_url) logger.info("[ENRICHED DATA] Start enriched data creation") create_table_regions_departments(engine) logger.info("[ENRICHED DATA] Imported regions departments table") create_enriched_offerer_data(engine) logger.info("[ENRICHED DATA] Created enriched offerer data") create_enriched_user_data(engine) logger.info("[ENRICHED DATA] Created enriched beneficiary user data") create_enriched_stock_data(engine) logger.info("[ENRICHED DATA] Created enriched stock data") create_enriched_offer_data(engine) logger.info("[ENRICHED DATA] Created enriched offer data") create_enriched_venue_data(engine) logger.info("[ENRICHED DATA] Created enriched venue data") create_enriched_booking_data(engine) logger.info("[ENRICHED DATA] Created enriched booking data") logger.info("[ENRICHED DATA] End enriched data creation")
def book(user: User, hotel: Hotel, br: BookingRequirement) -> Booking: ''' Books the set of rooms according to the BookingRequirement If the requirement is not met, raises ''' ipdb.set_trace() logger.info('Finding available rooms') if not BookingService.meets_requirements(br): logger.error('Rooms not available matching the requirement') raise BookingRequirementNotMetException() logger.info('Finding available rooms...') available_rooms = BookingService.getAvailableRoomsInDateRange( hotel, br.from_date, br.to_date) logger.info('Found available rooms: ') logger.info(available_rooms) logger.info('Assigning rooms...') assigned_rooms = set() for room_type in [x.name for x in RoomType]: number = 0 if room_type == RoomType.STANDARD: number = br.no_standard if room_type == RoomType.DELUXE: number = br.no_deluxe if room_type == RoomType.LUXURY: number = br.no_luxury if room_type == RoomType.SUITE: number = br.no_suite picked_rooms = BookingService.pickNRoomOfType( available_rooms, room_type, number) assigned_rooms.union(picked_rooms) logger.debug('Writing rooms to RoomDate') # Adding Room to RoomDate table for _assigned_room in assigned_rooms: for _date in getDatesInRange(br.from_date, br.to_date): room_date = RoomDate(_assigned_room, _date) room_dates.append(room_date) logger.debug('Finished writing rooms to RoomDate') booking = Booking(user, hotel, assigned_rooms, br.from_date, br.to_date) bookings.append(booking) logger.info("Booking completed for user %s. Booking id: %s", user, booking.booking_id) logger.info(booking) return booking
def test_data(loader, name, epoch): max_length = 0 start = time.time() min_ratios = [] audio_size = [] for i, x in enumerate(tqdm(loader)): img, label, label_lengths, image_lengths = x max_length = max(max_length, label_lengths.max().item()) ratios = img.shape[2] / label_lengths.max().item() audio_size.append(img.shape[2]) random_index = np.random.choice(range(len(label_lengths))) sample_label_index_start = label_lengths[:random_index].sum().item() sample_label_index_end = sample_label_index_start + label_lengths[ random_index].item() sample_label = label[ sample_label_index_start:sample_label_index_end].cpu().tolist() sample_text = label_re_transform(sample_label) logger.info(f"Sample text is: {sample_text}") min_ratios.append(ratios) end = time.time() logger.info( f"Average size, min, max, median of audio is {np.mean(audio_size)}, {np.min(audio_size)}, {np.max(audio_size)}, {np.median(audio_size)}" ) logger.info(f"Longest length of text in {name} dataset is {max_length}") logger.info( f"min_ratio text in {np.min(min_ratios)} and max_ratio is {np.max(min_ratios)}, avg is {np.mean(min_ratios)}, median is {np.median(min_ratios)}" ) logger.info( f"Ratio historgram is {np.histogram(min_ratios, density=True)}") logger.info(f"Time taken for epoch {epoch} was {end-start:.3f}s")