def save_cache(self, filename, data, mtime, expire_time=utils.CACHE_KEEP_TIME, unzip=False): if not filename or not data or mtime is None: return db = sqlite3.connect(utils.DSM_CACHE_PATH) cursor = db.cursor() if expire_time != 0: expire = expire_time + time() else: expire = 0 try: if unzip: pdata = pickle.dumps(data) else: pdata = pickle.dumps(data, pickle.HIGHEST_PROTOCOL) sql = "REPLACE INTO {} (name_hash,add_time, expire,add_data) VALUES (?,?,?,?)".format(self.table_name) cursor.execute(sql, (self.__url_md5(filename), mtime, expire, pdata)) db.commit() utils.add_log(self.logger, 'info', 'save_cache', filename, mtime, expire) except Exception as e: utils.add_log(self.logger, 'error', 'save_cache', filename, mtime, expire, e) finally: cursor.close() db.close()
def iter_results_pages(region, start_page, num_iter, user_agent): try: cookies, p_auth = request_home(user_agent=user_agent) results, total_num_pages = request_results(p_auth=p_auth, cookies=cookies, region=region, user_agent=user_agent) except Exception as e: utils.add_log(region, e, is_exception=True) return False if start_page == 1: sleep(np.random.random() * (MAX_SLEEP - MIN_SLEEP) + MIN_SLEEP) save_results(results=results, region=region, page=1) for page in range(max(2, start_page), start_page + num_iter): if page > total_num_pages: return False sleep(np.random.random() * (MAX_SLEEP - MIN_SLEEP) + MIN_SLEEP) try: results = request_results_page(page=page, cookies=cookies, user_agent=user_agent) save_results(results=results, region=region, page=page) except Exception as e: utils.add_log(region, e, is_exception=True) return True
def get_campaigns(list_id): config = frappe.get_single("MailChimp Settings") if not config.host or not config.api_key: frappe.throw( _("No configuration found. Please make sure that there is a MailChimpConnector configuration" )) if config.verify_ssl != 1: verify_ssl = False else: verify_ssl = True url = "{0}/campaigns?fields=campaigns.id,campaigns.status,campaigns.settings.title".format( config.host, list_id) raw = execute(url, config.api_key, None, verify_ssl, method="GET") results = json.loads(raw) for campaign in results['campaigns']: try: erp_campaign = frappe.get_doc("Campaign", campaign['settings']['title']) # update if applicable except: # erp does not know this campaignyet, create new_campaign = frappe.get_doc({'doctype': 'Campaign'}) new_campaign.campaign_name = campaign['settings']['title'] new_campaign.insert() add_log( title=_("Sync complete"), message=(_("Sync of campaigns from {0} completed.")).format(list_id), topic="MailChimp") return {'campaigns': results['campaigns']}
def del_expire(self): db = sqlite3.connect(utils.DSM_CACHE_PATH) cursor = db.cursor() tbs = ['spider_cache', 'dsm_cache'] try: for tb_name in tbs: cursor.execute('SELECT * FROM {}'.format(tb_name)) rss = cursor.fetchall() for rs in rss: name, ntime, expire, data, = rs if expire > 0 and time() > expire: cursor.execute('DELETE FROM {} WHERE name_hash=?'.format(tb_name), (name,)) utils.add_log(logging.getLogger('cache'), 'info', '删除过期缓存 del_expire:', tb_name, '-', name) db.commit() except Exception as e: utils.add_log(logging.getLogger('cache'), 'error', 'del_expire:', e) finally: cursor.close() db.close()
def request_url(url, type=None, params=None, headers=None, auth=None, files=None, setime=None): time = config.timeout if setime: time = setime try: data = requests.get(url, params=params, headers=headers, auth=auth, files=files, timeout=time) except Exception as error: utils.add_log( str(error) + '\nURL: ' + str(url), 'Request-Except', True) return False if data.status_code == 200: return data else: utils.add_log( 'Error in request! {}\n{}\n\n{}'.format(url, params, data.text), 'Request', True) return False
def start_thread(self, port): old_path = os.getcwd() if not os.path.exists(utils.CACHE_PATH): os.mkdir(utils.CACHE_PATH) web_dir = utils.CACHE_PATH os.chdir(web_dir) self.httpd = HTTPServer(("", port), SimpleHTTPRequestHandler) utils.add_log(self.logger, 'info', "serving at port {0}".format(port)) self.httpd.serve_forever() os.chdir(old_path)
def enqueue_get_campaigns(list_id): add_log(title=_("Starting sync"), message=(_("Starting to sync campaigns from {0}")).format(list_id), topic="MailChimp") kwargs = {'list_id': list_id} enqueue("lifefair.lifefair.mailchimp.get_campaigns", queue='long', timeout=15000, **kwargs) frappe.msgprint( _("Queued for syncing. It may take a few minutes to an hour.")) return
def crawl_region(region, start_page): utils.add_log(region, "-" * 50) should_continue = True while should_continue: should_continue = iter_results_pages( region=region, start_page=start_page, num_iter=NUM_REQUESTS_BEFORE_PAUSE, user_agent=utils.random_user_agent()) if should_continue: start_page += NUM_REQUESTS_BEFORE_PAUSE sleep(PAUSE_SLEEP) save_concatenated_excel(region)
def cron_sync_eqsl(dry_run=False): """ https://www.eqsl.cc/qslcard/ImportADIF.txt """ if dry_run: print("--- [DRY RUN] Sending logs to eQSL when requested") else: print("--- Sending logs to eQSL when requested") logs = Log.query.filter(Log.eqsl_qsl_sent == "R").all() config = Config.query.first() if not config: print("!!! Error: config not found") add_log(category="CONFIG", level="ERROR", message="Config not found") return for log in logs: status = eqsl_upload_log(log, config, dry_run) if dry_run: continue err = UserLogging() err.user_id = log.user.id err.log_id = log.id err.logbook_id = log.logbook.id err.category = "EQSL" if status["state"] == "error": err.level = "ERROR" print("!! Error uploading QSO {0} to eQSL: {1}".format( log.id, status["message"])) elif status["state"] == "rejected": log.eqsl_qsl_sent = "I" print("!! Rejected uploading QSO {0} to eQSL: {1}".format( log.id, status["message"])) else: err.level = "INFO" err.message = status["message"] + "\r\n" if "msgs" in status: for i in status["msgs"]: print("!! {0}: {1}".format(i[0], i[1])) err.message += "{0}: {1}\r\n".format(i[0], i[1]) if status["state"] == "success": log.eqsl_qsl_sent = "Y" print(status) db.session.add(err) db.session.commit()
def save_concatenated_excel(region): try: filenames = sorted(glob(os.path.join(RESULTS_DIR, region, "*")), key=lambda x: int(re.findall("\d+", x)[0])) dataset = pd.concat( (pd.read_csv(f, sep="\t", index_col=0) for f in filenames), ignore_index=True) dataset.to_excel(os.path.join(RESULTS_DIR, region + ".xls"), region, index=False, encoding="utf-8") utils.add_log(region=region, text="\nREGION DONE\nSaved %s" % os.path.join(RESULTS_DIR, region + ".xls")) except Exception as e: utils.add_log(region, e, is_exception=True)
def enqueue_sync_contacts(list_id, type="Alle", meeting=None, owner=None): add_log(title=_("Starting sync"), message=(_("Starting to sync contacts to {0}")).format(list_id), topic="MailChimp") kwargs = { 'list_id': list_id, 'type': type, 'meeting': meeting, 'owner': owner } enqueue("lifefair.lifefair.mailchimp.sync_contacts", queue='long', timeout=15000, **kwargs) frappe.msgprint( _("Queued for syncing. It may take a few minutes to an hour.")) return
def start_train(self, model, session, summary_op, summary_writer, checkpoint_prefix): for epoch in range(FLAGS.num_epochs): for batch in self.iter_batch(self.train_set, True): feed_dict = self.create_feed_dict(model, batch, True) step, summaries, loss, accuracy = model.run_step( session, True, feed_dict, summary_op) summary_writer.add_summary(summaries, step) if step % FLAGS.steps_check == 0: log_msg = 'epoch: {}/{}, step: {}, acc: {:g}, loss: {:g}'.format( epoch + 1, FLAGS.num_epochs, step, accuracy, loss) process_info(log_msg) if (epoch + 1) % FLAGS.save_epoch == 0: path = save_model(session, model, checkpoint_prefix, epoch) log_msg = 'epoch {} is over. Saved model checkpoint to {}\n'.format( epoch + 1, path) add_log(log_msg)
def post_books(): asin = request.args.get('asin') title = request.args.get('title') price = request.args.get('price') category = request.args.get('category') db = client['meta'] collection = db['newmetadata'] collection1 = db[ 'logs'] # Collection for logs -- create one in EC2 instance v = collection.find_one({"asin": asin}) try: if asin == v["asin"]: response_code = 404 add_log( request.method, request.url, { "book_information": { "title": title, "price": price, "category": category } }, response_code, collection1) return { 'message': 'Book exists. Please select another one!', 'data': {} }, 404 except: # add into metadata collection post_id = collection.insert_one(request.args.to_dict()) vv = collection.find_one({"asin": asin}) response_code = 201 # add into logs collection add_log( request.method, request.url, { "book_information": { "title": title, "price": price, "category": category } }, response_code, collection1) return {'message': 'Book added', 'data': str(vv)}, 201
def main(_): ensure_dir(FLAGS.models_dir) if FLAGS.is_train and FLAGS.clean: clean_model() init_logger(os.path.join(FLAGS.models_dir, FLAGS.log_file)) if FLAGS.preprocess: preprocess() data_manager = DataManager() out_dir = os.path.join('.', FLAGS.models_dir) checkpoint_dir = os.path.join(out_dir, FLAGS.ckpt_path) checkpoint_prefix = os.path.join(checkpoint_dir, 'model') ensure_dir(checkpoint_dir) gpu_options = tf.GPUOptions(allow_growth=True) session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement, gpu_options=gpu_options) with tf.Session(config=session_conf) as sess: wordembedding = data_manager.get_wordembedding() model = create_model(sess, Model, FLAGS.is_train, FLAGS.pretrain_model, wordembedding) model_engine = Engine(data_manager) if FLAGS.is_train: add_log('\nstart training...\n') summary_op, summary_writer = summary(out_dir, sess, model) model_engine.start_train(model, sess, summary_op, summary_writer, checkpoint_prefix) #model_engine.start_train(model, sess, summary_op, summary_writer) else: add_log('\nstart evaluating...\n') #reload_model(sess, model, checkpoint_dir) model_engine.start_test(model, sess, checkpoint_prefix)
def get_dxcc_from_clublog(callsign): config = Config.query.first() if not config: print("!!! Error: config not found") add_log(category="CONFIG", level="ERROR", message="Config not found") return clublog_api_key = config.clublog_api_key clublog_uri = "https://secure.clublog.org/dxcc?call={0}&api={1}&full=1".format( callsign, clublog_api_key) try: r = requests.get(clublog_uri) except: # noqa: E722 raise InvalidUsage("Error getting DXCC from ClubLog", status_code=500) if r.status_code != 200: raise InvalidUsage("Error getting DXCC from ClubLog", status_code=r.status_code) return json.loads(r.content)
def get_cache(self, filename, mtime=0): if not filename or mtime is None: return db = sqlite3.connect(utils.DSM_CACHE_PATH) cursor = db.cursor() try: cursor.execute('SELECT * FROM {} WHERE name_hash=?'.format(self.table_name), (self.__url_md5(filename),)) rs = cursor.fetchone() if rs: name, ntime, expire, data, = rs now = time() if not name or ntime is None or not data: return if (mtime > 0 and mtime == ntime) or (mtime == 0 and expire == 0) or (expire > 0 and now <= expire): utils.add_log(self.logger, 'info', 'get_cache:', filename, mtime, expire) return pickle.loads(data) utils.add_log(self.logger, 'info', '删除过期缓存:', filename, mtime, expire) cursor.execute('DELETE FROM {} WHERE name_hash=?'.format(self.table_name), (self.__url_md5(filename),)) return None except Exception as e: utils.add_log(self.logger, 'error', 'get_cache:', e) finally: cursor.close() db.close()
def delete_record(): db = client['meta'] collection = db['newmetadata'] collection1 = db['logs'] asin1 = request.args.get('asin') v = collection.find_one({"asin": asin1}) try: if asin1 == v["asin"]: collection.remove({"asin": asin1}) response_code = 201 add_log(request.method, request.url, {"book deleted": str(v)}, response_code, collection1) return {'message': 'Deleted metadata of book', 'data': str(v)}, 201 except: response_code = 404 add_log(request.method, request.url, {"book does not exist so not deleted": asin1}, response_code, collection1) return { 'message': 'Book does not exist so cannot delete metadata', 'data': {} }, 404
def preparation(args): # set model save path if not args.exp_id: args.exp_id = str(int(time.time())) args.current_save_path = os.path.join(args.dump_path, args.exp_name, args.exp_id) if not os.path.exists(args.current_save_path): os.makedirs(args.current_save_path, exist_ok=True) local_time = time.localtime() args.log_file = os.path.join( args.current_save_path, time.strftime("log_%Y_%m_%d_%H_%M_%S.txt", local_time)) args.output_file = os.path.join( args.current_save_path, time.strftime("output_%Y_%m_%d_%H_%M_%S.txt", local_time)) add_log(args, "exp id : %s" % args.exp_id) add_log(args, "Path: %s is created" % args.current_save_path) add_log(args, "create log file at path: %s" % args.log_file) # prepare data args.id_to_word, args.vocab_size = prepare_data(args)
def search_books(): page = request.args.get('page') if page == None: page = 2 page = int(page) db = client['meta'] collection = db['newmetadata'] collection1 = db['logs'] if (request.args.get("title") != None): mytitle = request.args.get("title") v = collection.find_one( {"title": { '$regex': mytitle, '$options': 'i' }}) if v != None: response_code = 201 add_log(request.method, request.url, { "book_information": { "title searched": request.args.get("title") } }, response_code, collection1) return dumps(v) else: response_code = 404 add_log(request.method, request.url, { "book_information": { "title searched": request.args.get("title") } }, response_code, collection1) return {'message': 'Book does not exists!', 'data': {}}, 404 if (request.args.get("author") != None): myauthor = request.args.get("author") v = collection.find({"author": { '$regex': myauthor, '$options': 'i' }}) # add .skip(page).limit(8) if v != None: response_code = 201 add_log(request.method, request.url, { "book_information": { "author searched": request.args.get("title") } }, response_code, collection1) return dumps(v) else: response_code = 404 add_log(request.method, request.url, { "book_information": { "author searched": request.args.get("title") } }, response_code, collection1) return {'message': 'Book does not exists!', 'data': {}}, 404 if (request.args.get("category") != None): v = collection.find({ "categories": { "$elemMatch": { "$elemMatch": { "$in": [request.args.get("category")] } } } }).skip(page).limit(8) if (v.count() == 0): response_code = 404 add_log( request.method, request.url, { "book_information": { "category searched": request.args.get("category") } }, response_code, collection1) return {'message': 'Book does not exists!', 'data': {}}, 404 if (v != None): response_code = 201 add_log( request.method, request.url, { "book_information": { "category searched": request.args.get("category") } }, response_code, collection1) return dumps(v) else: return {'message': "Error"}, 404
#!/usr/bin/env python3 import datetime import setup import config import utils import create_weekly_digest import logs_repo as logfile args = setup.args logfile.sync_local_logfile() if args.today: filter_today = utils.filter_by_date(config.today) utils.display_logs(filter_today, 'All logs from today: ') elif args.thisweek: filter_this_week = utils.filter_by_date(config.beginning_of_week) utils.display_logs(filter_this_week, 'All logs from this week: ') elif args.tags: filter_by_tag = utils.filter_by_tags(args.tags) utils.display_logs(filter_by_tag, f'All logs tagged with: {args.tags}') elif args.all: utils.display_logs(utils.return_all, 'All logs:') elif args.sync: logfile.sync_remote_logfile() elif args.digest: create_weekly_digest.create() else: utils.add_log() logfile.commit_local_logfile()
def sedat_eval(args, ae_model, f, deb): """ Input: Original latent representation z : (n_batch, batch_size, seq_length, latent_size) Output: An optimal modified latent representation z' """ max_sequence_length = args.max_sequence_length id_bos = args.id_bos id_to_word = args.id_to_word limit_batches = args.limit_batches eval_data_loader = non_pair_data_loader( batch_size=args.batch_size, id_bos=args.id_bos, id_eos=args.id_eos, id_unk=args.id_unk, max_sequence_length=args.max_sequence_length, vocab_size=args.vocab_size) file_list = [args.test_data_file] eval_data_loader.create_batches(args, file_list, if_shuffle=False, n_samples=args.test_n_samples) if args.references_files: gold_ans = load_human_answer(args.references_files, args.text_column) assert len(gold_ans) == eval_data_loader.num_batch else: gold_ans = None add_log(args, "Start eval process.") ae_model.eval() f.eval() deb.eval() text_z_prime = {} text_z_prime = { "source": [], "origin_labels": [], "before": [], "after": [], "change": [], "pred_label": [] } if gold_ans is not None: text_z_prime["gold_ans"] = [] z_prime = [] n_batches = 0 for it in tqdm.tqdm(list(range(eval_data_loader.num_batch)), desc="SEDAT"): _, tensor_labels, \ tensor_src, tensor_src_mask, tensor_src_attn_mask, tensor_tgt, tensor_tgt_y, \ tensor_tgt_mask, _ = eval_data_loader.next_batch() # only on negative example negative_examples = ~(tensor_labels.squeeze() == args.positive_label) tensor_labels = tensor_labels[negative_examples].squeeze( 0) # .view(1, -1) tensor_src = tensor_src[negative_examples].squeeze(0) tensor_src_mask = tensor_src_mask[negative_examples].squeeze(0) tensor_src_attn_mask = tensor_src_attn_mask[negative_examples].squeeze( 0) tensor_tgt_y = tensor_tgt_y[negative_examples].squeeze(0) tensor_tgt = tensor_tgt[negative_examples].squeeze(0) tensor_tgt_mask = tensor_tgt_mask[negative_examples].squeeze(0) if negative_examples.any(): if gold_ans is not None: text_z_prime["gold_ans"].append(gold_ans[it]) text_z_prime["source"].append( [id2text_sentence(t, args.id_to_word) for t in tensor_tgt_y]) text_z_prime["origin_labels"].append(tensor_labels.cpu().numpy()) origin_data, _ = ae_model.forward(tensor_src, tensor_tgt, tensor_src_mask, tensor_src_attn_mask, tensor_tgt_mask) generator_id = ae_model.greedy_decode(origin_data, max_len=max_sequence_length, start_id=id_bos) generator_text = [ id2text_sentence(gid, id_to_word) for gid in generator_id ] text_z_prime["before"].append(generator_text) data = deb(origin_data, mask=None) data = torch.sum(ae_model.sigmoid(data), dim=1) # (batch_size, d_model) #logit = ae_model.decode(data.unsqueeze(1), tensor_tgt, tensor_tgt_mask) # (batch_size, max_tgt_seq, d_model) #output = ae_model.generator(logit) # (batch_size, max_seq, vocab_size) y_hat = f.forward(data) y_hat = y_hat.round().int() z_prime.append(data) generator_id = ae_model.greedy_decode(data, max_len=max_sequence_length, start_id=id_bos) generator_text = [ id2text_sentence(gid, id_to_word) for gid in generator_id ] text_z_prime["after"].append(generator_text) text_z_prime["change"].append([True] * len(y_hat)) text_z_prime["pred_label"].append([y_.item() for y_ in y_hat]) n_batches += 1 if n_batches > limit_batches: break write_text_z_in_file(args, text_z_prime) add_log(args, "") add_log(args, "Saving model modify embedding %s ..." % args.current_save_path) torch.save(z_prime, os.path.join(args.current_save_path, 'z_prime_sedat.pkl')) return z_prime, text_z_prime
def add_log(self, *msg, level='info'): utils.add_log(self.logger, level, msg)
def download_page_request(self, url, retry=0, referer='', meth='GET', data=''): if not url or self.stoped: utils.add_log(self.logger, 'error', 'Url为空', url) return try: if meth != 'POST': res = self.cache.get_cache(url) if res: return res if meth == 'GET': res = self.RequestSession.get(url, timeout=utils.DOWN_TIME_OUT) elif meth == 'POST' and data: res = self.RequestSession.post(url, timeout=utils.DOWN_TIME_OUT, data=data) else: return None sleep(utils.SPIDER_DOWNLOAD_SLEEP_TIME) if res.status_code == 200: if referer == '': referer = res.url self.RequestSession.headers.update({ 'referer': referer, 'Referer': referer }) if meth != 'POST': utils.add_log( self.logger, 'info', 'save_cache expire:', url, ) if utils.IMG_CACHE_KEEP_INFINITE and utils.IsValidImage( res.content): self.cache.save_cache(url, res, expire_time=0, mtime=0) self.add_log('永久缓存', url) else: self.cache.save_cache(url, res, time()) return res else: if retry >= utils.RETRYMAX: utils.add_log(self.logger, 'info', 'Url下载出错,重试达到最大数', url) return None else: retry += 1 utils.add_log(self.logger, 'info', 'Url下载出错{},重试{}'.format(url, retry)) sleep(5) return self.download_page_request(url, retry) except (RequestException, OpenSSL.SSL.Error): if retry >= utils.RETRYMAX: utils.add_log(self.logger, 'info', 'Url下载出错,重试达到最大数', url) return None else: retry += 1 utils.add_log(self.logger, 'info', 'Url下载出错{},重试{}'.format(url, retry)) sleep(5) return self.download_page_request(url, retry)
def cron_sync_from_eqsl(dry_run=False): """ https://www.eqsl.cc/qslcard/DownloadInBox.txt """ """ todo: to avoid downloading too much (not really in fact) logs, get the oldest time with a eqsl_rcvd == 'N' and put it in query with RcvdSince = (YYYYMMDDHHMM) """ if dry_run: print("-- [DRY RUN] Fetching logs from eQSL") else: print("-- Fetching logs from eQSL") _logbooks = Logbook.query.filter( Logbook.eqsl_qth_nickname.isnot(None)).all() for logbook in _logbooks: if not logbook.user.eqsl_name or not logbook.user.eqsl_password: continue # Skip logbooks of user not using eQSL config = Config.query.first() if not config: print("!!! Error: config not found") add_log(category="CONFIG", level="ERROR", message="Config not found") return print("-- Working on logbook [{0}] {1}".format(logbook.id, logbook.name)) _payload = urllib.parse.urlencode({ "UserName": logbook.user.eqsl_name, "Password": logbook.user.eqsl_password, "QTHNickname": logbook.eqsl_qth_nickname, }) _url = "{0}?{1}".format(config.eqsl_download_url, _payload) _req = urllib.request.Request(_url) _text = None err_fetch = UserLogging() err_fetch.user_id = logbook.user.id err_fetch.logbook_id = logbook.id err_fetch.category = "EQSL FETCH" try: with urllib.request.urlopen(_req) as f: _text = f.read().decode("UTF-8") except urllib.error.URLError as e: err_fetch.level = "ERROR" err_fetch.message = "Error fetching from eQSL: {0}".format(e) db.session.add(err_fetch) db.session.commit() continue # skip to next if not _text: err_fetch.level = "ERROR" err_fetch.message = "Error fetching from EQSL, _text undefined" db.session.add(err_fetch) db.session.commit() continue # skip to next # Now get the download link # <li><a href="downloadedfiles/xxx.adi">.ADI file</a> m = re.search('<A HREF="(.*)">.ADI file</A>', _text) if m: _file_path = m.group(1) _url = "{0}/{1}".format(os.path.dirname(config.eqsl_download_url), _file_path) _req = urllib.request.Request(_url) _text = None try: print("-- Fetching ADIF {0}".format(_url)) with urllib.request.urlopen(_req) as f: # eQSL returns a file encoded in ISO8859-1 so decode it then re-encode it in UTF-8 _text = f.read().decode("ISO8859-1").encode("UTF-8") except urllib.error.URLError as e: err_fetch.level = "ERROR" err_fetch.message = "Error fetching from eQSL: {0}".format(e) db.session.add(err_fetch) db.session.commit() continue # skip to next if not _text: err_fetch.level = "ERROR" err_fetch.message = "Error fetching from EQSL, _text for final URL undefined" db.session.add(err_fetch) db.session.commit() continue # skip to next adif = parse_adif(_text) for log in adif: err_log = UserLogging() err_log.user_id = logbook.user.id err_log.logbook_id = logbook.id err_log.category = "EQSL LOG" _date = "{0} {1}".format(log["qso_date"], log["time_on"]) _date_first = datetime.datetime.strptime( _date + "00", "%Y%m%d %H%M%S") _date_second = datetime.datetime.strptime( _date + "59", "%Y%m%d %H%M%S") # Try to find a matching log entry qso = Log.query.filter( Log.logbook_id == logbook.id, Log.user_id == logbook.user.id, Log.call == log["call"].upper(), Log.time_on.between(_date_first, _date_second), ).first() if qso: if qso.eqsl_qsl_rcvd == "Y": continue # this eQSL have already been matched print( "-- Matching log found for {0} on {1} : ID {2}".format( log["call"], _date, qso.id)) if not dry_run: qso.eqsl_qsl_rcvd = "Y" err_log.log_id = qso.id err_log.level = "INFO" err_log.message = "QSO from eQSL by {0} on {1} received and updated".format( log["call"], _date) else: print("-- No matching log found for {0} on {1}".format( log["call"], _date)) err_log.level = "INFO" err_log.message = "QSO from eQSL by {0} on {1} not found in database".format( log["call"], _date) if not dry_run: db.session.add(err_log) db.session.commit() else: err_fetch.level = "ERROR" err_fetch.message = "Error fetching from EQSL, link not found in body" db.session.add(err_fetch) db.session.commit()
def sync_contacts(list_id, type, meeting=None, owner=None): # get settings config = frappe.get_single("MailChimp Settings") if not config.host or not config.api_key: frappe.throw( _("No configuration found. Please make sure that there is a MailChimp configuration" )) if config.verify_ssl != 1: verify_ssl = False else: verify_ssl = True # get the ERP contact list if type.lower() == "alle": sql_query = """SELECT `tabPerson`.`name` AS `name`, `tabPerson`.`letter_salutation` AS `letter_salutation`, `tabPerson`.`salutation` AS `salutation`, `tabPerson`.`email` AS `email`, `tabPerson`.`do_not_contact` AS `do_not_contact`, `tabPerson`.`first_name` AS `first_name`, `tabPerson`.`last_name` AS `last_name`, `tabPerson`.`primary_organisation` AS `organisation`, 'Person' AS `doctype` FROM `tabPerson` WHERE `tabPerson`.`email` LIKE '%@%.%' UNION SELECT `tabPublic Mail`.`name` AS `name`, `tabPublic Mail`.`letter_salutation` AS `letter_salutation`, '' AS `salutation`, `tabPublic Mail`.`email` AS `email`, `tabPublic Mail`.`do_not_contact` AS `do_not_contact`, '' AS `first_name`, '' AS `last_name`, `tabPublic Mail`.`organisation` AS `organisation`, 'Public Mail' AS `doctype` FROM `tabPublic Mail` WHERE `tabPublic Mail`.`email` LIKE '%@%.%'""" else: sql_query = """SELECT `tabPerson`.`name` AS `name`, `tabPerson`.`letter_salutation` AS `letter_salutation`, `tabPerson`.`salutation` AS `salutation`, `tabPerson`.`email` AS `email`, `tabPerson`.`do_not_contact` AS `do_not_contact`, `tabPerson`.`first_name` AS `first_name`, `tabPerson`.`last_name` AS `last_name`, `tabPerson`.`primary_organisation` AS `organisation`, 'Person' AS `doctype` FROM `tabPerson` WHERE `tabPerson`.`email` LIKE '%@%.%' AND `tabPerson`.`is_vip` = 0 AND !( `tabPerson`.`name` IN (SELECT `tabRegistration`.`person` FROM `tabRegistration` WHERE `tabRegistration`.`meeting` = '{meeting}')) UNION SELECT `tabPublic Mail`.`name` AS `name`, `tabPublic Mail`.`letter_salutation` AS `letter_salutation`, '' AS `salutation`, `tabPublic Mail`.`email` AS `email`, `tabPublic Mail`.`do_not_contact` AS `do_not_contact`, '' AS `first_name`, '' AS `last_name`, `tabPublic Mail`.`organisation` AS `organisation`, 'Public Mail' AS `doctype` FROM `tabPublic Mail` WHERE `tabPublic Mail`.`email` LIKE '%@%.%'""".format( meeting=meeting) erp_members = frappe.db.sql(sql_query, as_dict=True) # get all members from the MailChimp list repeat = True offset = 0 while repeat: mc_members = get_members(list_id, count=1000, offset=offset)['members'] if len(mc_members) > 0: for mc_member in mc_members: # print(mc_member['email_address']) check_mc_member_in_erp(mc_member, erp_members) offset += 1000 else: repeat = False # now get all erp members to MailChmp contact_written = [] for erp_member in erp_members: # compute mailchimp id (md5 hash of lower-case email) try: mc_id = hashlib.md5(erp_member['email'].lower()).hexdigest() except: frappe.log_error( "Invalid email address (unable to compute hash): {0}".format( erp_member['email']), "Invalid email address") # load subscription status from mailchimp if it is set as master # default is unsubscribed contact_status = "unsubscribed" if erp_member['do_not_contact'] == 1: contact_status = "unsubscribed" else: contact_status = "subscribed" url = "{0}/lists/{1}/members/{2}".format(config.host, list_id, mc_id) method = "PUT" contact_object = { "email_address": erp_member['email'], "status": contact_status, "merge_fields": { "FNAME": erp_member['first_name'] or "", "LNAME": erp_member['last_name'] or "", "TITEL": erp_member['salutation'] or "", "ANREDE": erp_member['letter_salutation'] or "" } } raw = execute(host=url, api_token=config.api_key, payload=contact_object, verify_ssl=verify_ssl, method=method) print("Updated to MailChimp {0}: {1}".format(erp_member['email'], raw)) contact_written.append(erp_member['email']) url = "{0}/lists/{1}/members?fields=members.id,members.email_address,members.status".format( config.host, list_id) raw = execute(url, config.api_key, None, verify_ssl) results = json.loads(raw) if owner: frappe.publish_realtime( event='msgprint', message=_("Synchronisation to MailChimp complete"), user=owner) add_log(title=_("Sync complete"), message=(_("Sync to {0} completed, written {1} contacts.")).format( list_id, len(contact_written)), topic="MailChimp") return {'members': results['members']}
def update_dxcc_from_cty_xml(_file=None, silent=False): if not silent: print( "--- Updating DXCC tables (prefixes, entities, exceptions) from cty.xml" ) fname = os.path.join(current_app.config["TEMP_DOWNLOAD_FOLDER"], "cty.xml") config = Config.query.first() if not config: if not silent: print("!!! Error: config not found") add_log(category="CONFIG", level="ERROR", message="Config not found") raise Exception("config not found") if os.path.isfile(fname): os.remove(fname) if not silent: print("-- Removed old file {0}".format(fname)) if not _file: if not silent: print("-- Downloading...") if not config.clublog_api_key: if not silent: print("!! Clublog API Key not defined") add_log(category="CRONS", level="ERROR", message="Clublog API Key not defined") raise Exception("no clublog_api_key in config") url = "https://secure.clublog.org/cty.php?api={0}".format( config.clublog_api_key) try: with urllib.request.urlopen(url) as response, open( fname, "wb") as out_file: with gzip.GzipFile(fileobj=response) as uncompressed: shutil.copyfileobj(uncompressed, out_file) except urllib.error.URLError as err: if not silent: print("!! Error: {0}".format(err)) raise Exception(f"error: {err}") if not silent: print("-- File downloaded at {0}".format(fname)) elif os.path.isfile(_file): fname = _file if not silent: print("-- File at {0}".format(fname)) else: if not silent: print("-- what are you trying to do ?") raise Exception("unknown error") # Now parse XML file tree = None try: tree = ElementTree.parse(fname) except FileNotFoundError as err: if not silent: print("!! Error: {0}".format(err)) raise Exception(f"file not found: {err}") except ElementTree.ParseError as err: if not silent: print("!! Error: {0}".format(err)) raise Exception(f"XML Parsing error: {err}") if not tree: raise Exception("XML tree is none") root = tree.getroot() for element in root: if element.tag == "{http://www.clublog.org/cty/v1.0}entities": if not silent: print("++ Parsing {0}".format(element.tag)) rmed = DxccEntities.query.delete() if not silent: print("-- Cleaned {0} old entries".format(rmed)) parse_element(element, silent) elif element.tag == "{http://www.clublog.org/cty/v1.0}exceptions": if not silent: print("++ Parsing {0}".format(element.tag)) rmed = DxccExceptions.query.delete() if not silent: print("-- Cleaned {0} old entries".format(rmed)) parse_element(element, silent) elif element.tag == "{http://www.clublog.org/cty/v1.0}prefixes": if not silent: print("++ Parsing {0}".format(element.tag)) rmed = DxccPrefixes.query.delete() if not silent: print("-- Cleaned {0} old entries".format(rmed)) parse_element(element, silent)
def start_test(self, model, session, checkpoint_prefix): epoch_range = eval(FLAGS.epoch_range) epoch_range = range(epoch_range[0], epoch_range[1]) save_x = None save_y = None best_auc = 0 best_epoch = -1 for epoch in epoch_range: if not os.path.exists('{}-{}.index'.format(checkpoint_prefix, epoch)): continue add_log('start testing model-{}'.format(epoch)) reload_model(session, model, checkpoint_prefix, epoch) test_result = [] total_recall = 0 for i, batch in enumerate(self.iter_batch(self.test_set, False)): feed_dict = self.create_feed_dict(model, batch, False) test_output = model.run_step(session, False, feed_dict)[0] for j in range(len(test_output)): pred = test_output[j] entity_pair = self.test_set.instance_entity[ j + batch.start_index] for rel in range(1, len(pred)): flag = int(((entity_pair[0], entity_pair[1], rel) in self.test_set.instance_triple)) total_recall += flag test_result.append([(entity_pair[0], entity_pair[1], rel), flag, pred[rel]]) if i % 100 == 0: process_info('predicting {} / {}\r'.format( i, batch.num_batches)) add_log('\nevaluating...') sorted_test_result = sorted(test_result, key=lambda x: x[2]) # Reference url: https://blog.csdn.net/zk_j1994/article/details/78478502 pr_result_x = [] # recall pr_result_y = [] # precision correct = 0 for i, item in enumerate(sorted_test_result[::-1]): if item[1] == 1: # flag == 1 correct += 1 pr_result_y.append(float(correct) / (i + 1)) pr_result_x.append(float(correct) / total_recall) auc = sklearn.metrics.auc(x=pr_result_x, y=pr_result_y) prec_mean = (pr_result_y[100] + pr_result_y[200] + pr_result_y[300]) / 3 add_log( 'auc: {:g}\np@100: {:g}\np@200: {:g}\np@300: {:g}\np@(100,200,300) mean: {:g}\n' .format(auc, pr_result_y[100], pr_result_y[200], pr_result_y[300], prec_mean)) if auc > best_auc: best_auc = auc best_epoch = epoch save_x = pr_result_x save_y = pr_result_y ensure_dir(os.path.join(FLAGS.models_dir, FLAGS.res_path)) new_res_path = os.path.join(FLAGS.models_dir, FLAGS.res_path) np.save(os.path.join(new_res_path, 'model_x.npy'), save_x) np.save(os.path.join(new_res_path, 'model_y.npy'), save_y) add_log('best_auc: {:g}; best_epoch: {}'.format(best_auc, best_epoch))
def main(args): model = utils.get_arch(arch=args.arch, dataset=args.dataset) if args.optim == 'sgd': optim = torch.optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, momentum=args.momentum) elif args.optim == 'adam': optim = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) criterion = torch.nn.CrossEntropyLoss() train_loader = utils.get_loader(args.dataset, args.batch_size, train=True, training=True) attacker = utils.PGDAttacker( radius=args.pgd_radius, steps=args.pgd_steps, step_size=args.pgd_step_size, random_start=args.pgd_random_start, norm_type=args.pgd_norm_type, ) log = dict() if not args.cpu: model.cuda() criterion = criterion.cuda() if args.resume: # raise NotImplementedError state_dict = torch.load('{}-model.pkl'.format(args.resume_path)) model.load_state_dict(state_dict['model_state_dict']) optim.load_state_dict(state_dict['optim_state_dict']) with open('{}-log.pkl'.format(args.resume_path), 'rb') as f: log = pickle.load(f) # x, y = next(train_loader) # if not args.cpu: x, y = x.cuda(), y.cuda() # adv_x = attacker.perturb(model, criterion, x, y) # max_grad_norm, grad_norm_list = get_grad_norm(model,criterion,adv_x,y) # utils.add_log(log, 'max_grad_norm', max_grad_norm) # utils.add_log(log, 'grad_norm_list', grad_norm_list) # logger.info('step [{}/{}]: max_grad_norm {:.3e}' # .format(0, args.train_steps, max_grad_norm)) # logger.info('') if not args.resume: ''' save the initial model parameter ''' save_checkpoint('ckpt-{}'.format(0), '{}/ckpts/'.format(args.save_dir), model, optim, log) start_step = args.resume_train_step if args.resume else 0 for step in range(start_step, args.train_steps, 1): lr = args.lr * (args.lr_decay_rate**(step // args.lr_decay_freq)) for group in optim.param_groups: group['lr'] = lr x, y = next(train_loader) if not args.cpu: x, y = x.cuda(), y.cuda() adv_x = attacker.perturb(model, criterion, x, y) if (step + 1) % args.calc_mg_freq == 0: max_grad_norm, grad_norm_list = get_grad_norm( model, criterion, adv_x, y) utils.add_log(log, 'max_grad_norm', max_grad_norm) utils.add_log(log, 'grad_norm_list', grad_norm_list) logger.info('step [{}/{}]: max_grad_norm {:.3e}'.format( step + 1, args.train_steps, max_grad_norm)) logger.info('') with torch.no_grad(): model.eval() _y = model(x) nat_loss = criterion(_y, y) nat_acc = (_y.argmax(dim=1) == y).sum().item() / len(y) utils.add_log(log, 'nat_loss', nat_loss.item()) utils.add_log(log, 'nat_acc', nat_acc) # ''' ERM begin ''' # model.train() # _y = model(x) # nat_loss = criterion(_y, y) # nat_acc = (_y.argmax(dim=1) == y).sum().item() / len(y) # utils.add_log(log, 'nat_loss', nat_loss.item()) # utils.add_log(log, 'nat_acc', nat_acc) # model.zero_grad() # nat_loss.backward() # nat_grad_norm = 0 # for pp in model.parameters(): # nat_grad_norm += (pp.grad.data**2).sum().item() # nat_grad_norm = np.sqrt(nat_grad_norm) # utils.add_log(log, 'nat_grad_norm', nat_grad_norm) # ''' ERM end ''' ''' adv begin (includes gradient descent) ''' model.train() _y = model(adv_x) adv_loss = criterion(_y, y) adv_acc = (_y.argmax(dim=1) == y).sum().item() / len(y) utils.add_log(log, 'adv_loss', adv_loss.item()) utils.add_log(log, 'adv_acc', adv_acc) optim.zero_grad() adv_loss.backward() optim.step() adv_grad_norm = 0 for pp in model.parameters(): adv_grad_norm += (pp.grad.data**2).sum().item() adv_grad_norm = np.sqrt(adv_grad_norm) utils.add_log(log, 'adv_grad_norm', adv_grad_norm) ''' adv end ''' # xjb_rate = batch_grad_norm / old_batch_grad_norm # logger.info('RI??? {:.3e}'.format(xjb_rate)) if (step + 1) % args.report_freq == 0: logger.info('step [{}/{}]:'.format(step + 1, args.train_steps)) logger.info('nat_acc {:.2%} \t nat_loss {:.3e}'.format( nat_acc, nat_loss.item())) logger.info('adv_acc {:.2%} \t adv_loss {:.3e}'.format( adv_acc, adv_loss.item())) # logger.info('nat_grad_norm {:.3e} \t adv_grad_norm {:.3e} \t rate {:.3e}' # .format( nat_grad_norm, adv_grad_norm, adv_grad_norm/nat_grad_norm )) logger.info('') if (step+1) % args.save_freq == 0 \ or (step+1) == args.train_steps: save_checkpoint('ckpt-{}'.format(step + 1), '{}/ckpts/'.format(args.save_dir), model, optim, log)
def save_results(results, region, page): filename = os.path.join(RESULTS_DIR, region, "page_%d.txt" % page) results.to_csv(filename, sep="\t") utils.add_log(region, "Page %d done" % page)
def train_step(args, data_loader, ae_model, dis_model, ae_optimizer, dis_optimizer, ae_criterion, dis_criterion, epoch): ae_model.train() dis_model.train() loss_ae, n_words_ae, xe_loss_ae, n_valid_ae = 0, 0, 0, 0 loss_clf, total_clf, n_valid_clf = 0, 0, 0 epoch_start_time = time.time() for it in range(data_loader.num_batch): flag_rec = True batch_sentences, tensor_labels, \ tensor_src, tensor_src_mask, tensor_src_attn_mask, tensor_tgt, tensor_tgt_y, \ tensor_tgt_mask, tensor_ntokens = data_loader.next_batch() # Forward pass latent, out = ae_model.forward(tensor_src, tensor_tgt, tensor_src_mask, tensor_src_attn_mask, tensor_tgt_mask) # Loss calculation if not args.sedat: loss_rec = ae_criterion(out.contiguous().view(-1, out.size(-1)), tensor_tgt_y.contiguous().view(-1)) / ( tensor_ntokens.data + eps) else: # only on positive example positive_examples = tensor_labels.squeeze() == args.positive_label out = out[positive_examples] # or out[positive_examples,:,:] tensor_tgt_y = tensor_tgt_y[ positive_examples] # or tensor_tgt_y[positive_examples,:] tensor_ntokens = (tensor_tgt_y != 0).data.sum().float() loss_rec = ae_criterion(out.contiguous().view(-1, out.size(-1)), tensor_tgt_y.contiguous().view(-1)) / ( tensor_ntokens.data + eps) flag_rec = positive_examples.any() out = out.squeeze(0) tensor_tgt_y = tensor_tgt_y.squeeze(0) if flag_rec: n_v, n_w = get_n_v_w(tensor_tgt_y, out) else: n_w = float("nan") n_v = float("nan") x_e = loss_rec.item() * n_w loss_ae += loss_rec.item() n_words_ae += n_w xe_loss_ae += x_e n_valid_ae += n_v ae_acc = 100. * n_v / (n_w + eps) avg_ae_acc = 100. * n_valid_ae / (n_words_ae + eps) avg_ae_loss = loss_ae / (it + 1) ae_ppl = np.exp(x_e / (n_w + eps)) avg_ae_ppl = np.exp(xe_loss_ae / (n_words_ae + eps)) ae_optimizer.zero_grad() loss_rec.backward(retain_graph=not args.detach_classif) ae_optimizer.step() # Classifier if args.detach_classif: dis_lop = dis_model.forward(to_var(latent.clone())) else: dis_lop = dis_model.forward(latent) loss_dis = dis_criterion(dis_lop, tensor_labels) dis_optimizer.zero_grad() loss_dis.backward() dis_optimizer.step() t_c = tensor_labels.view(-1).size(0) n_v = (dis_lop.round().int() == tensor_labels).sum().item() loss_clf += loss_dis.item() total_clf += t_c n_valid_clf += n_v clf_acc = 100. * n_v / (t_c + eps) avg_clf_acc = 100. * n_valid_clf / (total_clf + eps) avg_clf_loss = loss_clf / (it + 1) if it % args.log_interval == 0: add_log( args, 'epoch {:3d} | {:5d}/{:5d} batches |'.format( epoch, it, data_loader.num_batch)) add_log( args, 'Train : rec acc {:5.4f} | rec loss {:5.4f} | ppl {:5.4f} | dis acc {:5.4f} | dis loss {:5.4f} |' .format(ae_acc, loss_rec.item(), ae_ppl, clf_acc, loss_dis.item())) add_log( args, 'Train, avg : rec acc {:5.4f} | rec loss {:5.4f} | ppl {:5.4f} | dis acc {:5.4f} | dis loss {:5.4f} |' .format(avg_ae_acc, avg_ae_loss, avg_ae_ppl, avg_clf_acc, avg_clf_loss)) if flag_rec: i = random.randint(0, len(tensor_tgt_y) - 1) reference = id2text_sentence(tensor_tgt_y[i], args.id_to_word) add_log(args, "input : %s" % reference) generator_text = ae_model.greedy_decode( latent, max_len=args.max_sequence_length, start_id=args.id_bos) # batch_sentences hypothesis = id2text_sentence(generator_text[i], args.id_to_word) add_log(args, "gen : %s" % hypothesis) add_log( args, "bleu : %s" % calc_bleu(reference.split(" "), hypothesis.split(" "))) s = {} L = data_loader.num_batch + eps s["train_ae_loss"] = loss_ae / L s["train_ae_acc"] = 100. * n_valid_ae / (n_words_ae + eps) s["train_ae_ppl"] = np.exp(xe_loss_ae / (n_words_ae + eps)) s["train_clf_loss"] = loss_clf / L s["train_clf_acc"] = 100. * n_valid_clf / (total_clf + eps) add_log( args, '| end of epoch {:3d} | time: {:5.2f}s |'.format( epoch, (time.time() - epoch_start_time))) add_log( args, '| rec acc {:5.4f} | rec loss {:5.4f} | rec ppl {:5.4f} | dis acc {:5.4f} | dis loss {:5.4f} |' .format(s["train_ae_acc"], s["train_ae_loss"], s["train_ae_ppl"], s["train_clf_acc"], s["train_clf_loss"])) return s