def mongoDB_insert(self): logger.info('mongoDB insert start') # DB init HOSTNAME = '54.180.91.199' # HOSTNAME = '172.31.20.198' PORT = 27017 username = urllib.parse.quote_plus('test') password = urllib.parse.quote_plus('test12!@') connection = pymongo.MongoClient('mongodb://%s:%s@%s:%s/' % (username, password, HOSTNAME, PORT)) DB = connection['test_seibro'] DB_insert = DB['FICS_data2'] logger.success('mongoDB connection success') for i in self.df_dict: try: DB_insert.insert_one(i) logger.info(i) except: logger.info('Duplicate occurrence') pass now = datetime.now() now = str(now) self.dff.to_excel('2Digit_FICS(seibro)_Crawlling_' + now[:4] + '_' + now[5:7] + now[8:11] + '.xlsx') self.dff.to_excel('2Digit_FICS(seibro).xlsx') logger.success('mongoDB insert success') logger.complete()
def start(self): import datetime logger.add(log_config.filename + '_{time}.log', rotation=log_config.rotation, enqueue=True, encoding="utf8") logger.debug('Naver talk Crawling round start') now = datetime.datetime.now() start = time.time() # 시작 시간 저장 HOSTNAME = 'Ip' PORT = 27017 username = urllib.parse.quote_plus('ID') password = urllib.parse.quote_plus('PW') connection = pymongo.MongoClient('mongodb://%s:%s@%s:%s/' % (username, password, HOSTNAME, PORT)) DB = connection['test_talk_03'] DB_insert = DB['naver_talks'] start_count = DB_insert.count() p = Pool(8) p.map(self.talk_search,self.pool_list) p.close() time.sleep(2) end_count = DB_insert.count() work_time = time.time() - start talk_count = end_count - start_count logger.debug("%s/%s/%s %s:%s" % (now.year, now.month, now.day, now.hour, now.minute) + '/crawling count:'+str(talk_count), "/time:"+str(round(work_time / 60))) logger.debug('round end done') logger.complete() with open("output.txt", "a", encoding='utf-8') as f: print("%s/%s/%s %s:%s " % (now.year, now.month, now.day, now.hour, now.minute), 'crawling count:', talk_count, " time:", round(work_time / 60), file=f)
def mongoDB_insert(self): import pymongo import urllib logger.start('mongoDB insert start') # DB init HOSTNAME = 'IP' PORT = int(port) username = urllib.parse.quote_plus('ID') password = urllib.parse.quote_plus('PW') connection = pymongo.MongoClient('mongodb://%s:%s@%s:%s/' % (username, password, HOSTNAME, PORT)) DB = connection['test_seibro'] DB_insert = DB['FICS_data'] logger.success('mongoDB connection success') for i in self.df_dict: try: DB_insert.insert_one(i) logger.info(i) except: logger.info('Duplicate occurrence') pass now = datetime.now() now = str(now) self.dff.to_excel( '/home/2digit/tmp/seibro_FICS_datafile/2Digit_FICS(seibro)_Crawlling_' + now[:4] + '_' + now[5:7] + now[8:11] + '.xlsx') self.dff.to_excel( '/home/2digit/tmp/seibro_FICS_datafile/2Digit_FICS(seibro).xlsx') logger.success('mongoDB insert success') logger.complete()
def make_backup(folder: str) -> str: """ Сделай бэкап папки, загрузи его на Google Drive, удали локальный бэкап. :param folder: Путь к папке :return: ID загруженного на Google Drive файла """ SetLogging(log_filename=LOG_FILENAME, rotation="20 MB", backtrace=True, diagnose=True).level(LOG_LEVEL) zipfile_path = ZipMaker(folder=folder).run() google_drive_file_id = ConnGoogleDrive().upload_file(filepath=zipfile_path) os.remove(zipfile_path) log.complete() return google_drive_file_id
def test_wait_for_all_messages_enqueued(capsys): def slow_sink(message): time.sleep(0.01) sys.stderr.write(message) logger.add(slow_sink, enqueue=True, catch=False, format="{message}") for i in range(10): logger.info(i) logger.complete() out, err = capsys.readouterr() assert out == "" assert err == "".join("%d\n" % i for i in range(10))
def test_setup_logger(tmp_path: Path): log_filepath = tmp_path / "test.log" extra_log_filepath = tmp_path / "extra.log" setup_logger(INFO, log_filepath) extra_logger = create_task_logger(DEBUG, extra_log_filepath, True) logger.info("test info") extra_logger.info("extra info") logger.debug("test debug") extra_logger.complete() logger.complete() assert len(log_filepath.read_text(encoding="utf-8").splitlines()) == 1 assert len( extra_log_filepath.read_text(encoding="utf-8").splitlines()) == 1
def test_using_sink_without_running_loop_none(capsys): loop = asyncio.new_event_loop() logger.add(sys.stderr, format="=> {message}") logger.add(async_writer, format="{message}", loop=None) logger.info("A message") loop.run_until_complete(logger.complete()) out, err = capsys.readouterr() assert err == "=> A message\n" assert out == ""
def test_global_loop_not_used(capsys): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) logger.add(sys.stderr, format="=> {message}") logger.add(async_writer, format="{message}", loop=None) logger.info("A message") loop.run_until_complete(logger.complete()) out, err = capsys.readouterr() assert err == "=> A message\n" assert out == ""
def worker(barrier): barrier.wait() logger.complete()
# noinspection PyUnboundLocalVariable virus_import_parameters = known_settings.get(_fasta_target) if not virus_import_parameters: raise ValueError( f'{_fasta_target} is not recognised as an importable virus') virus_txid = virus_import_parameters["virus_taxon_id"] virus_folder = virus_import_parameters["generated_dir_name"] fasta_name = f'{_fasta_target}_{_method}_{date.today().strftime("%Y-%b-%d")}.fasta' fasta_path = generate_fasta(virus_txid, virus_folder, fasta_name, _method == 'only_new_sequences') # the following script runs pangolin and loads the result into the database db_name = get_database_config_params()["db_name"] os.system( f"bash .{sep}bash_scripts{sep}load_lineages.sh {fasta_path} {db_name}" ) elif 'overlaps' in action: from overlaps import overlaps_controller overlaps_controller.run() else: logger.error(f'the argument {action} is not recognised.\n' + wrong_arguments_message) except: logger.exception( 'FATAL ERROR' ) # this is just to make sure the exception is written to the log file before crashing sys.exit(1) finally: if 'import' in action: stats_module.check_samples_imported() logger.complete()