def mongoDB_insert(self):
logger.info('mongoDB insert start')
# DB init
HOSTNAME = '54.180.91.199'
# HOSTNAME = '172.31.20.198'
PORT = 27017
username = urllib.parse.quote_plus('test')
password = urllib.parse.quote_plus('test12!@')
connection = pymongo.MongoClient('mongodb://%s:%s@%s:%s/' % (username, password, HOSTNAME, PORT))
DB = connection['test_seibro']
DB_insert = DB['FICS_data2']
logger.success('mongoDB connection success')
for i in self.df_dict:
try:
DB_insert.insert_one(i)
logger.info(i)
except:
logger.info('Duplicate occurrence')
pass
now = datetime.now()
now = str(now)
self.dff.to_excel('2Digit_FICS(seibro)_Crawlling_' + now[:4] + '_' + now[5:7] + now[8:11] + '.xlsx')
self.dff.to_excel('2Digit_FICS(seibro).xlsx')
logger.success('mongoDB insert success')
logger.complete()
def start(self):
import datetime
logger.add(log_config.filename + '_{time}.log', rotation=log_config.rotation, enqueue=True, encoding="utf8")
logger.debug('Naver talk Crawling round start')
now = datetime.datetime.now()
start = time.time() # 시작 시간 저장
HOSTNAME = 'Ip'
PORT = 27017
username = urllib.parse.quote_plus('ID')
password = urllib.parse.quote_plus('PW')
connection = pymongo.MongoClient('mongodb://%s:%s@%s:%s/' % (username, password, HOSTNAME, PORT))
DB = connection['test_talk_03']
DB_insert = DB['naver_talks']
start_count = DB_insert.count()
p = Pool(8)
p.map(self.talk_search,self.pool_list)
p.close()
time.sleep(2)
end_count = DB_insert.count()
work_time = time.time() - start
talk_count = end_count - start_count
logger.debug("%s/%s/%s %s:%s" % (now.year, now.month, now.day, now.hour, now.minute) + '/crawling count:'+str(talk_count), "/time:"+str(round(work_time / 60)))
logger.debug('round end done')
logger.complete()
with open("output.txt", "a", encoding='utf-8') as f:
print("%s/%s/%s %s:%s " % (now.year, now.month, now.day, now.hour, now.minute), 'crawling count:', talk_count, " time:", round(work_time / 60), file=f)
def mongoDB_insert(self):
import pymongo
import urllib
logger.start('mongoDB insert start')
# DB init
HOSTNAME = 'IP'
PORT = int(port)
username = urllib.parse.quote_plus('ID')
password = urllib.parse.quote_plus('PW')
connection = pymongo.MongoClient('mongodb://%s:%s@%s:%s/' %
(username, password, HOSTNAME, PORT))
DB = connection['test_seibro']
DB_insert = DB['FICS_data']
logger.success('mongoDB connection success')
for i in self.df_dict:
try:
DB_insert.insert_one(i)
logger.info(i)
except:
logger.info('Duplicate occurrence')
pass
now = datetime.now()
now = str(now)
self.dff.to_excel(
'/home/2digit/tmp/seibro_FICS_datafile/2Digit_FICS(seibro)_Crawlling_'
+ now[:4] + '_' + now[5:7] + now[8:11] + '.xlsx')
self.dff.to_excel(
'/home/2digit/tmp/seibro_FICS_datafile/2Digit_FICS(seibro).xlsx')
logger.success('mongoDB insert success')
logger.complete()
def make_backup(folder: str) -> str:
"""
Сделай бэкап папки, загрузи его на Google Drive,
удали локальный бэкап.
:param folder: Путь к папке
:return: ID загруженного на Google Drive файла
"""
SetLogging(log_filename=LOG_FILENAME,
rotation="20 MB",
backtrace=True,
diagnose=True).level(LOG_LEVEL)
zipfile_path = ZipMaker(folder=folder).run()
google_drive_file_id = ConnGoogleDrive().upload_file(filepath=zipfile_path)
os.remove(zipfile_path)
log.complete()
return google_drive_file_id
def test_wait_for_all_messages_enqueued(capsys):
def slow_sink(message):
time.sleep(0.01)
sys.stderr.write(message)
logger.add(slow_sink, enqueue=True, catch=False, format="{message}")
for i in range(10):
logger.info(i)
logger.complete()
out, err = capsys.readouterr()
assert out == ""
assert err == "".join("%d\n" % i for i in range(10))
def test_setup_logger(tmp_path: Path):
log_filepath = tmp_path / "test.log"
extra_log_filepath = tmp_path / "extra.log"
setup_logger(INFO, log_filepath)
extra_logger = create_task_logger(DEBUG, extra_log_filepath, True)
logger.info("test info")
extra_logger.info("extra info")
logger.debug("test debug")
extra_logger.complete()
logger.complete()
assert len(log_filepath.read_text(encoding="utf-8").splitlines()) == 1
assert len(
extra_log_filepath.read_text(encoding="utf-8").splitlines()) == 1
def test_using_sink_without_running_loop_none(capsys):
loop = asyncio.new_event_loop()
logger.add(sys.stderr, format="=> {message}")
logger.add(async_writer, format="{message}", loop=None)
logger.info("A message")
loop.run_until_complete(logger.complete())
out, err = capsys.readouterr()
assert err == "=> A message\n"
assert out == ""
def test_global_loop_not_used(capsys):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
logger.add(sys.stderr, format="=> {message}")
logger.add(async_writer, format="{message}", loop=None)
logger.info("A message")
loop.run_until_complete(logger.complete())
out, err = capsys.readouterr()
assert err == "=> A message\n"
assert out == ""
def worker(barrier):
barrier.wait()
logger.complete()
# noinspection PyUnboundLocalVariable
virus_import_parameters = known_settings.get(_fasta_target)
if not virus_import_parameters:
raise ValueError(
f'{_fasta_target} is not recognised as an importable virus')
virus_txid = virus_import_parameters["virus_taxon_id"]
virus_folder = virus_import_parameters["generated_dir_name"]
fasta_name = f'{_fasta_target}_{_method}_{date.today().strftime("%Y-%b-%d")}.fasta'
fasta_path = generate_fasta(virus_txid, virus_folder, fasta_name,
_method == 'only_new_sequences')
# the following script runs pangolin and loads the result into the database
db_name = get_database_config_params()["db_name"]
os.system(
f"bash .{sep}bash_scripts{sep}load_lineages.sh {fasta_path} {db_name}"
)
elif 'overlaps' in action:
from overlaps import overlaps_controller
overlaps_controller.run()
else:
logger.error(f'the argument {action} is not recognised.\n' +
wrong_arguments_message)
except:
logger.exception(
'FATAL ERROR'
) # this is just to make sure the exception is written to the log file before crashing
sys.exit(1)
finally:
if 'import' in action:
stats_module.check_samples_imported()
logger.complete()
