def process(self, database):
if database == "mysql":
IpToMysql.get_instance().create_table()
filename = self.get_initial_resp()
file_new = self.unzip(filename)
self.write_data(database, file_new)
write_temp_file("init_version", file_new)
def insert_to_db(cls, database, json_data_list):
start_time = time.time()
if database == "mongodb":
IpToMongoDB.get_instance().batch_update(json_data_list)
elif database == "mysql":
IpToMysql.get_instance().execute_many_sql_with_commit(json_data_list)
logger.info("入库 耗时:{0}, 共插入数据{1}".format(time.time() - start_time, len(json_data_list)))
def run(self, database):
if database == "mysql":
IpToMysql.get_instance().create_table()
path_to_task = "./task/"
while True:
files = os.listdir(path_to_task)
if files:
files.sort()
self.deal_with_files(database, files)
else:
time.sleep(10)
pass
def batch_update_into_mysql(cls, values, files):
if exec_func_times_if_error(
IpToMysql.get_instance().execute_many_sql_with_commit,
values,
times=5):
cls.batch_upload(files, "db")
else:
logger.error("尝试5次未下载成功")
cls.batch_upload(files, "db_fall")
return values
def insert_to_db(cls, database, filename, json_data_list):
if database == "mongodb":
# start_time = time.time()
if exec_func_times_if_error(
IpToMongoDB.get_instance().batch_update,
json_data_list,
times=5):
status_upload(filename, "db")
else:
status_upload(filename, "db_fail")
# logger.info("入库 耗时:{0}, 共插入数据{1}".format(time.time() - start_time, len(json_data_list)))
if database == "mysql":
# start_time = time.time()
if exec_func_times_if_error(
IpToMysql.get_instance().execute_many_sql_with_commit,
json_data_list,
times=5):
status_upload(filename, "db")
else:
status_upload(filename, "db_fail")
def change_risk_tag_already_in_mysql(cls, datas, ips):
results = IpToMysql.get_instance().execute_sql_find_wigh_ips(ips)
cls.deal_with_results(datas, results)
def batch_update_into_mysql(cls, values):
if IpToMysql.get_instance().execute_many_sql_with_commit(values):
logger.error("尝试5次未下载成功")
return values