def insert_to_db(cls, database, json_data_list):
start_time = time.time()
if database == "mongodb":
IpToMongoDB.get_instance().batch_update(json_data_list)
elif database == "mysql":
IpToMysql.get_instance().execute_many_sql_with_commit(json_data_list)
logger.info("入库 耗时:{0}, 共插入数据{1}".format(time.time() - start_time, len(json_data_list)))
def get_conn(self):
"""
返回collection的连接
:return:
"""
db_name = G_CONFIG.mongodb["db"]
coll_name = G_CONFIG.mongodb["collection"]
logger.info("get_conn")
return self.conn[db_name][coll_name]
def get_initial_resp(self):
url = "https://{host}/api/v6/initial/".format(host=G_CONFIG.host)
payload = {"snuser": G_CONFIG.user["snuser"]}
logger.info(payload)
try:
r = requests.post(url, data=json.dumps(payload))
rjson = json.loads(r.text)
if rjson["status"] == 200:
result = json.loads(aes_decrypt_seg(rjson["data"]))
if self.download_initial(url=result["file"]["link"],
file_name=result["file"]["name"]):
# status_upload(result["file"]["name"].split(".")[0], "download")
return result["file"]["name"]
else:
logger.warn("error code is {}".format(rjson["status"]))
return ""
except Exception as e:
logger.error(e)
logger.error(traceback.format_exc())
return ""
def deal_with_files(cls, database, files):
try:
start_time = time.time()
logger.info("consume run")
lines = cls.load_file(files)
datas, ips = cls.init_data(lines)
if database == "mongodb":
logger.debug("mongo")
cls.change_risk_tag_already_in_mongo(datas, ips)
logger.debug("change_risk_tag_already_in_mongo done")
values = datas.values()
logger.debug(values)
cls.batch_update_into_mongo(values, files)
logger.info("入库 耗时:{0}, 共插入数据{1}".format(
time.time() - start_time, len(values)))
elif database == "mysql":
cls.change_risk_tag_already_in_mysql(datas, ips)
values = datas.values()
cls.batch_update_into_mysql(values, files)
logger.info("入库 耗时:{0}, 共插入数据{1}".format(
time.time() - start_time, len(values)))
pass
except Exception as e:
logger.error(e)
logger.error("报错文件:{0}/n, {1}".format(files,
traceback.format_exc()))
finally:
for file in files:
remove_file(file)
def deal_with_files(cls, database, files):
try:
start_time = time.time()
logger.info("consume run")
lines = cls.load_file(files)
datas, ips = cls.init_data(lines)
if database == "mongodb":
logger.debug("mongo")
values = datas.values()
logger.debug(values)
cls.batch_update_into_mongo(values)
write_temp_file("upgrade_version", files[len(files) - 1])
logger.info("入库 耗时:{0}, 共插入数据{1}".format(time.time() - start_time, len(values)))
elif database == "mysql":
values = datas.values()
cls.batch_update_into_mysql(values)
write_temp_file("upgrade_version", files[len(files) - 1])
logger.info("入库 耗时:{0}, 共插入数据{1}".format(time.time() - start_time, len(values)))
pass
except Exception as e:
logger.error(e)
logger.error("报错文件:{0}/n, {1}".format(files, traceback.format_exc()))
finally:
for file in files:
remove_file(file)
def execute_sql_find_wigh_ips(self, param_list):
results = []
if not param_list:
return results
try:
conn = self.pool.connection()
with conn.cursor() as cursor:
query_sql = 'SELECT * from `{table}` where ip in (%s)' % ",".join(
["%s"] * len(param_list))
query_sql = query_sql.format(table=self.table)
logger.info("query_sql")
logger.info(query_sql)
cursor.execute(query_sql, param_list)
for row in cursor.fetchall():
data = {
"ip": row[0],
"type": row[1],
"risk_tag": row[2],
"risk_score": row[3],
"risk_level": row[4],
"country": row[5],
"province": row[6],
"city": row[7],
"district": row[8],
"owner": row[9],
"latitude": row[10],
"longitude": row[11],
"adcode": row[12],
"areacode": row[13],
"continent": row[14],
}
results.append(data)
return results
except Exception as e:
logger.error(e)
logger.error(traceback.format_exc())
return results
def download(cls, url, file_name):
"""
从oss下载数据重命名后保存
:param url: oss下载链接
:param file_name: 文件名
:return: bool
"""
try:
r = requests.get(url)
file_path = "./download/" + file_name
with open(file_path, "wb") as code:
code.write(r.content)
cmd = "gunzip {file}".format(file=file_path)
logger.info(cmd)
subprocess.call(cmd, shell=True)
name_upzip = file_name.split(".")[0]
mv_cmd = "mv ./download/{file} ./task/".format(file=name_upzip)
logger.info(mv_cmd)
subprocess.call(mv_cmd, shell=True)
return True
except Exception as e:
logger.error(e)
logger.error(traceback.format_exc())
return False
def __init__(self):
logger.info("mongo init start")
self.conn = MongoClient(G_CONFIG.mongodb["uri"])
self.conn.get_database(G_CONFIG.mongodb["db"]).get_collection(G_CONFIG.mongodb["collection"]). \
create_index([("ip", 1)], unique=True)
logger.info("mongo init finish")
def get_instance(cls):
with cls._instance_lock: # 加锁
if not cls._instance:
logger.info("mongo has not instance")
cls._instance = IpToMongoDB()
return cls._instance