示例#1
0
    def insert_to_db(cls, database, json_data_list):
        start_time = time.time()
        if database == "mongodb":
            IpToMongoDB.get_instance().batch_update(json_data_list)
        elif database == "mysql":
            IpToMysql.get_instance().execute_many_sql_with_commit(json_data_list)

        logger.info("入库 耗时:{0}, 共插入数据{1}".format(time.time() - start_time, len(json_data_list)))
示例#2
0
 def get_conn(self):
     """
     返回collection的连接
     :return:
     """
     db_name = G_CONFIG.mongodb["db"]
     coll_name = G_CONFIG.mongodb["collection"]
     logger.info("get_conn")
     return self.conn[db_name][coll_name]
示例#3
0
 def get_initial_resp(self):
     url = "https://{host}/api/v6/initial/".format(host=G_CONFIG.host)
     payload = {"snuser": G_CONFIG.user["snuser"]}
     logger.info(payload)
     try:
         r = requests.post(url, data=json.dumps(payload))
         rjson = json.loads(r.text)
         if rjson["status"] == 200:
             result = json.loads(aes_decrypt_seg(rjson["data"]))
             if self.download_initial(url=result["file"]["link"],
                                      file_name=result["file"]["name"]):
                 # status_upload(result["file"]["name"].split(".")[0], "download")
                 return result["file"]["name"]
         else:
             logger.warn("error code is {}".format(rjson["status"]))
             return ""
     except Exception as e:
         logger.error(e)
         logger.error(traceback.format_exc())
         return ""
示例#4
0
    def deal_with_files(cls, database, files):

        try:
            start_time = time.time()
            logger.info("consume run")
            lines = cls.load_file(files)
            datas, ips = cls.init_data(lines)

            if database == "mongodb":
                logger.debug("mongo")
                cls.change_risk_tag_already_in_mongo(datas, ips)
                logger.debug("change_risk_tag_already_in_mongo done")
                values = datas.values()
                logger.debug(values)
                cls.batch_update_into_mongo(values, files)
                logger.info("入库 耗时:{0}, 共插入数据{1}".format(
                    time.time() - start_time, len(values)))

            elif database == "mysql":
                cls.change_risk_tag_already_in_mysql(datas, ips)
                values = datas.values()
                cls.batch_update_into_mysql(values, files)
                logger.info("入库 耗时:{0}, 共插入数据{1}".format(
                    time.time() - start_time, len(values)))
                pass
        except Exception as e:
            logger.error(e)
            logger.error("报错文件:{0}/n, {1}".format(files,
                                                  traceback.format_exc()))
        finally:
            for file in files:
                remove_file(file)
示例#5
0
    def deal_with_files(cls, database, files):

        try:
            start_time = time.time()
            logger.info("consume run")
            lines = cls.load_file(files)
            datas, ips = cls.init_data(lines)

            if database == "mongodb":
                logger.debug("mongo")
                values = datas.values()
                logger.debug(values)
                cls.batch_update_into_mongo(values)
                write_temp_file("upgrade_version", files[len(files) - 1])
                logger.info("入库 耗时:{0}, 共插入数据{1}".format(time.time() - start_time, len(values)))

            elif database == "mysql":
                values = datas.values()
                cls.batch_update_into_mysql(values)
                write_temp_file("upgrade_version", files[len(files) - 1])
                logger.info("入库 耗时:{0}, 共插入数据{1}".format(time.time() - start_time, len(values)))
                pass
        except Exception as e:
            logger.error(e)
            logger.error("报错文件:{0}/n, {1}".format(files, traceback.format_exc()))
        finally:
            for file in files:
                remove_file(file)
示例#6
0
    def execute_sql_find_wigh_ips(self, param_list):
        results = []
        if not param_list:
            return results
        try:
            conn = self.pool.connection()
            with conn.cursor() as cursor:
                query_sql = 'SELECT * from `{table}` where ip in (%s)' % ",".join(
                    ["%s"] * len(param_list))
                query_sql = query_sql.format(table=self.table)
                logger.info("query_sql")
                logger.info(query_sql)
                cursor.execute(query_sql, param_list)
                for row in cursor.fetchall():
                    data = {
                        "ip": row[0],
                        "type": row[1],
                        "risk_tag": row[2],
                        "risk_score": row[3],
                        "risk_level": row[4],
                        "country": row[5],
                        "province": row[6],
                        "city": row[7],
                        "district": row[8],
                        "owner": row[9],
                        "latitude": row[10],
                        "longitude": row[11],
                        "adcode": row[12],
                        "areacode": row[13],
                        "continent": row[14],
                    }
                    results.append(data)

            return results
        except Exception as e:
            logger.error(e)
            logger.error(traceback.format_exc())
            return results
示例#7
0
 def download(cls, url, file_name):
     """
     从oss下载数据重命名后保存
     :param url: oss下载链接
     :param file_name: 文件名
     :return: bool
     """
     try:
         r = requests.get(url)
         file_path = "./download/" + file_name
         with open(file_path, "wb") as code:
             code.write(r.content)
         cmd = "gunzip {file}".format(file=file_path)
         logger.info(cmd)
         subprocess.call(cmd, shell=True)
         name_upzip = file_name.split(".")[0]
         mv_cmd = "mv ./download/{file} ./task/".format(file=name_upzip)
         logger.info(mv_cmd)
         subprocess.call(mv_cmd, shell=True)
         return True
     except Exception as e:
         logger.error(e)
         logger.error(traceback.format_exc())
         return False
示例#8
0
 def __init__(self):
     logger.info("mongo init start")
     self.conn = MongoClient(G_CONFIG.mongodb["uri"])
     self.conn.get_database(G_CONFIG.mongodb["db"]).get_collection(G_CONFIG.mongodb["collection"]). \
         create_index([("ip", 1)], unique=True)
     logger.info("mongo init finish")
示例#9
0
 def get_instance(cls):
     with cls._instance_lock:  # 加锁
         if not cls._instance:
             logger.info("mongo has not instance")
             cls._instance = IpToMongoDB()
     return cls._instance