示例#1
0
class BaseSaver(object):
    # 文档字符串
    '''
    BaseSaver class allows users to save all infos data fetched from website.

    :Usage:

    '''
    # 数据存储器的静态成员定义
    SAVE_MODES = ('mongodb', 'neo4j', 'mysql')

    # 初始化方法:
    def __init__(self, save_mode="neo4j"):
        # 文档字符串
        '''
        Initialize an instance of BaseSaver.

        :Args:
         - save_mode : a str of database to save data in.

        '''
        # 方法实现
        if save_mode not in self.SAVE_MODES:
            raise RuntimeError('存储模式指定有误,请输入mongodb、neo4j或者mysql')
        self.save_mode = save_mode
        if self.save_mode == 'mongodb':
            # mongodb initialize
            print('>>>> we are in mongodb.')
            self.connector = MongoClient(
                **MONGO_CONF)[MONGO_CONF.get('authSource')]
        elif self.save_mode == 'neo4j':
            # neo4j initialize
            print('>>>> we are in neo4j.')
            self.connector = Graph(**NEO_CONF)
        else:
            # mysql initialize
            print('>>>> we are in mysql.')
            self.connector = pymysql.connect(**SQL_CONF)
            self.cursor = self.connector.cursor()
            sql = RESORT_SQL.format(table_name)
            print(sql)
            self.cursor.execute(sql)
            self.connector.commit()

    # 数据存储方法:
    def data_save(self, file_name):
        # 文档字符串
        '''
        Saves spider fetched data into different databases.
        Wipes out the old data and saves the new fetched ones.

        :Args:
         - file_name : a str of file name to fetch data from.

        '''
        # 方法实现
        # 此处可以拓展成任意文件类型,其他文件类型的数据转换成json再写即可
        file_path = os.path.join(save_path, file_name + '.json')
        if not os.access(file_path, os.F_OK):
            raise RuntimeError(f'数据文件{file_path}不存在,请检查数据!')
        with open(file_path, 'r', encoding='utf-8') as file:
            self.json_data = json.load(file, encoding='utf-8')

        if self.save_mode == 'mongodb':
            print('>>> we are saving to mongodb.')
            # 删除原始数据
            self.connector.drop_collection(collection)
            # 保存新数据
            self.connector[collection].insert_many(self.json_data)
        elif self.save_mode == 'neo4j':
            print('>>> we are saving to neo4j.')
            # 删除原始数据, 一定要小心使用
            self.graph_cleaner()
            # 保存新数据
            self.graph_builder()
        else:
            print('>>> we are saving to mysql.')
            # 删除原始数据,一定要小心使用
            self.cursor.execute(f"DELETE FROM {table_name}")
            # 准备sql语句
            data_key = self.json_data[0].keys()
            sql_key = ','.join(data_key)
            sql_value = ', '.join([f'%({key})s' for key in data_key])
            # 保存新数据
            sql = '''
            INSERT INTO {0}({1})
            VALUES ({2});
            '''.format(table_name, sql_key, sql_value)
            print(sql)
            self.cursor.executemany(sql, self.json_data)
            self.connector.commit()

    # 知识图谱删除方法:
    def graph_cleaner(self):
        pass

    # 知识图谱生成方法:
    def graph_builder(self):
        pass

    # 数据存储器退出方法:
    def __del__(self):
        # 文档字符串
        '''
        The deconstructor of BaseSaver class.

        Deconstructs an instance of BaseSaver, closes Databases.
        '''
        # 方法实现
        print(f'>>>> closing {self.save_mode}.')
        if self.save_mode == 'mongodb':
            self.connector.client.close()
        elif self.save_mode == 'mysql':
            self.connector.close()
示例#2
0
def commit(db_name: str, tx: Transaction):
    conn = Graph(URL, auth=(ADMIN_NAME, ADMIN_PASS), name=db_name)
    conn.commit(tx)