def get_db_data(self, index=0, count=0, reverse_read=False, **kwargs) -> MiningList: return_data = MiningList("Data") if index >= 0 and count > 0: with self._db_lock: db = CategoryDomainSiteDB(self._db_addr) return_data.data = db.get_next_patch_no_rollover(index=index, count=count, reverse_read=reverse_read, **kwargs) db.close() return return_data
def get_db_data(self, index=0, count=0, reverse_read=False, **kwargs) -> MiningList: return_data = MiningList("Data") if index >= 0 and count > 0: with self._db_lock: db = CategoryDomainSiteDB(self._db_addr) return_data.data = db.get_next_patch_no_rollover( index=index, count=count, reverse_read=reverse_read, **kwargs) db.close() return return_data
def db_update_process(skeleton_db_addr: str="", market_db_addr: str=""): if len(skeleton_db_addr) == 0: skeleton_db_addr = market_place_skeleton_db_addr if len(market_db_addr) == 0: market_db_addr = market_place_db_addr skeleton_db_manager = CategoryDBManager(skeleton_db_addr) skeleton_db_manager.reset_category_count() len_per_patch = 20000 db = CategoryDomainSiteDB(market_db_addr) site_count = db.site_count(False) current_count = 0 while current_count < site_count: sites = db.get_next_patch_no_rollover(current_count, len_per_patch) for site in sites: if isinstance(site, CatagoryDomainSiteDataStruct): for topic in site.get_categories(): sub_category = skeleton_db_manager.get_sub_category(CategoryManager.decode_sub_category(topic, False)) sub_category.count += 1 current_count += 1 skeleton_db_manager.re_calculate_stats() skeleton_db_manager.save()
def db_update_process(skeleton_db_addr: str = "", market_db_addr: str = ""): if len(skeleton_db_addr) == 0: skeleton_db_addr = market_place_skeleton_db_addr if len(market_db_addr) == 0: market_db_addr = market_place_db_addr skeleton_db_manager = CategoryDBManager(skeleton_db_addr) skeleton_db_manager.reset_category_count() len_per_patch = 20000 db = CategoryDomainSiteDB(market_db_addr) site_count = db.site_count(False) current_count = 0 while current_count < site_count: sites = db.get_next_patch_no_rollover(current_count, len_per_patch) for site in sites: if isinstance(site, CatagoryDomainSiteDataStruct): for topic in site.get_categories(): sub_category = skeleton_db_manager.get_sub_category( CategoryManager.decode_sub_category(topic, False)) sub_category.count += 1 current_count += 1 skeleton_db_manager.re_calculate_stats() skeleton_db_manager.save()