def get_db_data(self, index=0, count=0, reverse_read=False, **kwargs) -> MiningList: return_data = MiningList("Data") if index >= 0 and count > 0: with self._db_lock: db = CategoryDomainSiteDB(self._db_addr) return_data.data = db.get_next_patch_no_rollover(index=index, count=count, reverse_read=reverse_read, **kwargs) db.close() return return_data
def get_db_data(self, index=0, count=0, reverse_read=False, **kwargs) -> MiningList: return_data = MiningList("Data") if index >= 0 and count > 0: with self._db_lock: db = CategoryDomainSiteDB(self._db_addr) return_data.data = db.get_next_patch_no_rollover( index=index, count=count, reverse_read=reverse_read, **kwargs) db.close() return return_data
def upload_seeds(ref: str, target_host: Server, seed_server: ServerRequestHandler, niches=[], init_seeds=[], seed_source_addr="", seeds_per_niche=5000): seeds = init_seeds if len(seeds) == 0: if len(seed_source_addr) > 0: raise NotImplementedError else: for niche in niches: request = SeedDBRequest(niche=niche, random_read=True, reverse_read=True, data_len=seeds_per_niche) cmd = CommandStruct(cmd=ServerCommand.Com_Get_DB_DATA, target=ServerType.ty_Seed_Database, data=request) if isinstance(seed_server, ServerRequestHandler): # case when it is a local seed db temp = seed_server.handle_request(cmd) elif len(seed_source_addr) > 0: # TODO: case when it is a remote seed db temp = None else: raise NotImplementedError if isinstance(temp, MiningList): seeds += temp.data seeds = [x for x in set(seeds)] in_data = MiningList(ref=ref, data=seeds) hostController = HostController(target_host, cmd=ServerCommand.Com_Add_Seed, in_data=in_data) hostController.start() hostController.join() return len(seeds)
def get_db_stats(self) -> MiningList: return_obj = MiningList("DATA") with self._skeleton_db_lock: data = self._skeleton_db_manager.categories return_obj.data = data return return_obj