def get_db_data(self, index=0, count=0, reverse_read=False, **kwargs) -> MiningList:
return_data = MiningList("Data")
if index >= 0 and count > 0:
with self._db_lock:
db = CategoryDomainSiteDB(self._db_addr)
return_data.data = db.get_next_patch_no_rollover(index=index, count=count, reverse_read=reverse_read, **kwargs)
db.close()
return return_data
def get_db_data(self,
index=0,
count=0,
reverse_read=False,
**kwargs) -> MiningList:
return_data = MiningList("Data")
if index >= 0 and count > 0:
with self._db_lock:
db = CategoryDomainSiteDB(self._db_addr)
return_data.data = db.get_next_patch_no_rollover(
index=index,
count=count,
reverse_read=reverse_read,
**kwargs)
db.close()
return return_data
def upload_seeds(ref: str,
target_host: Server,
seed_server: ServerRequestHandler,
niches=[],
init_seeds=[],
seed_source_addr="",
seeds_per_niche=5000):
seeds = init_seeds
if len(seeds) == 0:
if len(seed_source_addr) > 0:
raise NotImplementedError
else:
for niche in niches:
request = SeedDBRequest(niche=niche,
random_read=True,
reverse_read=True,
data_len=seeds_per_niche)
cmd = CommandStruct(cmd=ServerCommand.Com_Get_DB_DATA,
target=ServerType.ty_Seed_Database,
data=request)
if isinstance(seed_server, ServerRequestHandler):
# case when it is a local seed db
temp = seed_server.handle_request(cmd)
elif len(seed_source_addr) > 0:
# TODO: case when it is a remote seed db
temp = None
else:
raise NotImplementedError
if isinstance(temp, MiningList):
seeds += temp.data
seeds = [x for x in set(seeds)]
in_data = MiningList(ref=ref, data=seeds)
hostController = HostController(target_host,
cmd=ServerCommand.Com_Add_Seed,
in_data=in_data)
hostController.start()
hostController.join()
return len(seeds)
def get_db_stats(self) -> MiningList:
return_obj = MiningList("DATA")
with self._skeleton_db_lock:
data = self._skeleton_db_manager.categories
return_obj.data = data
return return_obj
def get_db_stats(self) -> MiningList:
return_obj = MiningList("DATA")
with self._skeleton_db_lock:
data = self._skeleton_db_manager.categories
return_obj.data = data
return return_obj
