def __init__(self, stop_event: Event, input_queue: Queue=None, output_queue: Queue=None,
max_worker: int=10, dir_path="", is_debug=False, **kwargs):
self._is_debug = is_debug
FeedbackInterface.__init__(self, **kwargs)
ExternalTempInterface.__init__(self)
# do not use predefined queue here
# self._input_q = input_queue
# self._output_q = output_queue
self._stop_event = stop_event
self._internal_stop_event = Event()
self._max_worker = max_worker
self._job_done = 0
self._job_done_shadow = 0
self._job_done_lock = RLock()
self._input_period = 0.0001 # time to sample data into the buffer
self._max_sample_results = 100000
self._min_sampling_duration = 0.0001
self._sample_batch_size = 5000
self._sample_batch_timeout = 60
if is_debug:
self._min_buff_delete_threshold = 10000 # default is 100000
else:
self._min_buff_delete_threshold = 100000
self._speed_penalty_count = 0
self._finished = False
manager, self._output_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Output)
self._db_buffer = ExternalTempDataDiskBuffer("whois_check.db", self, self._internal_stop_event, buf_size=self._max_worker*50,
terminate_callback=WhoisChecker.terminate_callback, dir_path=dir_path)
self._populate_with_state() # FeedbackInterface
if not is_debug:
log_period = 120
else:
log_period = 10
self._progress_logger = ProgressLogger(log_period, self, self._internal_stop_event)
def __init__(self,
stop_event: Event,
input_queue: Queue = None,
output_queue: Queue = None,
max_worker: int = 10,
dir_path="",
is_debug=False,
**kwargs):
self._is_debug = is_debug
FeedbackInterface.__init__(self, **kwargs)
ExternalTempInterface.__init__(self)
# do not use predefined queue here
# self._input_q = input_queue
# self._output_q = output_queue
self._stop_event = stop_event
self._internal_stop_event = Event()
self._max_worker = max_worker
self._job_done = 0
self._job_done_shadow = 0
self._job_done_lock = RLock()
self._input_period = 0.0001 # time to sample data into the buffer
self._max_sample_results = 100000
self._min_sampling_duration = 0.0001
self._sample_batch_size = 5000
self._sample_batch_timeout = 60
if is_debug:
self._min_buff_delete_threshold = 10000 # default is 100000
else:
self._min_buff_delete_threshold = 100000
self._speed_penalty_count = 0
self._finished = False
manager, self._output_q = get_queue_client(
QueueManager.MachineSettingCrawler,
QueueManager.Method_Whois_Output)
self._db_buffer = ExternalTempDataDiskBuffer(
"whois_check.db",
self,
self._internal_stop_event,
buf_size=self._max_worker * 50,
terminate_callback=WhoisChecker.terminate_callback,
dir_path=dir_path)
self._populate_with_state() # FeedbackInterface
if not is_debug:
log_period = 120
else:
log_period = 10
self._progress_logger = ProgressLogger(log_period, self,
self._internal_stop_event)
def __init__(self, db_ref: str, db_dir: str, input_queue: Queue, output_queue: Queue, stop_event: Event,
matrix: CrawlMatrix, accounts: list, force_mode=False, force_mode_offset=0, force_mode_total=0, **kwargs):
FeedbackInterface.__init__(self, **kwargs)
self._stop_event = stop_event
self._matrix = matrix
self._db_ref = db_ref
self._input_queue = input_queue
self._output_queue = output_queue
self._pool_input = Queue()
self._pool = FilterPool(self._pool_input, self._output_queue, self._queue_lock, self._stop_event, self._matrix,
accounts=accounts)
self._db_buffer = ExternalTempDataDiskBuffer(self._db_ref, self, self._stop_event, dir_path=db_dir,
buf_size=2500, output_f=5000) # control how data flow speed,
# it can keep input:output ratio = 1:1 at max 10 milion data row per hour
#FeedbackInterface.__init__(self, **kwargs)
ExternalTempInterface.__init__(self)
self._populate_with_state()
if force_mode:
new_state = _FilterState(progress=force_mode_offset, all_data=force_mode_total)
self.populate_with_state(new_state)
def __init__(self,
db_ref: str,
db_dir: str,
input_queue: Queue,
output_queue: Queue,
stop_event: Event,
matrix: CrawlMatrix,
accounts: list,
force_mode=False,
force_mode_offset=0,
force_mode_total=0,
**kwargs):
FeedbackInterface.__init__(self, **kwargs)
self._stop_event = stop_event
self._matrix = matrix
self._db_ref = db_ref
self._input_queue = input_queue
self._output_queue = output_queue
self._pool_input = Queue()
self._pool = FilterPool(self._pool_input,
self._output_queue,
self._queue_lock,
self._stop_event,
self._matrix,
accounts=accounts)
self._db_buffer = ExternalTempDataDiskBuffer(
self._db_ref,
self,
self._stop_event,
dir_path=db_dir,
buf_size=2500,
output_f=5000) # control how data flow speed,
# it can keep input:output ratio = 1:1 at max 10 milion data row per hour
#FeedbackInterface.__init__(self, **kwargs)
ExternalTempInterface.__init__(self)
self._populate_with_state()
if force_mode:
new_state = _FilterState(progress=force_mode_offset,
all_data=force_mode_total)
self.populate_with_state(new_state)