def process_data_batch(self, data: collections.Iterable, **kwargs): #print("MozFilter processing: ", data) account = kwargs.get("Account") temp = [] try: if isinstance(data, collections.Iterable) and isinstance(account, SiteAccount): temp = [x for x in data if isinstance(x, FilteredDomainData) and TldUtility.is_top_tld(x.domain)] check_list = [y.domain for y in temp] sleep_time =random.randint(self._min_sleep_time, self._max_wait) time.sleep(sleep_time) moz = MozCom(account) if not self._is_throughput_debug: rankings = moz.get_ranking_data_batch(check_list, limit=len(check_list)) else: rankings = [100] * len(temp) for i in range(len(temp)): temp[i].da = rankings[i] else: raise ValueError("account is none in process_data_batch()") except Exception as ex: ErrorLogger.log_error("MozFilter", ex, "process_data_batch() " + str(data) + " account: " + account.userID) finally: PrintLogger.print("Moz processed: " + str(data) + " with: " + account.userID) with self._sync_lock: job_done = [x for x in data if x is not None] self._job_done += len(job_done) if account is not None: account.Available = True for item in temp: if isinstance(item, FilteredDomainData): # print("moz processed:", item.domain) if item.da >= self._min_DA_value: if not self._is_throughput_debug: CsvLogger.log_to_file(self._log_file, [(item.domain, item.da)]) # log this to file self._output_queue.put(item)
def process_data(self, data: FilteredDomainData, **kwargs): #print("MozFilter processing: ", data) account = kwargs.get("Account") try: if isinstance(data, FilteredDomainData) and isinstance(account, SiteAccount): if TldUtility.is_top_tld(data.domain): sleep_time =random.randint(self._min_sleep_time, self._max_wait) time.sleep(sleep_time) moz = MozCom(account) if not self._is_throughput_debug: ranking = moz.get_ranking_data(data.domain) else: ranking = 100 data.da = ranking else: pass else: raise ValueError("account is none in process_data") except Exception as ex: ErrorLogger.log_error("MozFilter", ex, "process_data() " + str(data) + " account: " + account.userID) finally: PrintLogger.print("Moz processed: " + str(data) + " with: " + account.userID) if isinstance(data, FilteredDomainData): with self._sync_lock: self._job_done += 1 if account is not None: account.Available = True if data.da >= self._min_DA_value: if not self._is_throughput_debug: CsvLogger.log_to_file(self._log_file, [(data.domain, data.da)]) # log this to file self._output_queue.put(data)