Пример #1
0
 def process_data_batch(self, data: collections.Iterable, **kwargs):
     #print("MozFilter processing: ", data)
     account = kwargs.get("Account")
     temp = []
     try:
         if isinstance(data, collections.Iterable) and isinstance(account, SiteAccount):
             temp = [x for x in data if isinstance(x, FilteredDomainData) and TldUtility.is_top_tld(x.domain)]
             check_list = [y.domain for y in temp]
             sleep_time =random.randint(self._min_sleep_time, self._max_wait)
             time.sleep(sleep_time)
             moz = MozCom(account)
             if not self._is_throughput_debug:
                 rankings = moz.get_ranking_data_batch(check_list, limit=len(check_list))
             else:
                 rankings = [100] * len(temp)
             for i in range(len(temp)):
                 temp[i].da = rankings[i]
         else:
             raise ValueError("account is none in process_data_batch()")
     except Exception as ex:
         ErrorLogger.log_error("MozFilter", ex, "process_data_batch() " + str(data) + " account: " + account.userID)
     finally:
         PrintLogger.print("Moz processed: " + str(data) + " with: " + account.userID)
         with self._sync_lock:
             job_done = [x for x in data if x is not None]
             self._job_done += len(job_done)
             if account is not None:
                 account.Available = True
             for item in temp:
                 if isinstance(item, FilteredDomainData):
                     # print("moz processed:", item.domain)
                     if item.da >= self._min_DA_value:
                         if not self._is_throughput_debug:
                             CsvLogger.log_to_file(self._log_file, [(item.domain, item.da)]) # log this to file
                         self._output_queue.put(item)
Пример #2
0
 def process_data(self, data: FilteredDomainData, **kwargs):
     #print("MozFilter processing: ", data)
     account = kwargs.get("Account")
     try:
         if isinstance(data, FilteredDomainData) and isinstance(account, SiteAccount):
             if TldUtility.is_top_tld(data.domain):
                 sleep_time =random.randint(self._min_sleep_time, self._max_wait)
                 time.sleep(sleep_time)
                 moz = MozCom(account)
                 if not self._is_throughput_debug:
                     ranking = moz.get_ranking_data(data.domain)
                 else:
                     ranking = 100
                 data.da = ranking
             else:
                 pass
         else:
             raise ValueError("account is none in process_data")
     except Exception as ex:
         ErrorLogger.log_error("MozFilter", ex, "process_data() " + str(data) + " account: " + account.userID)
     finally:
         PrintLogger.print("Moz processed: " + str(data) + " with: " + account.userID)
         if isinstance(data, FilteredDomainData):
             with self._sync_lock:
                 self._job_done += 1
                 if account is not None:
                     account.Available = True
             if data.da >= self._min_DA_value:
                 if not self._is_throughput_debug:
                     CsvLogger.log_to_file(self._log_file, [(data.domain, data.da)]) # log this to file
                 self._output_queue.put(data)