def process_data(self, data: FilteredDomainData, **kwargs): #print("MozFilter processing: ", data) account = kwargs.get("Account") try: if isinstance(data, FilteredDomainData) and isinstance(account, SiteAccount): if TldUtility.is_top_tld(data.domain): sleep_time =random.randint(self._min_sleep_time, self._max_wait) time.sleep(sleep_time) moz = MozCom(account) if not self._is_throughput_debug: ranking = moz.get_ranking_data(data.domain) else: ranking = 100 data.da = ranking else: pass else: raise ValueError("account is none in process_data") except Exception as ex: ErrorLogger.log_error("MozFilter", ex, "process_data() " + str(data) + " account: " + account.userID) finally: PrintLogger.print("Moz processed: " + str(data) + " with: " + account.userID) if isinstance(data, FilteredDomainData): with self._sync_lock: self._job_done += 1 if account is not None: account.Available = True if data.da >= self._min_DA_value: if not self._is_throughput_debug: CsvLogger.log_to_file(self._log_file, [(data.domain, data.da)]) # log this to file self._output_queue.put(data)
def testDA_bulk(self): log_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/accounts/good_accounts.csv" bad_log_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/accounts/bad_accounts.csv" good_rows = [] bad_rows = [] data_counter = 0 domains = [] data_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/03-09-2015-Bad-Results.csv" with open(data_path, mode='r', newline='') as csv_file: rd = csv.reader(csv_file, delimiter=',') for row in rd: if data_counter > 0: domains.append(row[0]) data_counter += 1 problem_account = [] file_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/accounts/good_accounts_backup.csv" count = 0 work_count = 0 non_work_count = 0 with open(file_path, mode='r', newline='') as csv_file: reader = csv.reader(csv_file, delimiter=',') for email, psd, user_name, access_id, api_key in reader: if count not in problem_account: try: print( "email:", email, "psd:", psd, " user_name:", user_name, " access_id:", access_id, ) account = MozCom( SiteAccount(siteType=AccountType.Moz, userID=email, password=psd, AccessID=access_id, APIkey=api_key)) da = account.get_ranking_data(domains[count]) print("count: ", count, " access id:", access_id, " site:", domains[count], " da:", da) time.sleep(0.2) work_count += 1 good_rows.append((count + 1, email, psd, user_name, access_id, api_key)) except Exception as ex: bad_rows.append((count + 1, email, psd, user_name, access_id, api_key)) print(ex) non_work_count += 1 count += 1 CsvLogger.log_to_file_path(log_path, good_rows) CsvLogger.log_to_file_path(bad_log_path, bad_rows) print("total:", count, " worked:", work_count, " not-worked:", non_work_count)
def testDA(self): moz = MozCom(moz_account) da = moz.get_ranking_data("") print("da:", da)