def process_data_batch(self, data: collections.Iterable, **kwargs):
#print("MozFilter processing: ", data)
account = kwargs.get("Account")
temp = []
try:
if isinstance(data, collections.Iterable) and isinstance(account, SiteAccount):
temp = [x for x in data if isinstance(x, FilteredDomainData) and TldUtility.is_top_tld(x.domain)]
check_list = [y.domain for y in temp]
sleep_time =random.randint(self._min_sleep_time, self._max_wait)
time.sleep(sleep_time)
moz = MozCom(account)
if not self._is_throughput_debug:
rankings = moz.get_ranking_data_batch(check_list, limit=len(check_list))
else:
rankings = [100] * len(temp)
for i in range(len(temp)):
temp[i].da = rankings[i]
else:
raise ValueError("account is none in process_data_batch()")
except Exception as ex:
ErrorLogger.log_error("MozFilter", ex, "process_data_batch() " + str(data) + " account: " + account.userID)
finally:
PrintLogger.print("Moz processed: " + str(data) + " with: " + account.userID)
with self._sync_lock:
job_done = [x for x in data if x is not None]
self._job_done += len(job_done)
if account is not None:
account.Available = True
for item in temp:
if isinstance(item, FilteredDomainData):
# print("moz processed:", item.domain)
if item.da >= self._min_DA_value:
if not self._is_throughput_debug:
CsvLogger.log_to_file(self._log_file, [(item.domain, item.da)]) # log this to file
self._output_queue.put(item)
def process_data(self, data: FilteredDomainData, **kwargs):
#print("MozFilter processing: ", data)
account = kwargs.get("Account")
try:
if isinstance(data, FilteredDomainData) and isinstance(account, SiteAccount):
if TldUtility.is_top_tld(data.domain):
sleep_time =random.randint(self._min_sleep_time, self._max_wait)
time.sleep(sleep_time)
moz = MozCom(account)
if not self._is_throughput_debug:
ranking = moz.get_ranking_data(data.domain)
else:
ranking = 100
data.da = ranking
else:
pass
else:
raise ValueError("account is none in process_data")
except Exception as ex:
ErrorLogger.log_error("MozFilter", ex, "process_data() " + str(data) + " account: " + account.userID)
finally:
PrintLogger.print("Moz processed: " + str(data) + " with: " + account.userID)
if isinstance(data, FilteredDomainData):
with self._sync_lock:
self._job_done += 1
if account is not None:
account.Available = True
if data.da >= self._min_DA_value:
if not self._is_throughput_debug:
CsvLogger.log_to_file(self._log_file, [(data.domain, data.da)]) # log this to file
self._output_queue.put(data)
def testDA_bulk(self):
log_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/accounts/good_accounts.csv"
bad_log_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/accounts/bad_accounts.csv"
good_rows = []
bad_rows = []
data_counter = 0
domains = []
data_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/03-09-2015-Bad-Results.csv"
with open(data_path, mode='r', newline='') as csv_file:
rd = csv.reader(csv_file, delimiter=',')
for row in rd:
if data_counter > 0:
domains.append(row[0])
data_counter += 1
problem_account = []
file_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/accounts/good_accounts_backup.csv"
count = 0
work_count = 0
non_work_count = 0
with open(file_path, mode='r', newline='') as csv_file:
reader = csv.reader(csv_file, delimiter=',')
for email, psd, user_name, access_id, api_key in reader:
if count not in problem_account:
try:
print(
"email:",
email,
"psd:",
psd,
" user_name:",
user_name,
" access_id:",
access_id,
)
account = MozCom(
SiteAccount(siteType=AccountType.Moz,
userID=email,
password=psd,
AccessID=access_id,
APIkey=api_key))
da = account.get_ranking_data(domains[count])
print("count: ", count, " access id:", access_id,
" site:", domains[count], " da:", da)
time.sleep(0.2)
work_count += 1
good_rows.append((count + 1, email, psd, user_name,
access_id, api_key))
except Exception as ex:
bad_rows.append((count + 1, email, psd, user_name,
access_id, api_key))
print(ex)
non_work_count += 1
count += 1
CsvLogger.log_to_file_path(log_path, good_rows)
CsvLogger.log_to_file_path(bad_log_path, bad_rows)
print("total:", count, " worked:", work_count, " not-worked:",
non_work_count)
def testDA_patch(self):
data_counter = 0
domains = []
limit = 50
data_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/17-09-2015-Bad-Results.csv"
with open(data_path, mode='r', newline='') as csv_file:
rd = csv.reader(csv_file, delimiter=',')
for row in rd:
if data_counter > 0:
domains.append(row[0])
data_counter += 1
if data_counter >= limit + 1:
break
moz = MozCom(moz_account)
data = moz.get_ranking_data_batch(domains, limit)
for item in zip(domains, data):
print(item)
def testDA(self):
moz = MozCom(moz_account)
da = moz.get_ranking_data("")
print("da:", da)
