def process_data(self, data: FilteredDomainData, **kwargs):
account = kwargs.get("Account")
# is_domain_good = False
is_spammed = False
try:
if isinstance(data, FilteredDomainData) and isinstance(account, SiteAccount):
majestic = MajesticCom(account)
if self._en_spam_check:
self._filter_domain_name(domain=data.domain)
# self._filter_anchor_text(majestic, data.domain)
# self._filter_ref_domains(majestic, data.domain)
if self._en_tf_check:
data = self._filter_tf_cf_backlink_ratio(majestic, data)
if not (data.tf >= self._min_tf and data.ref_domains >= self._min_ref_domains):
raise ValueError("tf or cf doesn't match. tf:" + str(data.tf) + " cf: " + str(data.cf) + " ref domain: " + str(data.ref_domains))
# if data.backlinks / data.ref_domains > self._max_backlink_to_ref_domain_ratio:
# raise MajesticSpamException("backlink to ref domain ratio is greater than {0:.1f}".format(self._max_backlink_to_ref_domain_ratio,))
if self._en_spam_check:
self._filter_anchor_text(majestic, data.domain)
self._filter_ref_domains(majestic, data.domain)
# is_domain_good = True
else:
raise ValueError("account is none in process_data")
except MajesticSpamException as mjx_ex:
is_spammed = True
data.exception = str(mjx_ex)
except Exception as ex:
data.exception = str(ex)
# ErrorLogger.log_error("MajesticFilter.process_data()", ex, str(data))
finally:
PrintLogger.print("Majestic processed: '" + str(data) + "' with: " + account.userID)
if isinstance(data, FilteredDomainData):
with self._sync_lock:
self._job_done += 1
if account is not None:
account.Available = True
# if data.cf >= self._min_cf and data.tf >= self._min_tf:
if data.tf >= self._min_tf and data.cf >= self._min_cf and data.ref_domains >= self._min_ref_domains:
# if data.tf >= self._min_tf and data.ref_domains >= self._min_ref_domains:
#print("Majatic output:", data)
# PrintLogger.print("domain: " + data.domain + " is good.")
if not self._is_throughput_debug:
if is_spammed:
CsvLogger.log_to_file(self._bad_log_file, [data.to_tuple()], dir_path=FilePath.get_temp_db_dir())
else:
CsvLogger.log_to_file(self._log_file, [data.to_tuple()], dir_path=FilePath.get_temp_db_dir()) # log this to file
self._output_queue.put(data)
return data
# elif is_spammed:
# if not self._is_throughput_debug:
# CsvLogger.log_to_file(self._bad_log_file, [data.to_tuple()], dir_path=FilePath.get_temp_db_dir())
# self._output_queue.put(data)
# return data
else:
if self._is_throughput_debug:
self._output_queue.put(data)
# return None
# print("domain: " + data.domain + " has exception:" + data.exception)
else:
pass
from DomainFinderSrc.MajesticCom import *
from DomainFinderSrc.SiteConst import *
majestic_account = SiteAccount(siteType=AccountType.Majestic,
userID="*****@*****.**",
password="******",
APIkey="1BB1D141D20CAF35D331F086F55C1CEE")
majestic = MajesticCom(majestic_account)
moz_account = SiteAccount(siteType=AccountType.Moz,
userID="*****@*****.**",
password="******",
AccessID="mozscape-320a4616a8",
APIkey="f03c19321b0973573137288c647b31ea")
moz_account_fake = SiteAccount(siteType=AccountType.Moz,
userID="*****@*****.**",
password="******",
AccessID="mozscape-44a37bfcd5",
APIkey="bedefa75b4c17317a94a421108974f1d")
amazon_ec2_account = SiteAccount(
siteType=AccountType.AmazonEC2,
userID="*****@*****.**",
AccessID="AKIAIPA2WM3ILJWR2KSA",
APIkey="7EisLQmbOv04ExZM9Fj1rxnmWiKw8wae5shRPDdx")
buy_proxy_org_account = SiteAccount(siteType=AccountType.BuyProxyOrg,
userID="*****@*****.**",
password="******",
AccessID="49885",
