예제 #1
0
 def process_data(self, data: FilteredDomainData, **kwargs):
     result_ok = False
     if isinstance(data, FilteredDomainData):
         try:
             if len(data.domain_var) == 0:
                 data.domain_var = data.domain
             links = ArchiveOrg.get_url_info(data.domain_var, min_size=self._min_page_size, limit=-100)
             count = len(links)
             data.archive = count
             if count < self._min_profile:
                 pass
                 # raise ValueError("profile count is less than:" + str(self._min_profile))
             result_ok = True
         except Exception as ex:
             if not self._is_throughput_debug:
                 pass
                 # ErrorLogger.log_error("ArchiveOrgFilter.process_data()", ex, data.domain_var)
         finally:
             with self._sync_lock:
                 self._job_done += 1
                     #with self._process_queue_lock:
                 if result_ok:
                     if not self._is_throughput_debug:
                         CsvLogger.log_to_file(self._log_file, [(data.domain, data.da, data.archive)]) # log this to file
                     self._output_queue.put(data)
                     # return data
                 else:
                     if self._is_throughput_debug:
                         self._output_queue.put(data)
                     # return None
     else:
         return None
예제 #2
0
 def process_data(self, data: FilteredDomainData, **kwargs):
     account = kwargs.get("Account")
     # is_domain_good = False
     is_spammed = False
     try:
         if isinstance(data, FilteredDomainData) and isinstance(account, SiteAccount):
             majestic = MajesticCom(account)
             if self._en_spam_check:
                 self._filter_domain_name(domain=data.domain)
                 # self._filter_anchor_text(majestic, data.domain)
                 # self._filter_ref_domains(majestic, data.domain)
             if self._en_tf_check:
                 data = self._filter_tf_cf_backlink_ratio(majestic, data)
             if not (data.tf >= self._min_tf and data.ref_domains >= self._min_ref_domains):
                 raise ValueError("tf or cf doesn't match. tf:" + str(data.tf) + " cf: " + str(data.cf) + " ref domain: " + str(data.ref_domains))
             # if data.backlinks / data.ref_domains > self._max_backlink_to_ref_domain_ratio:
             #     raise MajesticSpamException("backlink to ref domain ratio is greater than {0:.1f}".format(self._max_backlink_to_ref_domain_ratio,))
             if self._en_spam_check:
                 self._filter_anchor_text(majestic, data.domain)
                 self._filter_ref_domains(majestic, data.domain)
             # is_domain_good = True
         else:
             raise ValueError("account is none in process_data")
     except MajesticSpamException as mjx_ex:
         is_spammed = True
         data.exception = str(mjx_ex)
     except Exception as ex:
         data.exception = str(ex)
         # ErrorLogger.log_error("MajesticFilter.process_data()", ex, str(data))
     finally:
         PrintLogger.print("Majestic processed: '" + str(data) + "' with: " + account.userID)
         if isinstance(data, FilteredDomainData):
             with self._sync_lock:
                 self._job_done += 1
                 if account is not None:
                     account.Available = True
                 # if data.cf >= self._min_cf and data.tf >= self._min_tf:
                 if data.tf >= self._min_tf and data.cf >= self._min_cf and data.ref_domains >= self._min_ref_domains:
                 # if data.tf >= self._min_tf and data.ref_domains >= self._min_ref_domains:
                     #print("Majatic output:", data)
                     # PrintLogger.print("domain: " + data.domain + " is good.")
                     if not self._is_throughput_debug:
                         if is_spammed:
                             CsvLogger.log_to_file(self._bad_log_file, [data.to_tuple()], dir_path=FilePath.get_temp_db_dir())
                         else:
                             CsvLogger.log_to_file(self._log_file, [data.to_tuple()], dir_path=FilePath.get_temp_db_dir()) # log this to file
                     self._output_queue.put(data)
                     return data
                 # elif is_spammed:
                 #     if not self._is_throughput_debug:
                 #         CsvLogger.log_to_file(self._bad_log_file, [data.to_tuple()], dir_path=FilePath.get_temp_db_dir())
                 #     self._output_queue.put(data)
                     # return data
                 else:
                     if self._is_throughput_debug:
                         self._output_queue.put(data)
                     # return None
                     # print("domain: " + data.domain + " has exception:" + data.exception)
         else:
             pass
예제 #3
0
 def process_data(self, data: FilteredDomainData, **kwargs):
     #print("MozFilter processing: ", data)
     account = kwargs.get("Account")
     try:
         if isinstance(data, FilteredDomainData) and isinstance(account, SiteAccount):
             if TldUtility.is_top_tld(data.domain):
                 sleep_time =random.randint(self._min_sleep_time, self._max_wait)
                 time.sleep(sleep_time)
                 moz = MozCom(account)
                 if not self._is_throughput_debug:
                     ranking = moz.get_ranking_data(data.domain)
                 else:
                     ranking = 100
                 data.da = ranking
             else:
                 pass
         else:
             raise ValueError("account is none in process_data")
     except Exception as ex:
         ErrorLogger.log_error("MozFilter", ex, "process_data() " + str(data) + " account: " + account.userID)
     finally:
         PrintLogger.print("Moz processed: " + str(data) + " with: " + account.userID)
         if isinstance(data, FilteredDomainData):
             with self._sync_lock:
                 self._job_done += 1
                 if account is not None:
                     account.Available = True
             if data.da >= self._min_DA_value:
                 if not self._is_throughput_debug:
                     CsvLogger.log_to_file(self._log_file, [(data.domain, data.da)]) # log this to file
                 self._output_queue.put(data)
예제 #4
0
 def testArchiveProfileFilter1(self):
     input_param ={"input_queue": queue.Queue(), "output_queue": queue.Queue(), "stop_event": Event(),
                   "throughput_debug": True, "worker_number": 2}
     archive_filter = get_archive_filter(**input_param)
     links = FileIO.FileHandler.read_lines_from_file("/Users/superCat/Desktop/PycharmProjectPortable/test/archive_domain_test.txt")
     for link in links:
         site = FilteredDomainData(domain=link)
         archive_filter.process_data(data=site, )
예제 #5
0
    def testMajesticFilter(self):

        filter = get_majestic_filter(worker_number=1, input_queue=Queue(), output_queue=Queue(), stop_event=Event())
        param = {"Account": majestic_account}
        links = FileIO.FileHandler.read_lines_from_file("/Users/superCat/Desktop/PycharmProjectPortable/test/spam_test2.txt")

        for link in links:
            link = LinkChecker.get_root_domain(link)[1]
            print("doing link:", link)
            site = FilteredDomainData(domain=link)
            filter.process_data(data=site, **param)
예제 #6
0
    def _filter_tf_cf_backlink_ratio(self, majestic: MajesticCom, data: FilteredDomainData) -> FilteredDomainData:
        ranking = majestic.get_cf_tf_list(["http://"+data.domain,
                                           "www."+data.domain,
                                           "http://www."+data.domain],
                                          is_dev=DomainFinderSrc.IS_DEBUG)
        if ranking is not None and len(ranking) > 0:
            current_tf = 0
            for item in ranking:
                if isinstance(item, MajesticComStruct):
                    item_cf_tf_ratio = 999
                    data_cf_tf_ratio = 999
                    item_deviation = 999
                    data_deviation = 999
                    if item.tf > 0:
                        item_cf_tf_ratio1 = abs(1-item.cf/item.tf)
                        item_cf_tf_ratio2 = abs(1-item.tf/item.cf)
                        item_deviation = min([item_cf_tf_ratio1, item_cf_tf_ratio2])
                    else:
                        continue
                    if data.tf > 0:
                        data_cf_tf_ratio1 = abs(1-data.cf/data.tf)
                        data_cf_tf_ratio2 = abs(1-data.tf/data.cf)
                        data_deviation = min([data_cf_tf_ratio1, data_cf_tf_ratio2])
                        # data_deviation = abs(1-data_cf_tf_ratio)

                    if item.tf >= self._min_tf  and item.cf >=self._min_cf and item_deviation < data_deviation and item_deviation <= self._cf_tf_deviation:
                        data.domain_var = item.domain
                        data.tf = item.tf
                        data.cf = item.cf
                        data.backlinks = item.backlinks
                        data.ref_domains = item.ref_domains
                        data.topic = item.topic
        return data
예제 #7
0
        def input_thread():
            data_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/03-09-2015-Bad-Results.csv"
            input_c = 0
            with open(data_path, mode='r', newline='') as csv_file:
                rd = csv.reader(csv_file, delimiter=',')
                for row in rd:

                    if input_c < site_count:
                        input_q.put(FilteredDomainData(domain=row[0]))
                        time.sleep(0.0001)
                    else:
                        break
                    input_c += 1
 def testFilter(self):
     manager = AccountManager()
     manager.AccountList.append(majestic_account)
     input_param = {
         "input_queue": queue.Queue(),
         "output_queue": queue.Queue(),
         "stop_event": Event()
     }
     filter = MajesticFilter(manager=manager, **input_param)
     param = {"Account": majestic_account}
     links = FileIO.FileHandler.read_lines_from_file(
         "/Users/superCat/Desktop/PycharmProjectPortable/test/spam_test1.txt"
     )
     for link in links:
         site = FilteredDomainData(domain=link)
         filter.process_data(data=site, **param)
예제 #9
0
 def testWriteToDb(self):
     db_path = "/Users/superCat/Desktop/PycharmProjectPortable/sync/FilteredSitesList"
     good_db = "/Users/superCat/Desktop/PycharmProjectPortable/sync/Majestic_filtering_good.csv"
     table = "20/12/2015 Legal"
     db = FilteredResultDB(table=table, offset=0, db_addr=db_path)
     count = 0
     temp_sites = []
     with open(good_db, mode='r', newline='') as csv_file:
         rd = csv.reader(csv_file, delimiter=',')
         for row in rd:
             if int(row[10]) > 1450612100:
                 data = FilteredDomainData.from_tuple(row)
                 print(data.__dict__)
                 count += 1
                 temp_sites.append(data)
     print("total:", count)
     db.add_sites(temp_sites, skip_check=False)
     db.close()
예제 #10
0
 def testFilterAll1(self):
     db_path = "/Users/superCat/Desktop/PycharmProjectPortable/sync/"
     manager = AccountManager(db_path)
     input_param ={"input_queue": queue.Queue(), "output_queue": queue.Queue(), "stop_event": Event(),
                   "throughput_debug": True, "worker_number": 2}
     file_path = "/Users/superCat/Desktop/PycharmProjectPortable/sync/Moz_filtering.csv"
     archive_filter = get_archive_filter(**input_param)
     majestic_filter = get_majestic_filter(**input_param)
     param = {"Account": majestic_account}
     count = 0
     with open(file_path, 'rt') as csvfile:
         reader = csv.reader(csvfile, delimiter=',')
         for row in reader:
             link, da = row
             site = FilteredDomainData(domain=link, da=int(da))
             print(count, " process:", link)
             archive_data = archive_filter.process_data(data=site, )
             if archive_data is not None:
                 majestic_data = majestic_filter.process_data(data=site, **param)
                 print(majestic_data)
             count += 1
예제 #11
0
 def format_input(data):
     if isinstance(data, FilteredDomainData):
         return data
     elif isinstance(data, tuple) and len(data) == 2:
         return FilteredDomainData(domain=data[0])
예제 #12
0
 def input_thread():
     for i in range(site_count):
         input_q.put(FilteredDomainData(domain="domain{0:d}.com".format(i,)))
         time.sleep(0.0001)