def testFilterExportDB(self):
from_addr = "/Users/superCat/Desktop/PycharmProjectPortable/sync/FilteredSitesList_oLD"
to_addr = "/Users/superCat/Desktop/PycharmProjectPortable/sync/FilteredSitesList"
table_name = "20/11/2015"
from_db = FilteredResultDB(table_name, db_addr=from_addr)
to_db = FilteredResultDB(table_name, db_addr=to_addr)
results = [x for x in from_db.get_all_sites() if x[1] > 0]
count = 0
for item in results:
print("count:", count, "item:", item)
count += 1
def testExportCsv(self):
from_addr = "/Users/superCat/Desktop/PycharmProjectPortable/sync/Sum.db"
to_addr = "/Users/superCat/Desktop/PycharmProjectPortable/sync/2015_OLD.csv"
table_name = "2015 Old"
from_db = FilteredResultDB(table_name, db_addr=from_addr)
data = [x for x in from_db.get_all_sites() if x[1] > 0]
CsvLogger.log_to_file_path(to_addr, [FilteredResultDB.get_fields_names(),])
CsvLogger.log_to_file_path(to_addr, data)
def testFilterExportDB2(self):
from_addr = "/Users/superCat/Desktop/PycharmProjectPortable/sync/FilteredSitesList.db"
to_addr = "/Users/superCat/Desktop/PycharmProjectPortable/sync/Sum.db"
table_name = "20/11/2015"
from_db = FilteredResultDB(table_name, db_addr=from_addr)
from_db.cur.execute("SELECT name FROM sqlite_master WHERE type = 'table';")
table_names = [x[0] for x in from_db.cur.fetchall()]
to_db = FilteredResultDB("2015 Old", db_addr=to_addr)
for table_name in table_names:
print(table_name)
temp = FilteredResultDB(table_name, db_addr=from_addr)
results = [x for x in temp.get_all_sites() if x[1] > 0]
temp.close()
count = 0
for item in results:
print("count:", count, "item:", item)
count += 1
to_db.add_sites(results, skip_check=True)
from_db.close()
to_db.close()
def testReadFromDb(self):
db_path = "/Users/superCat/Desktop/PycharmProjectPortable/test/sync/FilteredSitesList"
good_db = "/Users/superCat/Desktop/PycharmProjectPortable/test/sync/FilteredSitesList_Good"
bad_db = "/Users/superCat/Desktop/PycharmProjectPortable/test/sync/FilteredSitesList_Bad"
table = "01/10/2015 Gambling"
db = FilteredResultDB(table=table, offset=0, db_addr=db_path)
total_record = 10000
patch = 10
count = 0
while count < total_record:
sites = db.get_next_patch(patch, rollover=False)
for item in sites:
print("item number:", count, " ", item)
count += 1
def testWriteToDb(self):
db_path = "/Users/superCat/Desktop/PycharmProjectPortable/sync/FilteredSitesList"
good_db = "/Users/superCat/Desktop/PycharmProjectPortable/sync/Majestic_filtering_good.csv"
table = "20/12/2015 Legal"
db = FilteredResultDB(table=table, offset=0, db_addr=db_path)
count = 0
temp_sites = []
with open(good_db, mode='r', newline='') as csv_file:
rd = csv.reader(csv_file, delimiter=',')
for row in rd:
if int(row[10]) > 1450612100:
data = FilteredDomainData.from_tuple(row)
print(data.__dict__)
count += 1
temp_sites.append(data)
print("total:", count)
db.add_sites(temp_sites, skip_check=False)
db.close()