org_false_results = tp_org_false_results\
.merge(to_org_false_results, how="outer", on=["test_key", "result_key"])
retall_results = l1s_retall_results\
.merge(l2_retall_results, how="outer", on=["test_key", "result_key"])
write_df(from_root("results\\predictions.csv"), results)
write_df(from_root("results\\predictions_org_false.csv"), org_false_results)
write_df(from_root("results\\predictions_retall.csv"), retall_results)
db.insert(results, "predictions", "dbo")
print("Finished writing results to CSV and database.")
if __name__ == "__main__":
print("Started executing script.\n")
start_time = datetime.now()
logger = logging.getLogger(__name__)
set_params(logger, from_root("log\\test.log"))
try:
main()
except Exception as e:
logger.exception("test.py: Fatal error")
sys.exit(1)
print(f"\nExecution time: {datetime.now() - start_time}")
print("Finished executing script.")
# Symbolic
l1s_module = Level1SymbolicModule(to_module)
l1s_module.retrain(l1_df)
l1s_module.save_to_file(from_root("pkl\\level_1_symbolic_module.pkl"))
# ==========================================================================
# Level 2
l2_df = db.extract(from_root("sql\\train\\level_2.sql"))
l2_module = Level2Module(l1s_module)
l2_module.retrain(l2_df)
l2_module.save_to_file(from_root("pkl\\level_2_module.pkl"))
if __name__ == "__main__":
print("Started executing script.\n")
start_time = datetime.now()
logger = logging.getLogger(__name__)
set_params(logger, from_root("log\\train.log"))
try:
main()
except Exception as e:
logger.exception("train.py: Fatal error")
sys.exit(1)
print(f"\nExecution time: {datetime.now() - start_time}")
print("Finished executing script.")
l2_df = db.extract(L2_SQL)
l2_times = timer(l2_module_factory, l2_df, SIZES)
l2_line, = plt.plot(SIZES, l2_times, marker="o", label="Level 2")
# ==========================================================================
plt.title("Time complexity plot")
plt.xlabel("Training data size (number of rows)")
plt.ylabel("Retrain time (seconds)")
plt.legend(handles=[tp_line, to_line, l1ml_line, l1s_line, l2_line])
write_plot(SAVE_TO, plt)
if __name__ == "__main__":
print("Started executing script.\n")
start_time = datetime.now()
logger = logging.getLogger(__name__)
set_params(logger, from_root("log\\complexity.log"))
try:
main()
except Exception as e:
logger.exception("complexity.py: Fatal error")
sys.exit(1)
print(f"\nExecution time: {datetime.now() - start_time}")
print("Finished executing script.")
def verify_level_2(db):
l1_df = db.extract(L1_SQL)
l1ml_module = Level1MLModule()
l1ml_module.retrain(l1_df)
def l2_module_factory():
return Level2Module(l1ml_module)
l2_df = db.extract(L2_SQL)
verify_module(l2_module_factory, l2_df, "level_2",
os.path.join(SAVE_TO, "level_2"))
if __name__ == "__main__":
print("Started executing script.\n")
start_time = datetime.now()
logger = logging.getLogger(__name__)
set_params(logger, from_root("log\\verify.log"))
try:
main()
except Exception as e:
logger.exception("verify.py: Fatal error")
sys.exit(1)
print(f"\nExecution time: {datetime.now() - start_time}")
print("Finished executing script.")
fold1 = "raw_html/apa/" + this_month()
fold2 = "raw_html/roo/" + this_month()
os.mkdir(fold1)
os.mkdir(fold2)
os.chdir('../rental_crawler')
if __name__ == '__main__':
print('Reminder: Press Ctrl+{0} to exit'.format('Break' if os.name ==
'nt' else 'C'))
# initiate logger to store error message
logger = logging.getLogger(__name__)
#logger.setLevel(logging.DEBUG)
set_params(logger, from_root("../log/scheduler.log"))
scheduler = TwistedScheduler()
process = CrawlerProcess()
## On the first day of each month, disable and reset scrapy deltafetch, retrieve all available ads on the domain
scheduler.add_job(mkdir, 'cron', day='1', hour=5, minute=rand_min())
scheduler.add_job(process.crawl,
'cron',
args=[ApaSpider],
day='1',
hour=8,
minute=rand_min())
scheduler.add_job(process.crawl,
'cron',
args=[RooSpider],
SQL_FILEPATH = from_root("sql\\needs_tagging.sql")
TABLE = "metamap"
SCHEMA = "dbo"
OBSERVATIONS = False
def main():
db = Database.get_instance()
df = db.extract(SQL_FILEPATH)
annotations = annotate(df, observations=OBSERVATIONS)
db.insert(annotations, TABLE, SCHEMA)
if __name__ == "__main__":
print("Started executing script.\n")
start_time = datetime.now()
logger = logging.getLogger(__name__)
set_params(logger, from_root("log\\metamap.log"))
try:
main()
except Exception as e:
logger.exception("metamap.py: Fatal error")
sys.exit(1)
print(f"\nExecution time: {datetime.now() - start_time}")
print("Finished executing script.")