def export_domains(site):
db_export = CrawlerDb(site)
db_export.connect()
logger.info("=" * 40)
logger.info("Processing...")
domains = db_export.get_all_domains()
logger.info("There are %d domains" % len(domains))
file = open(DOMAINS_FILENAME, "w+")
file.writelines("\n".join(domains))
file.close()
logger.info("All domains saved to ./data/domains.csv")
logger.info("=" * 40)
if (arg == '--emails') or (arg == '-e'):
# Get all the emails and save in a CSV
logger.info("="*40)
logger.info("Processing...")
emails = db.get_all_emails()
logger.info("There are %d emails" % len(emails))
file = open(EMAILS_FILENAME, "w+")
file.writelines("\n".join(emails))
file.close()
logger.info("All emails saved to ./data/emails.csv")
logger.info("="*40)
elif (arg == '--domains') or (arg == '-d'):
# Get all the domains and save in a CSV
logger.info("="*40)
logger.info("Processing...")
domains = db.get_all_domains()
logger.info("There are %d domains" % len(domains))
file = open(DOMAINS_FILENAME, "w+")
file.writelines("\n".join(domains))
file.close()
logger.info("All domains saved to ./data/domains.csv")
logger.info("="*40)
else:
# Crawl the supplied keywords!
crawl(arg)
except KeyboardInterrupt:
logger.error("Stopping (KeyboardInterrupt)")
sys.exit()
except Exception as e:
logger.error("EXCEPTION: %s " % e)
if (arg == '--emails') or (arg == '-e'):
# Get all the emails and save in a CSV
logger.info("="*40)
logger.info("Processing...")
emails = db.get_all_emails()
logger.info("There are %d emails" % len(emails))
file = open(EMAILS_FILENAME, "w+")
file.writelines("\n".join(emails))
file.close()
logger.info("All emails saved to ./data/emails.csv")
logger.info("="*40)
elif (arg == '--domains') or (arg == '-d'):
# Get all the domains and save in a CSV
logger.info("="*40)
logger.info("Processing...")
domains = db.get_all_domains()
logger.info("There are %d domains" % len(domains))
file = open(DOMAINS_FILENAME, "w+")
file.writelines("\n".join(domains))
file.close()
logger.info("All domains saved to ./data/domains.csv")
logger.info("="*40)
else:
arg = glob.glob(arg)
# Crawl the supplied keywords!
crawl(arg)
except KeyboardInterrupt:
logger.error("Stopping (KeyboardInterrupt)")
sys.exit()
except Exception, e: