def main(): argparser = argparse.ArgumentParser( description='Make quantum leaps through RPKI history') argparser.add_argument('-c, --config', nargs=1, help='configuration file to use', type=str, metavar='config_file', dest='config_file', required=True) argparser.add_argument('-d, --date', nargs=1, help='date to process', type=str, metavar='process_date', dest='process_date', required=True) args = argparser.parse_args() try: sc.load_config(args.config_file[0]) except Exception as e: print('Failed to load the configuration from {} ({})'.format( args.config_file[0], e)) sys.exit(1) try: process_date(dateutil.parser.parse(args.process_date[0]).date()) except Exception as e: print('Failed to process data for {} ({})'.format( args.process_date[0], e)) sys.exit(1)
#!/usr/bin/python import sys import os import csv from os.path import join from simple_config import load_config from forker import fork, call, check_call downloader_config = load_config("downloaders.conf") class Downloaders: def __init__(self, config=None): if config is None: config = downloader_config self.config = config self.code_dir = config["code_dir"] self.dpool_dir = join(self.code_dir, "BigSemanticsService", "DownloaderPool") self.downloaders = config["downloaders"] def build(self): check_call(["ant", "clean"], wd=self.dpool_dir) check_call(["ant", "war"], wd=self.dpool_dir) check_call(["ant", "downloader-jar"], wd=self.dpool_dir) def put(self, local_file, remote_file): code_out_errs = [] for d in self.downloaders:
#!/usr/bin/python import smtplib from email.mime.text import MIMEText from simple_config import load_config config = load_config("bot.conf") def send_bot_email(receiver, subject, message): login = config['bot_email_login'] passwd = config['bot_email_passwd'] # receiver = "*****@*****.**" mail_text = MIMEText(message) mail_text['Subject'] = subject mail_text['From'] = login mail_text['To'] = receiver smtp_server = smtplib.SMTP('smtp.gmail.com', 587) smtp_server.starttls() smtp_server.login(login, passwd) problems = smtp_server.sendmail(login, [receiver], mail_text.as_string()) quit_status = smtp_server.quit() return (quit_status, problems) def send_bot_email_to_maintainers(subject, message): return send_bot_email(config["receiver"], subject, message)
#!/usr/bin/python import urllib2 import urllib from simple_config import load_config tester_config = load_config("tester.conf") class ServiceTester: def __init__(self, config=None): if config is None: config = tester_config self.config = config self.tests = config["tests"] def test_service(self): code = -1 fatal = "" non_fatal = "" for t in self.tests: mmd = t.get("mmd") doc = t.get("doc") path = t.get("path") include = t.get("include") url = self.get_test_url(mmd, doc, path) (code, content, error) = self.access_and_download(url) if code < 0:
#!/usr/bin/python import exceptions import shutil import time import datetime from copy import copy from os import listdir, remove from os.path import dirname, join from simple_config import load_config from bot_email import send_bot_email_to_maintainers from forker import fork, call, check_call import bs_service_tester builder_config = load_config("builder.conf") class ServiceBuilder: def __init__(self, config=None): if config is None: config = builder_config self.config = config # paths: self.code_dir = config["code_dir"] self.wrapper_repo = join(self.code_dir, "BigSemanticsWrapperRepository") self.wrapper_proj = join(self.wrapper_repo, "BigSemanticsWrappers") self.service_repo = join(self.code_dir, "BigSemanticsService") self.service_proj = join(self.service_repo, "BigSemanticsService") self.service_build = join(self.service_proj, "build") self.jetty_dir = config["jetty_dir"] self.webapps_dir = join(self.jetty_dir, "webapps")
#!/usr/bin/python import urllib2 import urllib from simple_config import load_config tester_config = load_config("tester.conf") class ServiceTester: def __init__(self, config=None): if config is None: config = tester_config self.config = config self.tests = config["tests"] def test_service(self): code = -1 fatal = "" non_fatal = "" for t in self.tests: mmd = t.get("mmd") doc = t.get("doc") path = t.get("path") include = t.get("include") url = self.get_test_url(mmd, doc, path) (code, content, error) = self.access_and_download(url) if code < 0: error_msg = "(no error message available)"
def main(): argparser = argparse.ArgumentParser( description='Perform DNSSEC checks against Avro files in a directory') argparser.add_argument('-c, --config', nargs=1, help='configuration file to use', type=str, metavar='config_file', dest='config_file', required=True) argparser.add_argument('-d, --date', nargs=1, help='date to process (defaults to yesterday)', type=str, metavar='process_date', dest='process_date', required=False) args = argparser.parse_args() # Load configuration try: sc.load_config(args.config_file[0]) except Exception as e: print(e) sys.exit(1) oilog.set_log_dir(sc.get_config_item('log_dir')) day = datetime.date.today() - datetime.timedelta(days=1) if args.process_date is not None: day = dateutil.parser.parse(args.process_date[0]).date() logger = oilog.OILog() logger.open('oi-dnssecchecks-{}-{}.log'.format(day, sc.get_config_item('tld'))) # Download required data if not download_data(logger, day): logger.log_err( 'Failed to download data for {}, bailing out'.format(day)) sys.exit(1) # Load TLSA sets tlsa_one_set = load_tlsa_list( '{}/tlsa-all-{}-{}.txt'.format(sc.get_config_item('tmp_dir'), sc.get_config_item('tld'), day), logger) tlsa_all_set = load_tlsa_list( '{}/tlsa-one-{}-{}.txt'.format(sc.get_config_item('tmp_dir'), sc.get_config_item('tld'), day), logger) cleanup_tmp_file('tlsa-all-{}-{}.txt'.format(sc.get_config_item('tld'), day)) cleanup_tmp_file('tlsa-one-{}-{}.txt'.format(sc.get_config_item('tld'), day)) try: process_avro_files(logger, day, sc.get_config_item('tmp_dir'), sc.get_config_item('multi_process_count', 1), sc.get_config_item('out_dir'), sc.get_config_item('tld'), tlsa_one_set, tlsa_all_set) except Exception as e: logger.log_err('Process terminated with an exception') logger.log_err(e) logger.close()
#!/usr/bin/python import sys import os import csv from os.path import join from simple_config import load_config from forker import fork, call, check_call downloader_config = load_config("downloaders.conf") class Downloaders: def __init__(self, config = None): if config is None: config = downloader_config self.config = config self.code_dir = config["code_dir"] self.dpool_dir = join(self.code_dir, "BigSemanticsService", "DownloaderPool") self.downloaders = config["downloaders"] def build(self): check_call(["ant", "clean"], wd = self.dpool_dir) check_call(["ant", "war"], wd = self.dpool_dir) check_call(["ant", "downloader-jar"], wd = self.dpool_dir) def put(self, local_file, remote_file): code_out_errs = [] for d in self.downloaders: userhost = d.user + "@" + d.host code_out_errs.append(call(["scp", "-i", d.login_id, "-P", str(d.port),