def main():
argparser = argparse.ArgumentParser(
description='Make quantum leaps through RPKI history')
argparser.add_argument('-c, --config',
nargs=1,
help='configuration file to use',
type=str,
metavar='config_file',
dest='config_file',
required=True)
argparser.add_argument('-d, --date',
nargs=1,
help='date to process',
type=str,
metavar='process_date',
dest='process_date',
required=True)
args = argparser.parse_args()
try:
sc.load_config(args.config_file[0])
except Exception as e:
print('Failed to load the configuration from {} ({})'.format(
args.config_file[0], e))
sys.exit(1)
try:
process_date(dateutil.parser.parse(args.process_date[0]).date())
except Exception as e:
print('Failed to process data for {} ({})'.format(
args.process_date[0], e))
sys.exit(1)
#!/usr/bin/python
import sys
import os
import csv
from os.path import join
from simple_config import load_config
from forker import fork, call, check_call
downloader_config = load_config("downloaders.conf")
class Downloaders:
def __init__(self, config=None):
if config is None:
config = downloader_config
self.config = config
self.code_dir = config["code_dir"]
self.dpool_dir = join(self.code_dir, "BigSemanticsService",
"DownloaderPool")
self.downloaders = config["downloaders"]
def build(self):
check_call(["ant", "clean"], wd=self.dpool_dir)
check_call(["ant", "war"], wd=self.dpool_dir)
check_call(["ant", "downloader-jar"], wd=self.dpool_dir)
def put(self, local_file, remote_file):
code_out_errs = []
for d in self.downloaders:
#!/usr/bin/python
import smtplib
from email.mime.text import MIMEText
from simple_config import load_config
config = load_config("bot.conf")
def send_bot_email(receiver, subject, message):
login = config['bot_email_login']
passwd = config['bot_email_passwd']
# receiver = "*****@*****.**"
mail_text = MIMEText(message)
mail_text['Subject'] = subject
mail_text['From'] = login
mail_text['To'] = receiver
smtp_server = smtplib.SMTP('smtp.gmail.com', 587)
smtp_server.starttls()
smtp_server.login(login, passwd)
problems = smtp_server.sendmail(login, [receiver], mail_text.as_string())
quit_status = smtp_server.quit()
return (quit_status, problems)
def send_bot_email_to_maintainers(subject, message):
return send_bot_email(config["receiver"], subject, message)
#!/usr/bin/python
import urllib2
import urllib
from simple_config import load_config
tester_config = load_config("tester.conf")
class ServiceTester:
def __init__(self, config=None):
if config is None:
config = tester_config
self.config = config
self.tests = config["tests"]
def test_service(self):
code = -1
fatal = ""
non_fatal = ""
for t in self.tests:
mmd = t.get("mmd")
doc = t.get("doc")
path = t.get("path")
include = t.get("include")
url = self.get_test_url(mmd, doc, path)
(code, content, error) = self.access_and_download(url)
if code < 0:
#!/usr/bin/python
import smtplib
from email.mime.text import MIMEText
from simple_config import load_config
config = load_config("bot.conf")
def send_bot_email(receiver, subject, message):
login = config['bot_email_login']
passwd = config['bot_email_passwd']
# receiver = "*****@*****.**"
mail_text = MIMEText(message)
mail_text['Subject'] = subject
mail_text['From'] = login
mail_text['To'] = receiver
smtp_server = smtplib.SMTP('smtp.gmail.com', 587)
smtp_server.starttls()
smtp_server.login(login, passwd)
problems = smtp_server.sendmail(login, [receiver], mail_text.as_string())
quit_status = smtp_server.quit()
return (quit_status, problems)
def send_bot_email_to_maintainers(subject, message):
return send_bot_email(config["receiver"], subject, message)
#!/usr/bin/python
import exceptions
import shutil
import time
import datetime
from copy import copy
from os import listdir, remove
from os.path import dirname, join
from simple_config import load_config
from bot_email import send_bot_email_to_maintainers
from forker import fork, call, check_call
import bs_service_tester
builder_config = load_config("builder.conf")
class ServiceBuilder:
def __init__(self, config=None):
if config is None:
config = builder_config
self.config = config
# paths:
self.code_dir = config["code_dir"]
self.wrapper_repo = join(self.code_dir, "BigSemanticsWrapperRepository")
self.wrapper_proj = join(self.wrapper_repo, "BigSemanticsWrappers")
self.service_repo = join(self.code_dir, "BigSemanticsService")
self.service_proj = join(self.service_repo, "BigSemanticsService")
self.service_build = join(self.service_proj, "build")
self.jetty_dir = config["jetty_dir"]
self.webapps_dir = join(self.jetty_dir, "webapps")
#!/usr/bin/python
import urllib2
import urllib
from simple_config import load_config
tester_config = load_config("tester.conf")
class ServiceTester:
def __init__(self, config=None):
if config is None:
config = tester_config
self.config = config
self.tests = config["tests"]
def test_service(self):
code = -1
fatal = ""
non_fatal = ""
for t in self.tests:
mmd = t.get("mmd")
doc = t.get("doc")
path = t.get("path")
include = t.get("include")
url = self.get_test_url(mmd, doc, path)
(code, content, error) = self.access_and_download(url)
if code < 0:
error_msg = "(no error message available)"
def main():
argparser = argparse.ArgumentParser(
description='Perform DNSSEC checks against Avro files in a directory')
argparser.add_argument('-c, --config',
nargs=1,
help='configuration file to use',
type=str,
metavar='config_file',
dest='config_file',
required=True)
argparser.add_argument('-d, --date',
nargs=1,
help='date to process (defaults to yesterday)',
type=str,
metavar='process_date',
dest='process_date',
required=False)
args = argparser.parse_args()
# Load configuration
try:
sc.load_config(args.config_file[0])
except Exception as e:
print(e)
sys.exit(1)
oilog.set_log_dir(sc.get_config_item('log_dir'))
day = datetime.date.today() - datetime.timedelta(days=1)
if args.process_date is not None:
day = dateutil.parser.parse(args.process_date[0]).date()
logger = oilog.OILog()
logger.open('oi-dnssecchecks-{}-{}.log'.format(day,
sc.get_config_item('tld')))
# Download required data
if not download_data(logger, day):
logger.log_err(
'Failed to download data for {}, bailing out'.format(day))
sys.exit(1)
# Load TLSA sets
tlsa_one_set = load_tlsa_list(
'{}/tlsa-all-{}-{}.txt'.format(sc.get_config_item('tmp_dir'),
sc.get_config_item('tld'), day), logger)
tlsa_all_set = load_tlsa_list(
'{}/tlsa-one-{}-{}.txt'.format(sc.get_config_item('tmp_dir'),
sc.get_config_item('tld'), day), logger)
cleanup_tmp_file('tlsa-all-{}-{}.txt'.format(sc.get_config_item('tld'),
day))
cleanup_tmp_file('tlsa-one-{}-{}.txt'.format(sc.get_config_item('tld'),
day))
try:
process_avro_files(logger, day, sc.get_config_item('tmp_dir'),
sc.get_config_item('multi_process_count', 1),
sc.get_config_item('out_dir'),
sc.get_config_item('tld'), tlsa_one_set,
tlsa_all_set)
except Exception as e:
logger.log_err('Process terminated with an exception')
logger.log_err(e)
logger.close()
#!/usr/bin/python
import sys
import os
import csv
from os.path import join
from simple_config import load_config
from forker import fork, call, check_call
downloader_config = load_config("downloaders.conf")
class Downloaders:
def __init__(self, config = None):
if config is None:
config = downloader_config
self.config = config
self.code_dir = config["code_dir"]
self.dpool_dir = join(self.code_dir, "BigSemanticsService", "DownloaderPool")
self.downloaders = config["downloaders"]
def build(self):
check_call(["ant", "clean"], wd = self.dpool_dir)
check_call(["ant", "war"], wd = self.dpool_dir)
check_call(["ant", "downloader-jar"], wd = self.dpool_dir)
def put(self, local_file, remote_file):
code_out_errs = []
for d in self.downloaders:
userhost = d.user + "@" + d.host
code_out_errs.append(call(["scp", "-i", d.login_id, "-P", str(d.port),
#!/usr/bin/python
import exceptions
import shutil
import time
import datetime
from copy import copy
from os import listdir, remove
from os.path import dirname, join
from simple_config import load_config
from bot_email import send_bot_email_to_maintainers
from forker import fork, call, check_call
import bs_service_tester
builder_config = load_config("builder.conf")
class ServiceBuilder:
def __init__(self, config=None):
if config is None:
config = builder_config
self.config = config
# paths:
self.code_dir = config["code_dir"]
self.wrapper_repo = join(self.code_dir, "BigSemanticsWrapperRepository")
self.wrapper_proj = join(self.wrapper_repo, "BigSemanticsWrappers")
self.service_repo = join(self.code_dir, "BigSemanticsService")
self.service_proj = join(self.service_repo, "BigSemanticsService")
self.service_build = join(self.service_proj, "build")
self.jetty_dir = config["jetty_dir"]
self.webapps_dir = join(self.jetty_dir, "webapps")