Exemplo n.º 1
0
def main():
    argparser = argparse.ArgumentParser(
        description='Make quantum leaps through RPKI history')

    argparser.add_argument('-c, --config',
                           nargs=1,
                           help='configuration file to use',
                           type=str,
                           metavar='config_file',
                           dest='config_file',
                           required=True)
    argparser.add_argument('-d, --date',
                           nargs=1,
                           help='date to process',
                           type=str,
                           metavar='process_date',
                           dest='process_date',
                           required=True)

    args = argparser.parse_args()

    try:
        sc.load_config(args.config_file[0])
    except Exception as e:
        print('Failed to load the configuration from {} ({})'.format(
            args.config_file[0], e))
        sys.exit(1)

    try:
        process_date(dateutil.parser.parse(args.process_date[0]).date())
    except Exception as e:
        print('Failed to process data for {} ({})'.format(
            args.process_date[0], e))
        sys.exit(1)
Exemplo n.º 2
0
#!/usr/bin/python

import sys
import os
import csv
from os.path import join

from simple_config import load_config
from forker import fork, call, check_call

downloader_config = load_config("downloaders.conf")


class Downloaders:
    def __init__(self, config=None):
        if config is None:
            config = downloader_config
        self.config = config
        self.code_dir = config["code_dir"]
        self.dpool_dir = join(self.code_dir, "BigSemanticsService",
                              "DownloaderPool")
        self.downloaders = config["downloaders"]

    def build(self):
        check_call(["ant", "clean"], wd=self.dpool_dir)
        check_call(["ant", "war"], wd=self.dpool_dir)
        check_call(["ant", "downloader-jar"], wd=self.dpool_dir)

    def put(self, local_file, remote_file):
        code_out_errs = []
        for d in self.downloaders:
Exemplo n.º 3
0
#!/usr/bin/python

import smtplib
from email.mime.text import MIMEText
from simple_config import load_config

config = load_config("bot.conf")


def send_bot_email(receiver, subject, message):
    login = config['bot_email_login']
    passwd = config['bot_email_passwd']
    # receiver = "*****@*****.**"

    mail_text = MIMEText(message)
    mail_text['Subject'] = subject
    mail_text['From'] = login
    mail_text['To'] = receiver

    smtp_server = smtplib.SMTP('smtp.gmail.com', 587)
    smtp_server.starttls()
    smtp_server.login(login, passwd)
    problems = smtp_server.sendmail(login, [receiver], mail_text.as_string())
    quit_status = smtp_server.quit()
    return (quit_status, problems)


def send_bot_email_to_maintainers(subject, message):
    return send_bot_email(config["receiver"], subject, message)
Exemplo n.º 4
0
#!/usr/bin/python

import urllib2
import urllib
from simple_config import load_config

tester_config = load_config("tester.conf")


class ServiceTester:
    def __init__(self, config=None):
        if config is None:
            config = tester_config
        self.config = config
        self.tests = config["tests"]

    def test_service(self):
        code = -1
        fatal = ""
        non_fatal = ""

        for t in self.tests:
            mmd = t.get("mmd")
            doc = t.get("doc")
            path = t.get("path")
            include = t.get("include")

            url = self.get_test_url(mmd, doc, path)

            (code, content, error) = self.access_and_download(url)
            if code < 0:
Exemplo n.º 5
0
#!/usr/bin/python

import smtplib
from email.mime.text import MIMEText
from simple_config import load_config

config = load_config("bot.conf")

def send_bot_email(receiver, subject, message):
  login = config['bot_email_login']
  passwd = config['bot_email_passwd']
  # receiver = "*****@*****.**"

  mail_text = MIMEText(message)
  mail_text['Subject'] = subject
  mail_text['From'] = login
  mail_text['To'] = receiver

  smtp_server = smtplib.SMTP('smtp.gmail.com', 587)
  smtp_server.starttls()
  smtp_server.login(login, passwd)
  problems = smtp_server.sendmail(login, [receiver], mail_text.as_string())
  quit_status = smtp_server.quit()
  return (quit_status, problems)

def send_bot_email_to_maintainers(subject, message):
  return send_bot_email(config["receiver"], subject, message)

Exemplo n.º 6
0
#!/usr/bin/python

import exceptions
import shutil
import time
import datetime
from copy import copy
from os import listdir, remove
from os.path import dirname, join

from simple_config import load_config
from bot_email import send_bot_email_to_maintainers
from forker import fork, call, check_call
import bs_service_tester

builder_config = load_config("builder.conf")

class ServiceBuilder:
  def __init__(self, config=None):
    if config is None:
      config = builder_config
    self.config = config
    # paths:
    self.code_dir = config["code_dir"]
    self.wrapper_repo = join(self.code_dir, "BigSemanticsWrapperRepository")
    self.wrapper_proj = join(self.wrapper_repo, "BigSemanticsWrappers")
    self.service_repo = join(self.code_dir, "BigSemanticsService")
    self.service_proj = join(self.service_repo, "BigSemanticsService")
    self.service_build = join(self.service_proj, "build")
    self.jetty_dir = config["jetty_dir"]
    self.webapps_dir = join(self.jetty_dir, "webapps")
#!/usr/bin/python

import urllib2
import urllib
from simple_config import load_config

tester_config = load_config("tester.conf")

class ServiceTester:
  def __init__(self, config=None):
    if config is None:
      config = tester_config
    self.config = config
    self.tests = config["tests"]

  def test_service(self):
    code = -1
    fatal = ""
    non_fatal = ""

    for t in self.tests:
      mmd = t.get("mmd")
      doc = t.get("doc")
      path = t.get("path")
      include = t.get("include")

      url = self.get_test_url(mmd, doc, path)

      (code, content, error) = self.access_and_download(url)
      if code < 0:
        error_msg = "(no error message available)"
Exemplo n.º 8
0
def main():
    argparser = argparse.ArgumentParser(
        description='Perform DNSSEC checks against Avro files in a directory')

    argparser.add_argument('-c, --config',
                           nargs=1,
                           help='configuration file to use',
                           type=str,
                           metavar='config_file',
                           dest='config_file',
                           required=True)
    argparser.add_argument('-d, --date',
                           nargs=1,
                           help='date to process (defaults to yesterday)',
                           type=str,
                           metavar='process_date',
                           dest='process_date',
                           required=False)

    args = argparser.parse_args()

    # Load configuration
    try:
        sc.load_config(args.config_file[0])
    except Exception as e:
        print(e)
        sys.exit(1)

    oilog.set_log_dir(sc.get_config_item('log_dir'))

    day = datetime.date.today() - datetime.timedelta(days=1)

    if args.process_date is not None:
        day = dateutil.parser.parse(args.process_date[0]).date()

    logger = oilog.OILog()
    logger.open('oi-dnssecchecks-{}-{}.log'.format(day,
                                                   sc.get_config_item('tld')))

    # Download required data
    if not download_data(logger, day):
        logger.log_err(
            'Failed to download data for {}, bailing out'.format(day))
        sys.exit(1)

    # Load TLSA sets
    tlsa_one_set = load_tlsa_list(
        '{}/tlsa-all-{}-{}.txt'.format(sc.get_config_item('tmp_dir'),
                                       sc.get_config_item('tld'), day), logger)
    tlsa_all_set = load_tlsa_list(
        '{}/tlsa-one-{}-{}.txt'.format(sc.get_config_item('tmp_dir'),
                                       sc.get_config_item('tld'), day), logger)

    cleanup_tmp_file('tlsa-all-{}-{}.txt'.format(sc.get_config_item('tld'),
                                                 day))
    cleanup_tmp_file('tlsa-one-{}-{}.txt'.format(sc.get_config_item('tld'),
                                                 day))

    try:
        process_avro_files(logger, day, sc.get_config_item('tmp_dir'),
                           sc.get_config_item('multi_process_count', 1),
                           sc.get_config_item('out_dir'),
                           sc.get_config_item('tld'), tlsa_one_set,
                           tlsa_all_set)
    except Exception as e:
        logger.log_err('Process terminated with an exception')
        logger.log_err(e)

    logger.close()
Exemplo n.º 9
0
#!/usr/bin/python

import sys
import os
import csv
from os.path import join

from simple_config import load_config
from forker import fork, call, check_call

downloader_config = load_config("downloaders.conf")

class Downloaders:
  def __init__(self, config = None):
    if config is None:
      config = downloader_config
    self.config = config
    self.code_dir = config["code_dir"]
    self.dpool_dir = join(self.code_dir, "BigSemanticsService", "DownloaderPool")
    self.downloaders = config["downloaders"]

  def build(self):
    check_call(["ant", "clean"], wd = self.dpool_dir)
    check_call(["ant", "war"], wd = self.dpool_dir)
    check_call(["ant", "downloader-jar"], wd = self.dpool_dir)

  def put(self, local_file, remote_file):
    code_out_errs = []
    for d in self.downloaders:
      userhost = d.user + "@" + d.host
      code_out_errs.append(call(["scp", "-i", d.login_id, "-P", str(d.port),
Exemplo n.º 10
0
#!/usr/bin/python

import exceptions
import shutil
import time
import datetime
from copy import copy
from os import listdir, remove
from os.path import dirname, join

from simple_config import load_config
from bot_email import send_bot_email_to_maintainers
from forker import fork, call, check_call
import bs_service_tester

builder_config = load_config("builder.conf")

class ServiceBuilder:
  def __init__(self, config=None):
    if config is None:
      config = builder_config
    self.config = config
    # paths:
    self.code_dir = config["code_dir"]
    self.wrapper_repo = join(self.code_dir, "BigSemanticsWrapperRepository")
    self.wrapper_proj = join(self.wrapper_repo, "BigSemanticsWrappers")
    self.service_repo = join(self.code_dir, "BigSemanticsService")
    self.service_proj = join(self.service_repo, "BigSemanticsService")
    self.service_build = join(self.service_proj, "build")
    self.jetty_dir = config["jetty_dir"]
    self.webapps_dir = join(self.jetty_dir, "webapps")