def __init__(self, session=None, server=None):
self.logger = settings.get_logger(settings.LOG, self.__class__.__name__)
self.session = session
self.server = server
self.connect()
self.db = self._conn[self.Meta.db]
self.admin_db = self._conn[settings.ADMIN_DB]
self.collection = self._conn[self.Meta.db][self.Meta.collection]
def __init__(self, port):
self.logger = settings.get_logger(settings.LOG, self.__class__.__name__)
#sockets = tornado.netutil.bind_sockets(port)
#tornado.process.fork_processes(0)
server = tornado.httpserver.HTTPServer(TORNADO_APP)
server.bind(port)
server.start()
settings.startup()
self.logger.debug("Server Started: %s" % port)
tornado.ioloop.IOLoop.instance().start()
def __init__(self, view=View, *args, **kwargs):
super(Model, self).__init__(*args, **kwargs)
self.logger = settings.get_logger(settings.LOG, self.__class__.__name__)
self.admin_db = self.__conn__[self.__admindb__]
self.__logs__ = []
self.__view__ = view(context=self) if not self.__currentview__ else self.__currentview__(context=self)
try:
self.init(*args, **kwargs)
except Exception as e:
self.logger.exception(e)
pass
def get_file_config(filepath):
logger = settings.get_logger('blippd')
try:
with open(filepath) as f:
conf = f.read()
return json.loads(conf)
except IOError as e:
logger.exc('get_file_config', e)
logger.error('get_file_config',
msg="Could not open config file... exiting")
exit(1)
except ValueError as e:
logger.exc('get_file_config', e)
logger.error('get_file_config',
msg="Config file is not valid json... exiting")
exit(1)
def main():
"""Run periscope"""
ssl_opts = None
logger = settings.get_logger()
logger.info('periscope.start')
loop = tornado.ioloop.IOLoop.instance()
# parse command line options
tornado.options.parse_command_line()
app = PeriscopeApplication()
if settings.ENABLE_SSL:
ssl_opts = settings.SSL_OPTIONS
http_server = tornado.httpserver.HTTPServer(app, ssl_options=ssl_opts)
http_server.listen(options.port, address=options.address)
loop.start()
logger.info('periscope.end')
def main(options=None):
options = get_options() if not options else options
logger = settings.get_logger('blippd', options['--log'], options['--log-level'])
conf = deepcopy(settings.STANDALONE_DEFAULTS)
cconf = {
"id": options.get("--service-id", None),
"name": "blipp",
"properties": {
"configurations": {
"unis_url": options.get("--unis-url", None),
}
}
}
delete_nones(cconf)
merge_dicts(conf, cconf)
if options['--config-file']:
fconf = get_file_config(options['--config-file'])
merge_dicts(conf, fconf)
bconf = BlippConfigure(initial_config=conf,
node_id=options['--node-id'],
pre_existing_measurements=options['--existing'],
urn=options['--urn'])
bconf.initialize()
config = bconf.config
logger.info('main', config=pprint.pformat(config))
logger.warn('NODE: ' + HOSTNAME, config=pprint.pformat(config))
if options['--daemonize']:
with daemon.DaemonContext():
arbiter.main(bconf)
else:
arbiter.main(bconf)
parser.add_argument(
'--augment-size',
type=float,
help='Propertion of data to augment. Default to 0.2 of one-fifth.')
args = parser.parse_args()
LOG_LEVELS_MAP = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL,
}
LOGGER = get_logger('', LOG_LEVELS_MAP.get(args.log_level))
LOGGER.debug(f'Model argument: {args.model}')
MODEL_FILENAME = args.save_model
DATA_AUGMENTATION = args.data_augmentation
AUGMENTATION_SIZE = 0
if DATA_AUGMENTATION:
AUGMENTATION_SIZE = args.augment_size or 0.2
if args.model == 'naive':
LOGGER.info('Using naive bayes model')
# end run if already exists
mlflow.end_run()
import discord
from discord.ext import commands
from settings import get_config, get_logger
from utils import steam
parser = argparse.ArgumentParser(description='Process arguments.')
parser.add_argument('--config_file',
dest='config_file',
default='./config/local.yaml',
help='config file path')
PATH = parser.parse_args().config_file
CONFIG = get_config(PATH)
LOGGER = get_logger()
bot = commands.Bot(command_prefix='!')
@bot.event
async def on_error(event, *args, **kwargs):
if event == 'on_message':
LOGGER.info(f'Unhandled message: {args[0]}\n')
else:
raise
@bot.event
async def on_ready():
# license. See the COPYING file for details.
#
# This software was created at the Indiana University Center for Research in
# Extreme Scale Technologies (CREST).
# =============================================================================
'''
Server for blipp configuration and control via cmd line interface.
'''
import zmq
import settings
import json
import time
from config_server_api import RELOAD, GET_CONFIG, POLL_CONFIG
logger = settings.get_logger('config_server')
class ConfigServer:
def __init__(self, conf_obj):
self.context = zmq.Context()
self.conf_obj = conf_obj
self.changed = True
self.socket = self.context.socket(zmq.REP)
self.socket.bind("ipc:///tmp/blipp_socket_zcWcfO0ydo")
def listen(self, timeout):
cur_time = time.time()
finish_time = cur_time + timeout
while cur_time < finish_time:
logger.info("listen", msg="polling for %d"%(finish_time-cur_time))
if self.socket.poll((finish_time - cur_time)*1000):
from airflow.utils.db import create_session
from airflow.secrets import get_variable
from settings import Config, get_logger, ROOT_DIR
from workflow.conn import update_connection, ConnectionConfig
from workflow.helper import redis_session
from workflow.plugins.operators.commons import CallbackMixin, VariableMixin
from workflow.exceptions import (
DataCheckFailed,
NoRecordsFoundError,
ProcessFailedError,
TableDoesNotExistError,
)
logger = get_logger('workflow.plugins.operators.etl')
def _setup_local():
"Create directory used for processing if not exists." ""
for cat in Config.DATA_CATEGORIES:
path = os.path.join(Config.DATA_DIR, 'uploaded', 'reviews', cat)
if not os.path.isdir(path):
os.makedirs(path)
path = os.path.join(Config.DATA_DIR, 'raw', 'reviews', cat)
if not os.path.isdir(path):
os.makedirs(path)
path = os.path.join(Config.DATA_DIR, 'staging', 'reviews', cat)
if not os.path.isdir(path):
import json
import requests
import sys
import uuid
import os
from stat import ST_MTIME
from multiprocessing import Process, Pipe
from ms_client import MSInstance
from unis_client import UNISInstance
from collector import Collector
from sched_obj import SchedObj, FakeDict
import settings
logger = settings.get_logger("sched")
SETTINGS_FILE = os.path.dirname(__file__) + "/settings.py"
if settings.DEBUG:
for item in dir(settings):
if ord(item[0]) >= 65 and ord(item[0]) <= 90:
logger.debug("settings", item=item, value=str(settings.__getattribute__(item))) # @UndefinedVariable
# loop every check interval:
# check settings file and reload
# check probe_settings files and add to probes_reload if changed
# loop probes:
# does it have a process?
# no: create and start it
# yes: is it not alive?
# join it check exit code
from settings import get_logger
logger = get_logger('views_logger')
import ftplib
import os
import json
import gzip
import io
from concurrent.futures import ThreadPoolExecutor
import time
import pprint
import settings
import convert_band_data
import get_cytobands_from_remote_dbs
output_dir = '../../data/bands/native/'
logger = settings.get_logger(output_dir, 'get_chromosomes')
if os.path.exists(output_dir) == False:
os.mkdir(output_dir)
orgs_with_centromere_data = {}
ftp_domain = 'ftp.ncbi.nlm.nih.gov'
manifest = {}
def get_chromosome_object(agp):
"""Extracts centromere coordinates and chromosome length from AGP data,
and returns a chromosome object formatted in JSON"""
import subprocess
import re
from utils import full_event_types
import shlex
import settings
logger = settings.get_logger('cmd_line_probe')
rtt_list = []
e_list = []
class Probe:
def __init__(self, service, measurement):
self.service = service
self.measurement = measurement
self.config = measurement["configuration"]
self.command = self._substitute_command(str(self.config.get("command")), self.config)
try:
self.data_regex = re.compile(
str(self.config["regex"]),
flags=re.S|re.M)
except Exception:
self.data_regex = None
try:
self.EVENT_TYPES = self.config["eventTypes"]
except Exception:
self.EVENT_TYPES = {}
def get_data(self, ip):
proc = subprocess.Popen(self.command,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
'''
Script for converting and storing video files.
'''
import argparse
import datetime
import exceptions
import os
import re
import shutil
import sys
_CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(_CURRENT_DIR, '..'))
import settings # pylint: disable=import-error, wrong-import-position
log = settings.get_logger('import_video') # pylint: disable=invalid-name
def copy(source, destination):
'''
Copies file. In case destination directory not exist it is created.
:param source: Source filename.
:param destination: Destination filename.
:return: None.
'''
destdir = os.path.dirname(destination)
if not os.path.exists(destdir):
os.makedirs(destdir, mode=0o755)
shutil.copy2(source, destination)
log.info('Copy from: \t' + source + ' to ' + destination)
# -*- coding: utf-8 -*-
import tensorflow as tf
from airflow.models import BaseOperator
from airflow.hooks.postgres_hook import PostgresHook
from airflow.utils.decorators import apply_defaults
from settings import Config, get_logger
from amzn_review.tasks.writer import TFRecordReviewDataWriter
from rating_model.models import get_tokenizer
from workflow.plugins.operators.commons import VariableMixin
from workflow.exceptions import (InvalidRecordsFoundError, NoRecordsFoundError)
logger = get_logger('workflow.plugins.operators.serialize')
class ReviewToTFRecordOperator(VariableMixin, BaseOperator):
ui_color = '#FFCC99'
@apply_defaults
def __init__(self, conn_id: str, dirpath: str, *args, **kwargs):
"""Write review data to TFRecord files.
Args:
conn_id: str
connection_id to PostgreSQL registered on Airflow
dirpath: str
path to directory to store tfrecord files
import os
import glob
import json
import shutil
from airflow.models import BaseOperator
from airflow.hooks.postgres_hook import PostgresHook
from airflow.utils.decorators import apply_defaults
from airflow.secrets import get_variable
from settings import get_logger
from workflow.helper import makedir, redis_session
from workflow.exceptions import (InvalidDataFormatError, ProcessFailedError)
logger = get_logger('workflow.plugins.operators.commons')
class VariableMixin(object):
def __init__(self, *args, **kwargs):
if 'variable_keys' in kwargs:
variable_keys = kwargs.pop('variable_keys')
self.load_variables(variable_keys)
super(VariableMixin, self).__init__(*args, **kwargs)
def load_variables(self, keys: str or List[str]):
"""Take variables from registered json formatted variables
and store them to operator's attributes.
If variable is empty or inappropriately formatted,
from rdflib import Graph, Literal, RDF, RDFS, URIRef
from namespaces import rq_prefixes
from namespaces import (
D,
VIVO,
WOS,
)
from lib import backend
from wos_categories import get_category_uri
from publications import waan_uri
import settings
logger = settings.get_logger()
def hash_uri(prefix, value):
return D[prefix + '-' + hashlib.md5(value).hexdigest()]
def local_name(uri):
return uri.split('/')[-1]
def get_unified_orgs():
q = rq_prefixes + """
select ?wosU ?org
where {
?wosU a wos:UnifiedOrganization ;
def test_get_logger():
"""Test get_logger()"""
test_logger = get_logger("test_logger")
assert type(test_logger) == logging.Logger
assert test_logger.name == "test_logger"
from conf import ServiceConfigure
from utils import merge_into, blipp_import, get_most_recent
from schema_cache import SchemaCache
import settings
from validation import validate_add_defaults
import pprint
# should be blipp_conf, but netlogger doesn't like that for some reason
logger = settings.get_logger('confblipp')
class BlippConfigure(ServiceConfigure):
def __init__(self, initial_config={}, node_id=None,
pre_existing_measurements="ignore", urn=None):
if "name" not in initial_config:
initial_config['name']="blipp"
self.pem = pre_existing_measurements
self.schema_cache = SchemaCache()
self.probe_defaults = None
self.measurements = []
super(BlippConfigure, self).__init__(initial_config, node_id, urn)
def initialize(self):
super(BlippConfigure, self).initialize()
if not self.service_setup:
logger.error("initialize", msg="Could not reach UNIS to initialize service")
exit(-1)
self.initial_measurements = self.unis.get("/measurements?service=" +
self.config["selfRef"])
self.initial_measurements = get_most_recent(self.initial_measurements)
# use any pre-existing measurements found in UNIS right away
from ms_client import MSInstance
from data_logger import DataLogger
from unis_client import UNISInstance
import settings
logger = settings.get_logger("collector")
class Collector:
"""Collects reported measurements and aggregates them for
sending to MS at appropriate intervals.
Also does a bunch of other stuff which should probably be handled by separate classes.
Creates all the metadata objects, and the measurement object in UNIS for all data inserted.
Depends directly on the MS and UNIS... output could be far more modular.
"""
def __init__(self, service, measurement):
self.config = measurement["configuration"]
self.service = service
self.measurement = measurement
self.collections_created = False
self.ms = MSInstance(service, measurement)
self.dl = DataLogger(service, measurement)
self.mids = {} # {subject1: {metric1:mid, metric2:mid}, subj2: {...}}
# {mid: [{"ts": ts, "value": val}, {"ts": ts, "value": val}]}
self.mid_to_data = {}
self.mid_to_et = {}
self.unis = UNISInstance(service)
self.num_collected = 0
from settings import settings, get_logger
logger = get_logger('objectivefunc')
# returns true if stop
def stop_condition(generation, stale_generations, fitness):
if settings["MAX_GENERATIONS"] < generation:
logger.debug("Max generations count achieved. Stopping.")
return True
if settings["MAX_STALE_GENERATIONS"] < stale_generations:
logger.debug("Max stale generations count achieved. Stopping.")
return True
if settings["TARGET_FITNESS"] > fitness: # minimizing fitness
logger.debug("Target fitness achieved. Stopping.")
return True
else:
return False
from ms_client import MSInstance
from data_logger import DataLogger
from unis_client import UNISInstance
import settings
logger = settings.get_logger('collector')
class Collector:
"""Collects reported measurements and aggregates them for
sending to MS at appropriate intervals.
Also does a bunch of other stuff which should probably be handled by separate classes.
Creates all the metadata objects, and the measurement object in UNIS for all data inserted.
Depends directly on the MS and UNIS... output could be far more modular.
"""
def __init__(self, service, measurement):
self.config = measurement["configuration"]
self.service = service
self.measurement = measurement
self.collections_created = False
self.ms = MSInstance(service, measurement)
self.dl = DataLogger(service, measurement)
self.mids = {} # {subject1: {metric1:mid, metric2:mid}, subj2: {...}}
# {mid: [{"ts": ts, "value": val}, {"ts": ts, "value": val}]}
self.mid_to_data = {}
self.mid_to_et = {}
self.unis = UNISInstance(service)
self.num_collected = 0
def insert(self, data, ts):
'''
def __init__(self, view=View, *args, **kwargs):
super(EmbeddedModel, self).__init__(*args, **kwargs)
self.logger = settings.get_logger(settings.LOG, self.__class__.__name__)
self.__logs__ = []
self.__view__ = view(self)
self.init(*args, **kwargs)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Handle connections in Airflow."""
from dataclasses import dataclass
from settings import get_logger, Config
logger = get_logger('workflow.conn')
@dataclass
class ConnectionConfig(object):
conn_id: str
host: str
login: str
password: str
schema: str
port: int
class AirflowConnection:
def __init__(self):
self.Connection = None
self.create_session = None
def _set_airflow_connection(self):
from airflow.models import Connection
from airflow.utils.db import create_session
self.Connection = Connection
import requests
import settings
import json
import logging
import os
import datetime
from time import sleep
logger = settings.get_logger(os.path.realpath(__file__))
def run_collect(company, total_req):
logger.info(company + " started")
# files and vars
# today = datetime.date.today()
# yesterday = today - datetime.timedelta(1)
# two_days_ago = today - datetime.timedelta(2)
# file_name = settings.DOWNLOADS_STWITS + "/" + company + "/stwits-" + company + "-" + str(yesterday) + ".csv"
# dir = os.path.dirname(os.path.realpath(file_name))
# os.makedirs(dir, exist_ok=True)
# output = open(file_name, "a")
max_id = '999999999'
previous_date = datetime.date.today()
for j in range(0, 100000):
total_req += 1
# if total_req == 200:
# sleep(3600)
# Unix lockfiles
import os, os.path, time, warnings, errno
import settings
logger = settings.get_logger('pnntprss.lockfile')
# we use tempnam safely.
warnings.filterwarnings('ignore', 'tempnam', RuntimeWarning, 'lockfile')
class LockFileStateError(Exception):
"""An Exception indicating that the lock was not in the appropriate state"""
pass
class LockFile:
"""An object representing a Unix lockfile."""
def __init__(self, path, expiry_time=30 * 60):
self.locked = False
self.path = path
self.expiry_time = expiry_time
(self.dir, self.prefix) = os.path.split(path)
if self.prefix[0] != '.':
self.prefix = '.' + self.prefix
def trylock(self, directory_must_exist=False):
"""Try to acquire the lock."""
if self.locked:
raise LockFileStateError("already holding lock file: %s" % self.path)
from unis_client import UNISInstance
import settings
import pprint
from utils import merge_dicts
logger = settings.get_logger('conf')
class ServiceConfigure(object):
'''
ServiceConfigure is meant to be a generic class for any service
which registers itself to, and gets configuration from UNIS. It
was originally developed for BLiPP, but BLiPP specific features
should be in the BlippConfigure class which extends
ServiceConfigure.
'''
def __init__(self, initial_config={}, node_id=None, urn=None):
if not node_id:
node_id = settings.UNIS_ID
self.node_id = node_id
self.urn = urn
self.config = initial_config
self.unis = UNISInstance(self.config)
self.service_setup = False
def initialize(self):
self._setup_node(self.node_id)
self._setup_service()
def refresh(self):
r = self.unis.get("/services/" + self.config["id"])
import settings
import time
import socket
from collector import Collector
from utils import blipp_import
import pprint
HOSTNAME = socket.gethostname()
logger = settings.get_logger('probe_runner')
rtt_list = []
collect_rtt_list = []
class ProbeRunner:
'''
Class to handle a single probe. Creates the scheduler, collector,
and probe module associated with this probe. The run method should
be called as a subprocess (by arbiter) and passed a
connection object so that it can receive a "stop" if necessary.
'''
def __init__(self, service, measurement):
self.measurement = measurement
self.config = measurement['configuration']
self.service = service.config
self.probe_defaults = service.probe_defaults
self.setup()
def run(self, conn, ip):
if conn.poll() and conn.recv() == "stop":
self._cleanup()
exit()
import itertools
import subprocess
import argcomplete
from amzn_review.cli import get_parser
from amzn_review.aws import (
create_redshift_cluster,
redshift_role_creation,
show_cluster_info,
)
from settings import Config, get_logger
logger = get_logger('amzn_review.main')
def _handle_app(args):
if args.type == 'config':
if args.subtype == 'show':
print('\nMain App Configuration')
print('----------------------\n')
Config.print_all()
def _handle_db(args):
# TODO
warnings.warn('Main database on local. This is not implemented yet')
# Copyright (c) 2013-2016, Trustees of Indiana University,
# All rights reserved.
#
# This software may be modified and distributed under the terms of the BSD
# license. See the COPYING file for details.
#
# This software was created at the Indiana University Center for Research in
# Extreme Scale Technologies (CREST).
# =============================================================================
import settings
import os
import errno
import psutil
from unis_client import UNISInstance
logger = settings.get_logger("registration_probe")
class Probe:
def __init__(self, service, measurement):
self.config = measurement["configuration"]
self.service = service
self.unis = UNISInstance(service)
self.pname = None
self.pidfile = None
self.id = None
try:
self.serviceType = self.config["service_type"]
except Exception:
logger.error("__init__", msg="Must specify service_type!")
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import json
from typing import List, Tuple
import numpy as np
import tensorflow as tf
from settings import get_logger
logger = get_logger('amzn_review.tasks.writer')
def _bytes_feature(value): # pragma: no cover
"""Convert a value to byte type which is compatible with tf.Example."""
if isinstance(value, type(tf.constant(0))):
value = value.numpy()
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def _float_feature(value): # pragma: no cover
"""Convert a value to float type which is compatible with tf.Example."""
return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
class TFRecordReviewDataWriter(object):
"""Write TFRecord file from review dataset.
Example usage:
# periscope-ps (blipp)
#
# Copyright (c) 2013-2016, Trustees of Indiana University,
# All rights reserved.
#
# This software may be modified and distributed under the terms of the BSD
# license. See the COPYING file for details.
#
# This software was created at the Indiana University Center for Research in
# Extreme Scale Technologies (CREST).
# =============================================================================
import settings
from utils import get_most_recent
from periscope_client import PeriscopeClient
logger = settings.get_logger('unis_client')
class UNISInstance:
def __init__(self, service):
self.service = service
self.config = service["properties"]["configurations"]
unis_url=self.config["unis_url"]
if unis_url and unis_url[-1]=="/":
unis_url = unis_url[:-1]
self.pc = PeriscopeClient(service, unis_url)
self.meas_to_mds = {}
def post(self, url, data={}):
return self.pc.do_req('post', url, data)
def get(self, url, data=None):
import csv
from collections import OrderedDict
import gevent
from itertools import chain, combinations, ifilterfalse, permutations
import os.path
import re
import string
import sys
from xml.dom import minidom
from elasticsearch import TransportError
from elasticsearch.helpers import bulk, scan
from connections import current_customer, redis_connection, search_connection
from settings import live_config, get_logger
log = get_logger(__name__)
rds = redis_connection
es = search_connection
# Blacklist - one time
# filter(lambda x: x not in ['lower', 'injury', 'congenital', 'disease',
# 'degree', 'right', 'left', 'nec'],
# dict(Counter(e for rec in rds.zrange('icd', 0, -1)
# for e in rec.partition('::')[0].strip().split())
# .most_common(20)).keys())
BLACK_LIST = ['and', 'type', 'of', 'in', 'unspecified', 'due', 'by', 'to',
'other', 'specified', 'with', 'or', 'than']
filtered = lambda words: ifilterfalse(lambda x: x in BLACK_LIST, words)
# pridame lokalni knihovny
sys.path.append(os.path.join(os.path.split(os.path.dirname(os.path.abspath(__file__)))[0],'lib','python2.7'))
sys.path.append(os.path.join(os.path.split(os.path.dirname(os.path.abspath(__file__)))[0],'etc'))
if 'XML_catalog_files' in os.environ:
os.environ['XML_CATALOG_FILES'] = os.environ['XML_CATALOG_FILES'] + " " + os.path.join(os.path.split(os.path.dirname(os.path.abspath(__file__)))[0],'lib','schema','catalog.xml')
else:
os.environ['XML_CATALOG_FILES'] = os.path.join(os.path.split(os.path.dirname(os.path.abspath(__file__)))[0],'lib','schema','catalog.xml')
from directories import WorkDir
from psp import PSP
from validator import Validator
from settings import workdir, get_logger, get_file_log_handler
import logging
# http://www.cafeconleche.org/books/effectivexml/chapters/47.html
logger = get_logger()
parser = argparse.ArgumentParser(description="""Program validuje PSP balicky.
Umi validovat na trech urovnich:
- METS soubor
- hodnoty z ciselniku existuji
- linky, na ktere se v balicku odkazuje existuji
Kazda z techto voleb se da vypnout.
Program rozbali zadany/zadane PSP balicek/balicky do adresare %s
Pokud je zadany adresar, tak ho bere jako rozbaleny PSP balicek
a pokusi se je zkontrolovat.
Pokud se pri kontrole souboru v adresari amdSec, ... objevi chyba,
import warnings
import requests
import sys
from ibm_watson import AssistantV2, ApiException
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
from flask import Flask, request, Response
import cache
from classes import EventType, SlackEvent
import settings
import sessions
import action_handler
import traceback
# Configure Logger
LOGGER = settings.get_logger("main")
# Initialize flask
APP = Flask(__name__)
BOT_ID = settings.BOT_ID
WA = {}
if not settings.CALL_PROXY:
authenticator = IAMAuthenticator(settings.WA_IAM_KEY)
WA = AssistantV2(version=settings.WA_VERSION, authenticator=authenticator)
if settings.WA_OPT_OUT:
WA.set_default_headers({'x-watson-learning-opt-out': "true"})
THREADS = {}
from sndlibparser import demand_array
from settings import settings, get_logger
import random
logger = get_logger('mutation')
def mutation(chromosome):
ch = chromosome
if settings["MUTATION_FREQUENCY"] == -1:
return single_mutation(ch)
for _ in range(settings["MUTATION_FREQUENCY"]):
ch = single_mutation(ch)
return ch
def single_mutation(chromosome):
MUTATION_PROB = settings["MUTATION_PROB"]
p = random.uniform(0, 1)
# mutate only with MUTATION_PROB
if p < MUTATION_PROB:
logger.debug(f'Mutating! Mutation prob is {MUTATION_PROB}')
# choose one gene for mutation
index = random.randrange(len(chromosome))
logger.debug(f"Random index: {index}")
chosen_gene = chromosome[index] # for debug
logger.debug(f"Chosen gene: {chosen_gene}")
# mutate chosen gene
if settings["DISTRIBUTED"]:
chromosome[index] = random.uniform(0, 1)
import glob
import gzip
import json
import random
import shutil
import time
import threading
from typing import Callable, Coroutine, List
from botocore.exceptions import ClientError
from settings import Config, get_logger
from amzn_review.aws import AWS
from amzn_review.data.gen import MetadataGenerator
logger = get_logger('amzn_review.data.uploader')
random.seed(1234)
@dataclass(frozen=True)
class FileItem:
category: str
filepath: str
class MetadataUploader(object):
def __init__(self, categories: List[str], bucket: str):
"""Server to send review data.
Args:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
from collections import namedtuple
from functools import partialmethod
import json
import time
import boto3
import botocore
from botocore.exceptions import ClientError
from settings import Config, get_logger
logger = get_logger(__name__, level='DEBUG')
Role = namedtuple('Role', 'client,role,arn')
class AWS(object):
"""AWS client factory.
Used variable set in configuration
(Config object and/or `.env` file)
Example usage:
>>> client = AWS.get_client('s3')
"""
_client = partialmethod(boto3.client,
# -*- coding: utf-8 -*-
import os
from contextlib import contextmanager
from datetime import datetime
import json
import gzip
from typing import List, Tuple
import redis
from settings import Config, get_logger
from workflow.exceptions import BaseOperatorError
logger = get_logger('workflow.helper')
def makedir(path):
if not os.path.isdir(path):
os.makedirs(path)
@contextmanager
def redis_session():
# TODO: store connections to pool and reuse them
r = redis.Redis(host=Config.REDIS['HOST'],
port=Config.REDIS['PORT'],
db=Config.REDIS['DB'])
try:
yield r
#
# This software may be modified and distributed under the terms of the BSD
# license. See the COPYING file for details.
#
# This software was created at the Indiana University Center for Research in
# Extreme Scale Technologies (CREST).
# =============================================================================
import time
import settings
from probe_runner import ProbeRunner
from multiprocessing import Process, Pipe
from copy import copy
from config_server import ConfigServer
import pprint
logger = settings.get_logger('arbiter')
PROBE_GRACE_PERIOD = 10
class Arbiter():
'''
The arbiter handles all of the Probes. It reloads the config every
time the unis poll interval comes around. If new probes have been
defined or if old probes removed, it starts and stops existing
probes.
Each probe is run by the ProbeRunner class in a separate
subprocess. The arbiter has a dictionary which contains the
process object for each probe, and the connection object to that
process. The arbiter can send a "stop" string through the
connection to give a probe the chance to shut down gracefully. If
import xml.etree.ElementTree as ET
import json
from settings import get_logger, CONFIG_DIR
import os
logger = get_logger('sndlibparser')
def extract_indexes(text):
k1 = text.find('_')
k2 = text.find('_', k1 + 1)
first = int(text[k1 + 1:k2])
last = int(text[k2 + 1:])
return (first, last)
def extract_indexes_from_text(nodes, element):
source_index = nodes.index(element.find('source').text)
target_index = nodes.index(element.find('target').text)
first = min(source_index, target_index)
last = max(source_index, target_index)
return (first, last)
def create_polska_json():
XML_NETWORK_PATH = '../dane/polska.xml'
tree = ET.parse(XML_NETWORK_PATH)
root = tree.getroot()
nodes = [n.get('id') for n in root.iter('node')]
#!/usr/bin/python
# Go through all the groups, expiring articles that have exceeded
# their lifetime.
import os, time
import settings, group
logger = settings.get_logger('pnntprss.expire')
def expire(g):
"""Expire articles in the given group."""
if not g.lockfile.trylock():
# we are already updating, expiring, or otherwise messing with
# this group. No problem, we'll try again next time round.
return
try:
lifetime = g.config.get('article_lifetime', settings.article_lifetime)
if lifetime:
now = time.time()
to_remove = set()
for art in g.article_numbers():
stat = os.stat(g.article_file(art))
if now - stat.st_mtime > lifetime:
to_remove.add(art)
if to_remove:
import settings
import shlex
import dateutil.parser
import calendar
import re
logger = settings.get_logger("netlogger_probe")
class Probe:
def __init__(self, service, measurement):
self.config = measurement["configuration"]
self.logfile = None
try:
self.logfile = open(self.config["logfile"])
except KeyError:
logger.warn("__init__", msg="Config does not specify logfile!")
except IOError:
logger.warn("__init__", msg="Could not open logfile: %s" % self.config["logfile"])
self.app_name = self.config.get("appname", "")
self.et_string = "ps:tools:blipp:netlogger:"
if self.app_name:
self.et_string += self.app_name + ":"
def get_data(self):
if self.logfile:
ret = []
self.logfile.seek(self.logfile.tell())
for line in self.logfile:
if "VAL" not in line:
from blipp_conf import BlippConfigure
import settings
import arbiter
from utils import merge_dicts, delete_nones
FIELD_LEN = 10
VERSION = "0.0.1"
REQUEST_TYPE = "1"
MESSAGE_LEN = 512
MAX_HOST_LEN = 256
SUCCESS = 1
FAILURE = 2
HOSTNAME = socket.gethostname()
logger = settings.get_logger('ablippd')
iplist = []
class LboneServer:
def __init__(self, host, port):
self.host = host
self.port = int(port)
def GetDepot(self, numDepots, hard, soft, duration, location, timeout):
str_final_req = []
str_final_req.append(self.PadField(VERSION))
str_final_req.append(self.PadField(REQUEST_TYPE))
str_final_req.append(self.PadField(str(numDepots)))
str_final_req.append(self.PadField(str(hard)))
# Copyright (c) 2013-2016, Trustees of Indiana University,
# All rights reserved.
#
# This software may be modified and distributed under the terms of the BSD
# license. See the COPYING file for details.
#
# This software was created at the Indiana University Center for Research in
# Extreme Scale Technologies (CREST).
# =============================================================================
import http
import json
import settings
import re
from requests.exceptions import ConnectionError
logger = settings.get_logger('periscope_client')
class PeriscopeClient:
def __init__(self, service_entry, url):
self.service_entry = service_entry
self.config = service_entry["properties"]["configurations"]
self.url = url
def do_req(self, rtype, url, data=None, headers=None):
config = self.config
url, schema, dheaders = self._url_schema_headers(url)
headers = dheaders if not headers else headers
if isinstance(data, dict):
if schema:
data.setdefault('$schema', schema)
if rtype=='post' or rtype=='put':
#!/usr/bin/env python
'''
Creates image with size settings.THUMBNAIL_WIDTH, settings.THUMBNAIL_HEIGHT for
each video file.
'''
import os
import shutil
import subprocess
import sys
_CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(_CURRENT_DIR, '..'))
import settings # pylint: disable=import-error, wrong-import-position
log = settings.get_logger('create_video_thumbnails') # pylint: disable=invalid-name
def check_ffmpeg():
'''
Checks that ffmpeg is available.
:return: True if ffmpeg has been found, False otherwise.
'''
try:
subprocess.check_call(['ffmpeg', '-version'])
except subprocess.CalledProcessError:
return False
return True
def get_video_files_list(path):
from routes import setup_routes
from settings import get_config, get_logger
if __name__ == '__main__':
app = web.Application()
parser = argparse.ArgumentParser(description='Process arguments.')
parser.add_argument('--config_file',
dest='config_file',
default='./config/local.yaml',
help='config file path')
path = parser.parse_args().config_file
app['config'] = get_config(path)
app['logger'] = get_logger()
setup_routes(app)
aiohttp_jinja2.setup(app,
loader=jinja2.FileSystemLoader(
os.path.abspath('templates')))
logging.basicConfig(level=logging.INFO)
logging.info('Web server started on port %s' % app['config']['port'])
logging.info('Config file: %s' % path)
web.run_app(
app,
host=app['config']['host'],
port=app['config']['port'],
#
# Copyright (c) 2013-2016, Trustees of Indiana University,
# All rights reserved.
#
# This software may be modified and distributed under the terms of the BSD
# license. See the COPYING file for details.
#
# This software was created at the Indiana University Center for Research in
# Extreme Scale Technologies (CREST).
# =============================================================================
import requests
import settings
import pprint
import json
logger = settings.get_logger('http')
def handle(resp):
logger.debug('handle', status=resp.status_code, text=resp.text)
if 200 <= resp.status_code <= 299:
if resp.text:
return resp.text
else:
return resp.status_code
elif 400 <= resp.status_code <= 499:
return None
else:
raise Blipp_HTTP_Error(resp.status_code, resp.text)
def make_request(rtype, rurl, headers, data,
ssl_cert=None, ssl_key=None, ssl_cafile=None):
import sys
import time
from unis_client import UNISInstance
from collector import Collector
from multiprocessing import Process, Pipe
from utils import *
import settings
from pprint import pprint
logger = settings.get_logger('sched_obj')
class SchedObj:
def __init__(self, probe_name, settings_name="settings"):
self.probe_name = probe_name
self.probe_settings=try__import__(probe_name+"_settings")
self.settings=try__import__(settings_name)
self.probe_module=try__import__(probe_name)
self.mids = {}
self.collector = None
self.unis = None
self.probe = None
self.start_time = None
self.end_time = None
self.c_iteration = 0 # which collection are we on
self.r_iteration = 1
def run(self, conn):
self.read_settings()
if self.probe==None:
return
# main collection loop
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import sys
from datetime import datetime
from settings import Config, get_logger
from rating_model.data import ReviewDataset
from rating_model.train import Trainer
from workflow.helper import makedir, redis_session
logger = get_logger('workflow.dags.run_model')
def main():
dttm = datetime.now().strftime('%Y%m%d_%H00')
train = os.scandir(os.path.join(Config.DATA_DIR, 'tfrecord'))
test = os.scandir(os.path.join(Config.DATA_DIR, 'test', 'tfrecord'))
train = [
f.path for f in train if f.is_file() and f.name.endswith('.tfrecord')
]
test = [
f.path for f in test if f.is_file() and f.name.endswith('.tfrecord')
]
logger.info(f'Train files: {len(train)} Test files: {len(test)}')
# =============================================================================
# periscope-ps (blipp)
#
# Copyright (c) 2013-2016, Trustees of Indiana University,
# All rights reserved.
#
# This software may be modified and distributed under the terms of the BSD
# license. See the COPYING file for details.
#
# This software was created at the Indiana University Center for Research in
# Extreme Scale Technologies (CREST).
# =============================================================================
import settings
logger = settings.get_logger('data_logger')
class DataLogger:
def __init__(self, service_entry, measurement):
self.service_entry = service_entry
self.sconfig = service_entry["properties"]["configurations"]
self.mconfig = measurement["configuration"]
self.filepath=self.mconfig.get("data_file", None)
if self.filepath:
try:
self.datafile = open(self.filepath, "a")
except IOError:
logger.warn("__init__", msg="Could not open datafile: %s" % self.filepath)
def write_data(self, data, mid_to_et=None):
if not self.filepath:
return None
from settings import settings, get_logger
import sys
import random
logger = get_logger('select')
# (default binary) tournament selection without replacement
def select(population, k):
population_fitnesses = [fitness for _, fitness in population]
logger.debug(f"all population fitnesses: {population_fitnesses}")
ret_list = []
TOURNAMENT_COMPETITION_COUNT = settings["TOURNAMENT_COMPETITION_COUNT"]
logger.debug(f"competition size: {TOURNAMENT_COMPETITION_COUNT}")
for _ in range(k):
# COMPETITION (SZRANKI)
competition_chromosomes = random.sample(population, TOURNAMENT_COMPETITION_COUNT)
competition_fitnesses = [fitness for _, fitness in competition_chromosomes] # only for debug TODO remove later
logger.debug(f"competition chromosomes {competition_fitnesses}")
best_chromosome = None
best_fitness = sys.maxsize
for chromosome, fitness in competition_chromosomes:
if fitness < best_fitness:
best_chromosome = chromosome
best_fitness = fitness
logger.debug(f"winning fitness {best_fitness}")
if not best_chromosome:
logger.error('Chromosome in tournament selection not found')
sys.exit(1)
ret_list.append(best_chromosome)
"""
Tracks, stores and maintains WA sessions and the local session cache
"""
import datetime
import traceback
import settings
import sys
LOGGER = settings.get_logger("sessions")
SESSIONS = {}
def check_expired(session):
"""Checks to see if a session time is passed the allotted timeout"""
elapsed_time = datetime.datetime.now() - session[1]
LOGGER.debug("Session Expired == " +
str(elapsed_time.seconds >= settings.SESSION_TIMEOUT))
return elapsed_time.seconds >= settings.SESSION_TIMEOUT
def new_session_for_user(slack_user, watson_assistant):
"""Creates a new WA session for a user"""
SESSIONS[slack_user] = create_wa_session(watson_assistant)
return SESSIONS[slack_user]
def get_wa_session(slack_user, watson_assistant, create_if_needed=True):
"""Gets a session for a user or creates one if nonexistent"""
import logging
import random
import logging
import sys
from sndlibparser import demand_array, link_keys
from settings import settings, get_logger
logger = get_logger('fitness')
MODULARITY = settings["MODULARITY"]
logger.debug('modularity is {}'.format(MODULARITY))
def ceildiv(a, b):
return -(-a // b)
def calc_fitness_aggregated(chromosome):
gene_i = 0
edges_loads = [0] * len(link_keys)
for demand_data in demand_array:
path_index = chromosome[gene_i]
path = demand_data['admissiblePaths'][path_index]
for edge in path:
link_index = link_keys.index(edge)
if edge != link_keys[link_index]:
logger.warning('Desired link index does not apply to real')
edges_loads[link_index] += demand_data['demand']
from fitness import *
import random
from settings import settings, get_logger
logger = get_logger('init')
def createInitPopulationDistributed(mi):
population = []
for _ in range(mi):
chromosome = []
for demand_data in demand_array:
paths_no = len(demand_data['admissiblePaths'])
for index in range(paths_no):
chromosome.append(random.uniform(0, 1))
fitness = calc_fitness_distributed(chromosome)
population.append((chromosome, fitness))
return population
def createInitPopulationAggregated(mi):
population = []
for _ in range(mi):
chromosome = [
random.randrange(len(demand_data['admissiblePaths']))
for demand_data in demand_array
]
fitness = calc_fitness_aggregated(chromosome)
population.append((chromosome, fitness))
import random
import logging
from settings import settings, get_logger
logger = get_logger('crossover')
def crossover(elements): # list of 2 elements
crossover_points_count = settings["CROSSOVER_POINTS_COUNT"]
# logger.debug(f"Parent0: {elements[0]}") # checked
# logger.debug(f"Parent1: {elements[1]}") # checked
crossover_points = set()
while len(crossover_points) != crossover_points_count:
crossover_points.add(random.randrange(len(elements[0])))
crossover_points = sorted(crossover_points)
logger.debug(
f"Chosen crossover points: {crossover_points}, len(elements[0]): {len(elements[0])}"
)
crossover_points.append(len(elements[0]))
offspring = []
start = 0
parent_no = 0
for point in crossover_points:
offspring += elements[parent_no][start:point]
parent_no = (parent_no + 1) % 2
start = point
# logger.debug(f"Offspring: {offspring}") # checked
return offspring
import calendar
import datetime
import logging
from optparse import OptionParser
from pprint import pprint
import time
from twython import Twython
import settings
NUM_RETRIES = 3
logger = settings.get_logger(__name__)
class TwitterLibException(Exception):
pass
class TwitterLib(object):
# Default for unauth'd req's
# TODO: auth'd requests
total_requests_per_hour = 150
def __init__(self):
self._setup_twython()
self._check_and_reset_rate_limits()
def _setup_twython(self):
self.client = Twython(
import settings
from periscope_client import PeriscopeClient
logger = settings.get_logger("unis_client")
class UNISInstance:
def __init__(self, service):
self.service = service
self.config = service["properties"]["configurations"]
unis_url = self.config["unis_url"]
if unis_url and unis_url[-1] == "/":
unis_url = unis_url[:-1]
self.pc = PeriscopeClient(service, unis_url)
self.meas_to_mds = {}
def post(self, url, data={}):
return self.pc.do_req("post", url, data)
def get(self, url, data=None):
return self.pc.do_req("get", url, data)
def delete(self, url, data=None):
return self.pc.do_req("delete", url, data)
def put(self, url, data=None):
if "ts" in data:
del data["ts"]
return self.pc.do_req("put", url, data)
def find_or_create_metadata(self, subject, metric, measurement):
