def main():
"""Main function
"""
args_parser = argparse.ArgumentParser()
args_parser.add_argument("cfg",
type=argparse.FileType("rb"),
help="configuration file")
args_parser.add_argument("target", type=str, help="target folder")
args_parser.add_argument("-v",
"--verbosity",
action="count",
help="be more verbose",
default=0)
args_parser.add_argument("-e",
"--erase",
action="store_true",
help="erase target folder")
args_parser.add_argument("-q",
"--quiet",
action="store_true",
help="no user interaction")
args = args_parser.parse_args()
# Setup logging system and show necessary messages
setup_logging(logging.INFO if args.verbosity == 0 else logging.DEBUG,
module_name=True if args.verbosity > 1 else False)
INFO("Information logging turned on")
DEBUG("Debug logging turned on")
# Parsing config file
DEBUG("Parsing config file")
config = json_load(args.cfg, critical=True)
INFO("Config file parsed successfully")
config["target"] = {
"path": args.target,
"erase": args.erase,
"quiet": args.quiet,
}
# Main process starting
make(config)
INFO("")
INFO("+" * 70)
INFO("\nPath to application:\n{}".format(config["target"]["path"]))
def make_lookout_csv(state, debug='INFO'):
"""Desc.
:param 1: asdf.
:return: asdf.
"""
helpers.setup_logging(level=debug)
logger.info('STATE IS: {}'.format(state))
lookout_urls = get_lookout_urls(state)
logger.debug('LOOKOUT URLS: {}'.format(lookout_urls))
lookouts = make_lookouts(state, lookout_urls)
write_csv(state, lookouts)
def main():
"""
"""
#--setup: parse commandline arguments---------------------------------------
args = setup_parse_args()
#--setup: logging and data output-------------------------------------------
logger = setup_logging(
path_log=args.path_log + "velocity_data_log",
log_level=args.log_level,
)
setup_output_path(args.path_data_output)
#--setup: main objects for using BlockSci-----------------------------------
Velo.setup(
logger=logger,
args=args,
)
Multiprocess.setup(logger=logger, args=args)
#--Retrieval of basic blockchain data, money supply and velocity measures---
results_raw = Multiprocess.run()
#--get csv of final pandas data frame---------------------------------------
Velo.get_results_finalized(
results_raw=results_raw,
index_label="date",
)
print("Exiting program")
exit(0)
def etl_job():
data = json.load(open('/home/ubuntu/ti_etl/secrets.json'))
logger = helpers.setup_logging()
s3_client = boto3.client(
's3',
aws_access_key_id=data['aws_access_key_id'],
aws_secret_access_key=data['aws_secret_access_key'])
s3_resource = boto3.resource(
's3',
aws_access_key_id=data['aws_access_key_id'],
aws_secret_access_key=data['aws_secret_access_key'])
keys = []
resp = s3_client.list_objects_v2(Bucket='dealer-churn-analysis')
for obj in resp['Contents']:
keys.append(obj['Key'])
for key in keys:
names = key.split("/")
obj = s3_resource.Bucket('dealer-churn-analysis').Object(
helpers.zip_file_name())
file_name = 'praxis/etl/logs/log_{file}.txt'.format(file=helpers.date())
obj_log = s3_resource.Bucket('dealer-churn-analysis').Object(file_name)
buffer = io.BytesIO(obj.get()["Body"].read())
zip_file = zipfile.ZipFile(buffer, 'r')
logger.info("Name of csv in zip file :%s", zip_file.namelist())
logs = ""
dataframe = pd.DataFrame()
for name_of_zipfile in zip_file.namelist():
zip_open = pd.read_csv(zip_file.open(name_of_zipfile))
dataframe['created_at'] = pd.Series([datetime.datetime.now()] *
len(zip_open))
dataframe['last_updated_at'] = pd.Series([datetime.datetime.now()] *
len(zip_open))
zip_open = pd.concat([dataframe, zip_open], axis=1)
zip_open = zip_open.dropna()
table_name = "{name}_table".format(
name=name_of_zipfile.replace('.csv', ''))
#print (zip_open)
try:
zip_open.to_sql(name=name_of_zipfile.replace('.csv', ''),
con=database.db_connection(),
if_exists='append',
index=False)
except SQLAlchemyError as sqlalchemy_error:
print(sqlalchemy_error)
logs = '\n{table_name}\n{error}\n{logs}'.format(
logs=logs, error=sqlalchemy_error, table_name=table_name)
logger.error(" %s", sqlalchemy_error)
database.db_connection().execute('SET FOREIGN_KEY_CHECKS=1;')
end_time = datetime.datetime.now()
logger.info("End time of program : %s", end_time)
logs = '{logs} \nstart_time : {start_time} \nend_time : {end_time}'.format(
start_time=helpers.start_time(), logs=logs, end_time=end_time)
print(logs)
obj_log.put(Body=logs)
def main():
"""Main function
"""
args_parser = argparse.ArgumentParser()
args_parser.add_argument("source", type=str,
help="aplication source folder")
args_parser.add_argument("target", type=str,
help="target XML file")
args_parser.add_argument("-v", "--verbosity", action="count",
help="be more verbose",
default=0)
args = args_parser.parse_args()
# Setup logging system and show necessary messages
setup_logging(logging.INFO if args.verbosity == 0 else logging.DEBUG,
module_name=True if args.verbosity > 1 else False)
INFO("")
INFO("Information logging turned on")
DEBUG("Debug logging turned on")
INFO("")
INFO(BLOCK_END)
INFO("")
config = {
"target": {
"path": args.target,
},
"source": args.source
}
# Main process starting
build(config)
INFO("\nPath to application XML:\n{}".format(config["target"]["path"]))
def process_args(argv):
try:
opts, args = getopt.getopt(argv, "gh", ["grade", "help"])
except getopt.GetoptError:
usage()
sys.exit(1)
extended_logging = False
for opt, arg in opts:
if opt in ("-g", "--grade"):
extended_logging = True
elif opt in ("-h", "--help"):
usage()
sys.exit()
filename = "".join(args)
if filename == "":
filename = "output.log"
if extended_logging:
print "Verbose logging is turned on; outputting to {0}".format(filename)
setup_logging(filename, grading=True)
else:
setup_logging(filename, grading=False)
def db_connection():
data = json.load(open('/home/ubuntu/ti_etl/secrets.json'))
Base = declarative_base()
logger = helpers.setup_logging()
DB_URI = "mysql+pymysql://{user}:{password}@{host}:{port}/{db}"
try:
engine = create_engine(DB_URI.format(user=data['user'],
password=data['password'],
port=data['port'],
host=data['host'],
db=data['db']),
echo=True)
engine.execute('SET FOREIGN_KEY_CHECKS=0;')
except SQLAlchemyError as sqlalchemy_error:
logger.error("%s", sqlalchemy_error)
sys.exit()
return engine
#!/usr/bin/python
#
# Decrypts encrypted files for the given environment.
#
import helpers
import traceback
import argparse
# Check parameters
parser = argparse.ArgumentParser(description='Decrypt environment files')
parser.add_argument('env_name', type=str, help='Name of the environment')
args = parser.parse_args()
helpers.logger = helpers.setup_logging('decrypt_env.log')
helpers.log('Decrypting environment: ' + args.env_name, as_banner=True, bold=True)
try:
helpers.decrypt_env(args.env_name)
except Exception, e:
helpers.logger.debug('Failure: ' + str(e) + ", exception: " + traceback.format_exc().replace('\n', ' | '))
raise
def create_tables():
logger = helpers.setup_logging()
logger.info("Start time of program :%s", helpers.start_time())
db = database.db_connection()
meta = MetaData(bind=db)
logger.info("INFO: Creating tables schema")
try:
table_realm_type = Table(
'realm_type', meta,
Column('id', Integer, primary_key=True, autoincrement=True),
Column('name', TEXT(50), nullable=True),
Column('category', TEXT(50), nullable=True),
Column('is_active', BOOLEAN, nullable=True),
Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now),
Column('last_updated_at',
TIMESTAMP,
onupdate=datetime.datetime.now))
table_category = Table(
'category', meta,
Column('id', Integer, primary_key=True, autoincrement=True),
Column('name', TEXT(50), nullable=True),
Column('is_active', BOOLEAN, nullable=True),
Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now),
Column('last_updated_at',
TIMESTAMP,
onupdate=datetime.datetime.now))
table_dealer_master = Table(
'dealer_master', meta,
Column('dealer_id', Integer, primary_key=True,
autoincrement=False),
Column('dealer_name', TEXT(255), nullable=True),
Column('realm_name', TEXT(50), nullable=True),
Column('customer_id', Integer, nullable=True),
Column('customer_name', TEXT(255), nullable=True),
Column('category_name', TEXT(50), nullable=True),
Column('city', TEXT(255), nullable=True),
Column('state_code', TEXT(10), nullable=False),
Column('zip_code', Integer, nullable=True),
Column('country_code', TEXT(10), nullable=False),
Column('area_code', Integer, nullable=True),
Column('start_date', TIMESTAMP, nullable=True),
Column('expire_date', TIMESTAMP, nullable=True),
Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now),
Column('last_updated_at',
TIMESTAMP,
onupdate=datetime.datetime.now))
table_ads_data = Table(
'ads_data', meta,
Column('ad_id', Integer, primary_key=True, autoincrement=False),
Column('dealer_id', Integer,
ForeignKey('dealer_master.dealer_id')),
Column('make_id', Integer, nullable=True),
Column('make_name', TEXT, nullable=True),
Column('ad_status', TEXT, nullable=True),
Column('created_at', TIMESTAMP, nullable=True),
Column('last_updated_at', TIMESTAMP, nullable=True),
Column('expire_date', TIMESTAMP, nullable=True),
Column('create_date', TIMESTAMP, nullable=True),
Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now),
Column('last_updated_at',
TIMESTAMP,
onupdate=datetime.datetime.now))
table_rep_master = Table(
'rep_master', meta,
Column('rep_id', Integer, primary_key=True, autoincrement=False),
Column('rep_name', TEXT(255), nullable=True),
Column('is_active', BOOLEAN, nullable=True),
Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now),
Column('last_updated_at',
TIMESTAMP,
onupdate=datetime.datetime.now))
table_ManagerMaster = Table(
'manager_master', meta,
Column('manager_id',
Integer,
primary_key=True,
autoincrement=False),
Column('manager_name', TEXT(255), nullable=True),
Column('is_active', BOOLEAN, nullable=True),
Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now),
Column('last_updated_at',
TIMESTAMP,
onupdate=datetime.datetime.now))
table_DealerRepManagerMapping = Table(
'dealer_rep_manager_mapping', meta,
Column('id', Integer, primary_key=True, autoincrement=True),
Column('dealer_id', Integer,
ForeignKey('dealer_master.dealer_id')),
Column('rep_id', Integer, ForeignKey('rep_master.rep_id')),
Column('manager_id', Integer,
ForeignKey('manager_master.manager_id')),
Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now),
Column('last_updated_at',
TIMESTAMP,
onupdate=datetime.datetime.now))
table_Billing = Table(
'billing_data', meta,
Column('dealer_id', Integer,
ForeignKey('dealer_master.dealer_id')),
Column('billing_id',
Integer,
primary_key=True,
autoincrement=False),
Column('bill_amount', Integer, nullable=True),
Column('billing_month', Integer, nullable=True),
Column('billing_year', Integer, nullable=True),
Column('payment_method', TEXT(255), nullable=True),
Column('package_name', TEXT(255), nullable=True),
Column('dealer_change_status', TEXT(2), default='A'),
Column('billing_change_previous_month', Integer, nullable=True),
Column('billing_date', TIMESTAMP, nullable=True),
Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now),
Column('last_updated_at',
TIMESTAMP,
onupdate=datetime.datetime.now))
table_CoopData = Table(
'co_op_data', meta,
Column('id', Integer, primary_key=True, autoincrement=True),
Column('dealer_id', Integer,
ForeignKey('dealer_master.dealer_id')),
Column('co_op_month', Integer, nullable=True),
Column('co_op_year', Integer, nullable=True),
Column('co_op_flag', BOOLEAN, nullable=True),
Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now),
Column('last_updated_at',
TIMESTAMP,
onupdate=datetime.datetime.now))
meta.create_all(db)
except SQLAlchemyError as sqlalchemy_error:
logger.error("ERROR: %s", sqlalchemy_error)
sys.exit()
logger.info("SUCCESS: Tables created")
insert_to_realm = table_realm_type.insert()
try:
insert_to_realm.execute(
{
'id': 1,
'name': 'CYCLE',
'category': 'COMMERCIAL',
'is_active': 1
}, {
'id': 2,
'name': 'RV',
'category': 'COMMERCIAL',
'is_active': 1
}, {
'id': 3,
'name': 'TRUCK',
'category': 'RECREATIONAL',
'is_active': 1
}, {
'id': 4,
'name': 'EQUIPMENT',
'category': 'RECREATIONAL',
'is_active': 1
})
except SQLAlchemyError as err:
print(err)
logger.error("ERROR: %s", err)
logger.info("SUCCESS: Inserted values")
return meta
#!/usr/bin/python
#
# Re-encrypts (git checkout) encrypted files for the given environment.
#
import helpers
import traceback
import argparse
# Check parameters
parser = argparse.ArgumentParser(description='Decrypt environment files')
parser.add_argument('env_name', type=str, help='Name of the environment')
args = parser.parse_args()
helpers.logger = helpers.setup_logging('reencrypt_env.log')
helpers.log('Re-encrypting environment: %s' % args.env_name, as_banner=True, bold=True)
try:
helpers.reencrypt_env(args.env_name)
except Exception, e:
helpers.logger.debug('Failure: ' + str(e) + ", exception: " + traceback.format_exc().replace('\n', ' | '))
raise
# arrival_date = stop_events[0][0]
# departure_date = stop_events[-1][0]
# if i > 0:
# diff = (arrival_date - starting_time[-1]).total_seconds()
# if diff < 0:
# diff = (arrival_date -starting_time[0]).total_seconds()
# arrival_times.append((diff, name))
# starting_time = (arrival_date, departure_date)
#pprint(arrival_times)
arrival_times_list.append(arrival_times)
assert len(segments_list) == len(arrival_times_list)
return segments_list, arrival_times_list
def is_stop_added(segment_starts, event, index):
if not segment_starts:
return False
if segment_starts[-1][0][-1] == index:
return True
if "stop.name" in event and segment_starts[-1][1] == event["stop.name"]:
return True
return False
if __name__ == "__main__":
logger = setup_logging("forecasting.py", "forecasting.log")
BASE_DIR = "forecasting/"
main(sys.argv[1:])
parser.add_argument('--tags', type=str, default='', help='Command-separated list of tags to apply', required=False)
parser.add_argument('--force', help='Whether to force the operation without prompt', action='store_true')
parser.add_argument('--healthcheck', help='Whether to do health check', action='store_true')
parser.add_argument('--logdir', type=str, default='/tmp', help='Directory to stash logs', required=False)
parser.add_argument('--playbook', type=str, default='site.yml', help='Playbook to execute', required=False)
parser.add_argument('--extra_vars', type=str, default='', help='Extra variables for the playbook', required=False)
args = parser.parse_args()
env_name = args.env_name
if not args.force and not helpers.yes_or_no('Are you SURE you want to deploy %s to environment: %s with tags: \'%s\' ?'
% (args.playbook, env_name, args.tags)):
sys.exit(1)
# Global default logger
logger = helpers.setup_logging('ansible_deploy_env.log', log_dir=args.logdir)
helpers.logger = logger
health.logger = logger
helpers.log('[%s] Ready to deploy Ansible changes' % env_name, as_banner=True, bold=True)
# Populate the environment
try:
helpers.populate_env(env_name)
except Exception, e:
logger.error('Failure: ' + str(e) + ", exception: " + traceback.format_exc().replace('\n', ' | '))
raise Exception('[%s] Populate failed' % env_name)
try:
# Deploy Ansible
helpers.log('[%s] Deploying Ansible' % env_name, as_banner=True)
#
import argparse
import traceback
import health
import helpers
import testhelpers
import random, string
import json
import os
import sys
import requests
import time
TESTS_DIR = os.path.abspath(os.path.dirname(__file__))
logger = helpers.setup_logging('integration_tests.log')
helpers.logger = logger
PHASE_BEFORE = "before"
PHASE_AFTER = "after"
def get_random_id():
return ''.join([random.choice(string.lowercase) for i in xrange(20)])
def execute_before_phase(phase):
return phase is None or phase == PHASE_BEFORE
def execute_after_phase(phase):
return phase is None or phase == PHASE_AFTER
"""
Example code to demonstrate MySQL -> Redshift replication using pandas
"""
import click
from datetime import datetime
from db_utils import extract_table_into_dataframe, write_to_redshift
from helpers import setup_logging
import json
import sys
logger = setup_logging('etl')
with open('config.json', 'r') as f:
config = json.load(f)
@click.group()
def cli():
pass
@cli.command()
@click.option('--run_date',
type=str,
default=datetime.today().strftime('%Y-%m-%d'),
help='Please enter the date to run the ETL script for')
@click.option('--table_name',
type=str,
default=None,
help='Please enter the table name to replicate')
import boto3
from dotenv import load_dotenv
import os
import pandas as pd
from sqlalchemy import create_engine
from helpers import setup_logging
logger = setup_logging('load')
# load dotenv in the base root
ROOT_DIR = os.path.dirname(
os.path.abspath(__file__)) #refers to application_top
dotenv_path = os.path.join(ROOT_DIR, '.env')
load_dotenv(dotenv_path)
DB_USER = os.getenv('WS_RS_USER')
DB_SECRET = os.getenv('WS_RS_SECRET')
AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
S3_BUCKET_NAME = os.getenv('S3_BUCKET_NAME', 'dev-bucket')
if not DB_USER or not DB_SECRET:
raise Exception('Environment Variables missing.')
exit(0)
def execute_query(source_config, sql):
""" Executes SQL query against source database. Used for reading
data into ETL process.
for event in events:
system_time.set_time(event['timestamp'])
getattr(stats, event['event_name'])(**(event['event_data']))
stats.job_ended()
if print_stats:
print stats
return stats.stats
class SystemTime(object):
def __init__(self):
self._t = 0
def set_time(self, t):
if t < self._t:
raise RuntimeError('Time decreased')
self._t = t
def get_time(self):
return self._t
if __name__ == '__main__':
if len(sys.argv) != 2:
usage()
sys.exit(1)
setup_logging()
analyze_basic(sys.argv[1])
analyze_distn(sys.argv[1])
config = ConfigParser.RawConfigParser()
try:
config.readfp(open('config'))
except IOError:
print "Error: No configuration file 'config' found"
exit(1)
try:
download_dir = config.get('General', 'download-dir')
quality = config.get('General', 'quality')
log_file_path = config.get('Logging', 'file-path')
log_file_size = config.get('Logging', 'max-size')
except ConfigParser.NoOptionError, e:
print "Error:", e
exit(1)
helpers.setup_logging(log_file_path, log_file_size)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
classes = [{
"name": "Intro to DSA - Why Learning DSA is Important",
"videoId": "YEOuqKT-svE",
"day": datetime.strptime("19/11/20", '%d/%m/%y').date()
}, {
"name": "Basic Maths + Practise Questions + Q/A",
"videoId": "POehjAlYqNw",
"day": datetime.strptime("25/11/20", '%d/%m/%y').date()
}, {
"name": "Introduction to Arrays and Operations",
"videoId": "uFdm_kXGJkU",
import fileinput
from collections import OrderedDict
SCRIPTS_DIR = helpers.PROJECT_ROOT_DIR + '/scripts'
TEMPLATES_DIR = SCRIPTS_DIR + '/templates'
PARAMS_JSON = SCRIPTS_DIR + '/create_env_params.json'
ANSIBLE_VAULT_CHALLENGE_FILE = SCRIPTS_DIR + '/ansible-vault-challenge.txt'
RESUMABLE_FILE_NAME = 'resumeable.txt'
TERRAFORM_SHA_FILE = 'terraform-sha.txt'
RESUME_SECTION = 'RESUME'
K8S_SECTION = 'K8S'
DESTROY_FILE_NAME = 'destroy.sh'
PREFS_FILE_DEFAULT = os.path.expanduser('~') + '/.k8s/config'
NUM_ADS = 3
helpers.logger = helpers.setup_logging('create_env.log')
def get_git_branch_name(env_name):
(stage_name, team_name) = env_name.split('/')
return '%s/create-%s-%s-env' % (getpass.getuser(), stage_name, team_name)
def generate_destroy_env_script(args):
env_dir = helpers.ENVS_DIR + '/' + args.env_name
destroy_file = env_dir + '/' + DESTROY_FILE_NAME
f = open(destroy_file, 'w')
f.write('set -e\n')
f.write('cd %s\n' % env_dir)
f.write(
'terragrunt destroy -force -state=`pwd`/terraform.tfstate -var disable_auto_retries=true '
def main_wc_gc_model():
config = ConfigParser.ConfigParser()
config.read('config/model_config.config')
app_log = setup_logging('wc_gc_logger',
config.get('logger', 'log_file_name'))
app_log.info('Scoring DUNS number: %d' % user_duns_number)
rate_lookup_table = read_rate_lookup(
config.get('data_files', 'rate_lookup'), user_is_uslh)
input_data['lookup_key'] = input_data['state'] + input_data['class_code']
avg_rates = merge_rate_lookup(input_data, rate_lookup_table)
entered_ratios = calc_entered_payroll_ratios(input_data)
inputs_valid, reason = check_inputs(input_data, entered_ratios)
if not inputs_valid:
return (numpy.NaN, numpy.NaN, numpy.NaN, reason)
payrolls = calc_payroll_ratio(input_data)
calc_allocate_clerical_payroll(payrolls, user_estimated_clerical_payroll)
calc_clerical_class_premium(payrolls, rate_lookup_table)
calc_standard_premium(payrolls, user_experience_mod)
standard_premium_to_allocate = calc_missing_standard_premium(
payrolls, avg_rates, user_experience_mod)
calc_allocated_standard_premium(payrolls, standard_premium_to_allocate)
calc_premium_discount(payrolls,
config.getfloat('constants', 'other_loadings'),
eval(config.get('data_files', 'ncci_tier_files')))
state_rate_data = merge_wcng_lr_rate_need(
payrolls, user_division, user_effective_date, user_is_uslh,
config.get('data_files', 'state_rate_need_lookup'),
config.get('data_files', 'wcng_lr'))
credit_scores = get_dnb_scores(
user_duns_number,
default_credit_score_pct=config.get('constants',
'default_duns_cs_pct'),
default_financial_score_pct=config.get('constants',
'default_duns_fs_pct'))
total_class_premium = input_data['class_premium'].sum()
predom_state = input_data.groupby(
by='state')['class_premium'].sum().idxmax(axis=1)
model_inputs = calc_normalized_claim_counts(
input_history, predom_state, eval(config.get('aqi', 'aqi_data')),
total_class_premium, config.get('data_files', 'cdf_file'))
model_inputs['credit_score_pct'] = credit_scores['credit_score_pct']
model_inputs['financial_score_pct'] = credit_scores['financial_score_pct']
model_inputs['payroll'] = user_total_projected_payroll
model_inputs['major_group'] = get_sic_major_group(user_sic_code)
predicted_lr = run_model(
model_inputs, config.get('data_files', 'model_coefficients_file'),
eval(config.get('model_rules', 'rules')))
state_rate_data['target_pricing_deviation_factor'] = (
((predicted_lr / state_rate_data['avg_wcng_loss_ratio']) *
state_rate_data['variable_rate_need']) +
state_rate_data['fix_rate_need'])
state_rate_data['estimated_premium'] = state_rate_data[
'target_pricing_deviation_factor'] * state_rate_data[
'manual_rate_pre_model']
output_midpoint = state_rate_data['estimated_premium'].sum()
lower_ratio, upper_ratio = calc_diamond_bound_ratios(
entered_ratios['clerical'], entered_ratios['non_clerical'],
config.get('data_files', 'bound_ratios'))
return (output_midpoint * lower_ratio, output_midpoint,
output_midpoint * upper_ratio, '')
def parse_journeystartend_event(header_fields, body_fields):
"""
Header: date | 2 (?), event_type, event_id, status (Normal/Warning/Error), 2793946901 (ref event_id?)
Body:
0: vehicle
1: junk
2: gid (vehicle_id)
3: vehicle_id
4: gps
5: gid (line)
"""
#print(header_fields, body_fields)
return {
"date": dateutil.parser.parse(header_fields[0]),
"event.type": header_fields[2],
"event.id": int(header_fields[3]),
"ref.id": int(header_fields[5]),
"vehicle.type": body_fields[0],
"gid": int(body_fields[2]),
"vehicle.id": int(body_fields[3]),
#"gps": str_gps_to_tuple(body_fields[4]),
"line": int(body_fields[5][7:11]),
"line.id": int(body_fields[5][11:]),
}
if __name__ == "__main__":
logger = setup_logging("eda.py", "main.log")
main(sys.argv[1:])
import datetime
from flask import Flask, jsonify, redirect, render_template, url_for
from flask_moment import Moment
from helpers import (build_schedule, elapsed_percent, ensure_logs_dir,
human_readable_time, load_yaml, schedule_expired,
setup_logging)
from vlc_client.vlc_client import VLCClient
app = Flask(__name__)
moment = Moment(app)
config = load_yaml('config.yaml')
ensure_logs_dir(config['LOGGING']['path'])
log_file = setup_logging(config['LOGGING']['path'])
app.logger.addHandler(log_file)
vlc = VLCClient(config['VLC'])
SCHEDULE = {"exp": datetime.datetime.now(), "playlist": {}, "current": {}}
@app.route('/')
def index():
# TODO 6.26.2020: Throw a 404 if vlc is not running instead of breaking app
# rebuild the schedule if the cache has expired
if schedule_expired(SCHEDULE):
try:
current = vlc.get_status()
playlist = vlc.get_playlist()
SCHEDULE.clear()
SCHEDULE.update(
unfilled_plot = sns.heatmap(
df[unfilled_cols].isnull().sample(frac=1).reset_index(drop=True),
cmap=sns.color_palette(colours))
unfilled_plot.figure.savefig("../output/unfilled_plot.png")
# Drop based on NAs
# Drop based on correlation
pass
if __name__ == "__main__":
get_parser()
PATH_TO_CONFIG = "config.json"
config = get_config(PATH_TO_CONFIG)
setup_logging(config)
path_to_dataset = config["dataset"]["path"]
df = pd.read_csv(path_to_dataset,
infer_datetime_format=True,
parse_dates=['timestamp'])
logging.info("Dataset loaded")
logging.info(f"Dataset has the following shape {df.shape}")
main(df, config)
def main():
"""Main function
"""
args_parser = argparse.ArgumentParser()
args_parser.add_argument("source",
type=argparse.FileType("rb"),
help="application XML file")
args_parser.add_argument("-t", "--target", type=str, help="target folder")
args_parser.add_argument("-v",
"--verbosity",
action="count",
help="be more verbose",
default=0)
args_parser.add_argument("-e",
"--erase",
action="store_true",
help="erase target folder")
args_parser.add_argument("-q",
"--quiet",
action="store_true",
help="no user interaction")
args_parser.add_argument("-i",
"--ignore-cfg",
type=argparse.FileType("rb"),
help="ignore config file")
args_parser.add_argument("-l",
"--libraries",
action="store_true",
help="parse libraries")
args_parser.add_argument("-p",
"--pages",
action="store_true",
help="parse pages")
args_parser.add_argument("-d",
"--databases",
action="store_true",
help="parse databases")
args_parser.add_argument("-r",
"--resources",
action="store_true",
help="parse resources")
args_parser.add_argument("-n",
"--info",
action="store_true",
help="parse information")
args_parser.add_argument("-s",
"--security",
action="store_true",
help="parse security")
args_parser.add_argument("-u",
"--structure",
action="store_true",
help="parse structure")
args_parser.add_argument("-o",
"--e2vdom",
action="store_true",
help="parse e2vdom")
args_parser.add_argument("-c",
"--app-actions",
action="store_true",
help="parse application actions")
args_parser.add_argument("-ds",
"--delete-source",
action="store_true",
help="delete source .xml file")
args = args_parser.parse_args()
# Setup logging system and show necessary messages
log_level = logging.INFO if not args.verbosity else logging.DEBUG
show_module_name = args.verbosity > 1
setup_logging(log_level, module_name=show_module_name)
INFO("")
INFO("Information logging turned on")
DEBUG("Debug logging turned on")
INFO("")
INFO(BLOCK_END)
INFO("")
ignore = args.ignore_cfg
if ignore:
INFO("Parsing: 'ignore' configuration file")
ignore = json_load(ignore, critical=True)
INFO("Done: 'ignore' configuration file")
config = {
"target": {
"path":
args.target or os.path.split(args.source.name)[-1].split(".")[0],
"erase":
args.erase,
"quiet":
args.quiet,
},
"source": args.source,
"ignore": ignore,
"delete_source": args.delete_source,
"parse": {
"app_actions": args.app_actions,
"e2vdom": args.e2vdom,
"structure": args.structure,
"security": args.security,
"info": args.info,
"resources": args.resources,
"databases": args.databases,
"pages": args.pages,
"libraries": args.libraries
},
}
parse_all = False
for val in config["parse"].values():
parse_all = parse_all or val
config["parse_all"] = not parse_all
# Main process starting
parse(config)
if config["delete_source"] and os.path.exists(args.source.name):
args.source.close()
os.remove(args.source.name)
INFO("\nPath to application:\n{}".format(config["target"]["path"]))
for i in range(segments):
plt.figure()
trajectory_ids = sorted([32, 34, 4, 6, 11, 16, 20, 24, 27])
markers = [""] * len(trajectory_ids) #["o", "s", "p", "*", "+", "x", "1", "2", "3", "4"]
for traj, mark in zip(trajectory_ids, markers):
result = hlp.load_array("{}{}/{}/predicted".format(BASE_DIR, traj, i))
feature = result["feature"]
feature = (feature - np.min(feature)) / np.ptp(feature)
truth = result["truth"]
pred = result["predicted"]
plt.plot(feature, [abs(t - p) for t, p in zip(truth, pred)], "{}--".format(mark), label="Traj. {}".format(traj))
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.xlabel("Segment progress")
plt.ylabel("Absolute error [s]")
plt.title("Predicted arrival time errors for test trajectories (segment {})".format(i))
plt.savefig("forecasting/metrics/segment_{}.png".format(i), bbox_inches="tight")
plt.close()
#truths.extend(result["truth"])
#predicted.extend(result["predicted"])
#truths = np.array(truths)
#predicted = np.array(predicted)
#logger.info("RMSE: {}".format(round(np.sqrt(mean_squared_error(truths, predicted)), 2)))
#logger.info("MAE: {}".format(round(mean_absolute_error(truths, predicted), 2)))
if __name__ == "__main__":
logger = setup_logging("file_lengths.py", "file_lengths.log")
main(sys.argv[1:])
#!/usr/bin/python
#
# Unpopulates dynamic files for the given environment.
#
import helpers
import traceback
import argparse
# Check parameters
parser = argparse.ArgumentParser(description='Unpopulate dynamic files for an environment')
parser.add_argument('env_name', type=str, help='Name of the environment')
args = parser.parse_args()
helpers.logger = helpers.setup_logging('unpopulate_env.log')
helpers.log('Unopulating environment: %s' % args.env_name, as_banner=True, bold=True)
try:
helpers.unpopulate_env(args.env_name)
except Exception, e:
helpers.logger.debug('Failure: ' + str(e) + ", exception: " + traceback.format_exc().replace('\n', ' | '))
raise
plt.clf()
plt.close()
def plot_gp_grid_region(grid, gp_mean, gp_var):
plt.plot(grid, gp_mean, lw=1, color="red")
plt.fill_between(grid,
gp_mean[:, 0] - 2 * np.sqrt(gp_var[:, 0]),
gp_mean[:, 0] + 2 * np.sqrt(gp_var[:, 0]),
color="red",
alpha=0.2)
def train_GP(X_tau, Y, session, number, gp_name):
"""GP which maps tau -> lng or lat."""
with gpflow.defer_build():
m = gpflow.models.GPR(X_tau, Y, kern=gpflow.kernels.RBF(1))
m.likelihood.variance = 1e-03
m.likelihood.variance.trainable = False
m.compile()
opt = gpflow.train.ScipyOptimizer()
opt.minimize(m)
logger.info("{} GP #{} trained.".format(gp_name, number))
session.save(BASE_DIR + "GP/model_{}_{}".format(number, gp_name), m)
return m
if __name__ == "__main__":
logger = setup_logging("gps_var.py", "gps_var.log")
BASE_DIR = "gps_var/"
main(sys.argv[1:])
#!/usr/bin/python
import json
import traceback
import argparse
from argparse import RawTextHelpFormatter
from helpers import setup_logging
# Global default logger
logger = setup_logging('health.log')
def test_k8s(kubeconfig):
# TODO - add k8s general health check
pass
def health(healthfile, logger=logger):
# here we try to figure out who we want to monitor by looking at the passed in health file
# and also to other parameters passed in which may change the behavior
endpoints = []
try:
logger.info("Checking with health file: %s" % healthfile)
f = open(healthfile, "r")
health_config = json.load(f)
kubeconfig = health_config['k8s']['kubeconfig']
test_k8s(kubeconfig)
logger.info('Health Check succeeded.')
except Exception, e:
python interact_server.py --max_history 4 --top_p 0.8 --fp16 O2 --model_checkpoint runs/Jul19_14-38-58_ip-172-31-39-133_goood
"""
import collections
import logging
import os
import random
from argparse import ArgumentParser
import irc3
from irc3.plugins.command import command
os.sys.path.append("..")
from interact_server import ModelAPI, TOPICS
import logging
from helpers import setup_logging
setup_logging("irc_bot", level=logging.INFO)
logger = logging.getLogger(__file__)
logging.getLogger("zmqtest").setLevel(logging.INFO)
@irc3.plugin
class Plugin:
requires = [
"irc3.plugins.core", "irc3.plugins.command", "irc3.plugins.log"
]
def __init__(self, bot):
self.bot = bot
self.model_api = ModelAPI(port=bot.config["model_api"]["port"])
