def main(): """Main function """ args_parser = argparse.ArgumentParser() args_parser.add_argument("cfg", type=argparse.FileType("rb"), help="configuration file") args_parser.add_argument("target", type=str, help="target folder") args_parser.add_argument("-v", "--verbosity", action="count", help="be more verbose", default=0) args_parser.add_argument("-e", "--erase", action="store_true", help="erase target folder") args_parser.add_argument("-q", "--quiet", action="store_true", help="no user interaction") args = args_parser.parse_args() # Setup logging system and show necessary messages setup_logging(logging.INFO if args.verbosity == 0 else logging.DEBUG, module_name=True if args.verbosity > 1 else False) INFO("Information logging turned on") DEBUG("Debug logging turned on") # Parsing config file DEBUG("Parsing config file") config = json_load(args.cfg, critical=True) INFO("Config file parsed successfully") config["target"] = { "path": args.target, "erase": args.erase, "quiet": args.quiet, } # Main process starting make(config) INFO("") INFO("+" * 70) INFO("\nPath to application:\n{}".format(config["target"]["path"]))
def make_lookout_csv(state, debug='INFO'): """Desc. :param 1: asdf. :return: asdf. """ helpers.setup_logging(level=debug) logger.info('STATE IS: {}'.format(state)) lookout_urls = get_lookout_urls(state) logger.debug('LOOKOUT URLS: {}'.format(lookout_urls)) lookouts = make_lookouts(state, lookout_urls) write_csv(state, lookouts)
def main(): """ """ #--setup: parse commandline arguments--------------------------------------- args = setup_parse_args() #--setup: logging and data output------------------------------------------- logger = setup_logging( path_log=args.path_log + "velocity_data_log", log_level=args.log_level, ) setup_output_path(args.path_data_output) #--setup: main objects for using BlockSci----------------------------------- Velo.setup( logger=logger, args=args, ) Multiprocess.setup(logger=logger, args=args) #--Retrieval of basic blockchain data, money supply and velocity measures--- results_raw = Multiprocess.run() #--get csv of final pandas data frame--------------------------------------- Velo.get_results_finalized( results_raw=results_raw, index_label="date", ) print("Exiting program") exit(0)
def etl_job(): data = json.load(open('/home/ubuntu/ti_etl/secrets.json')) logger = helpers.setup_logging() s3_client = boto3.client( 's3', aws_access_key_id=data['aws_access_key_id'], aws_secret_access_key=data['aws_secret_access_key']) s3_resource = boto3.resource( 's3', aws_access_key_id=data['aws_access_key_id'], aws_secret_access_key=data['aws_secret_access_key']) keys = [] resp = s3_client.list_objects_v2(Bucket='dealer-churn-analysis') for obj in resp['Contents']: keys.append(obj['Key']) for key in keys: names = key.split("/") obj = s3_resource.Bucket('dealer-churn-analysis').Object( helpers.zip_file_name()) file_name = 'praxis/etl/logs/log_{file}.txt'.format(file=helpers.date()) obj_log = s3_resource.Bucket('dealer-churn-analysis').Object(file_name) buffer = io.BytesIO(obj.get()["Body"].read()) zip_file = zipfile.ZipFile(buffer, 'r') logger.info("Name of csv in zip file :%s", zip_file.namelist()) logs = "" dataframe = pd.DataFrame() for name_of_zipfile in zip_file.namelist(): zip_open = pd.read_csv(zip_file.open(name_of_zipfile)) dataframe['created_at'] = pd.Series([datetime.datetime.now()] * len(zip_open)) dataframe['last_updated_at'] = pd.Series([datetime.datetime.now()] * len(zip_open)) zip_open = pd.concat([dataframe, zip_open], axis=1) zip_open = zip_open.dropna() table_name = "{name}_table".format( name=name_of_zipfile.replace('.csv', '')) #print (zip_open) try: zip_open.to_sql(name=name_of_zipfile.replace('.csv', ''), con=database.db_connection(), if_exists='append', index=False) except SQLAlchemyError as sqlalchemy_error: print(sqlalchemy_error) logs = '\n{table_name}\n{error}\n{logs}'.format( logs=logs, error=sqlalchemy_error, table_name=table_name) logger.error(" %s", sqlalchemy_error) database.db_connection().execute('SET FOREIGN_KEY_CHECKS=1;') end_time = datetime.datetime.now() logger.info("End time of program : %s", end_time) logs = '{logs} \nstart_time : {start_time} \nend_time : {end_time}'.format( start_time=helpers.start_time(), logs=logs, end_time=end_time) print(logs) obj_log.put(Body=logs)
def main(): """Main function """ args_parser = argparse.ArgumentParser() args_parser.add_argument("source", type=str, help="aplication source folder") args_parser.add_argument("target", type=str, help="target XML file") args_parser.add_argument("-v", "--verbosity", action="count", help="be more verbose", default=0) args = args_parser.parse_args() # Setup logging system and show necessary messages setup_logging(logging.INFO if args.verbosity == 0 else logging.DEBUG, module_name=True if args.verbosity > 1 else False) INFO("") INFO("Information logging turned on") DEBUG("Debug logging turned on") INFO("") INFO(BLOCK_END) INFO("") config = { "target": { "path": args.target, }, "source": args.source } # Main process starting build(config) INFO("\nPath to application XML:\n{}".format(config["target"]["path"]))
def process_args(argv): try: opts, args = getopt.getopt(argv, "gh", ["grade", "help"]) except getopt.GetoptError: usage() sys.exit(1) extended_logging = False for opt, arg in opts: if opt in ("-g", "--grade"): extended_logging = True elif opt in ("-h", "--help"): usage() sys.exit() filename = "".join(args) if filename == "": filename = "output.log" if extended_logging: print "Verbose logging is turned on; outputting to {0}".format(filename) setup_logging(filename, grading=True) else: setup_logging(filename, grading=False)
def db_connection(): data = json.load(open('/home/ubuntu/ti_etl/secrets.json')) Base = declarative_base() logger = helpers.setup_logging() DB_URI = "mysql+pymysql://{user}:{password}@{host}:{port}/{db}" try: engine = create_engine(DB_URI.format(user=data['user'], password=data['password'], port=data['port'], host=data['host'], db=data['db']), echo=True) engine.execute('SET FOREIGN_KEY_CHECKS=0;') except SQLAlchemyError as sqlalchemy_error: logger.error("%s", sqlalchemy_error) sys.exit() return engine
#!/usr/bin/python # # Decrypts encrypted files for the given environment. # import helpers import traceback import argparse # Check parameters parser = argparse.ArgumentParser(description='Decrypt environment files') parser.add_argument('env_name', type=str, help='Name of the environment') args = parser.parse_args() helpers.logger = helpers.setup_logging('decrypt_env.log') helpers.log('Decrypting environment: ' + args.env_name, as_banner=True, bold=True) try: helpers.decrypt_env(args.env_name) except Exception, e: helpers.logger.debug('Failure: ' + str(e) + ", exception: " + traceback.format_exc().replace('\n', ' | ')) raise
def create_tables(): logger = helpers.setup_logging() logger.info("Start time of program :%s", helpers.start_time()) db = database.db_connection() meta = MetaData(bind=db) logger.info("INFO: Creating tables schema") try: table_realm_type = Table( 'realm_type', meta, Column('id', Integer, primary_key=True, autoincrement=True), Column('name', TEXT(50), nullable=True), Column('category', TEXT(50), nullable=True), Column('is_active', BOOLEAN, nullable=True), Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now), Column('last_updated_at', TIMESTAMP, onupdate=datetime.datetime.now)) table_category = Table( 'category', meta, Column('id', Integer, primary_key=True, autoincrement=True), Column('name', TEXT(50), nullable=True), Column('is_active', BOOLEAN, nullable=True), Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now), Column('last_updated_at', TIMESTAMP, onupdate=datetime.datetime.now)) table_dealer_master = Table( 'dealer_master', meta, Column('dealer_id', Integer, primary_key=True, autoincrement=False), Column('dealer_name', TEXT(255), nullable=True), Column('realm_name', TEXT(50), nullable=True), Column('customer_id', Integer, nullable=True), Column('customer_name', TEXT(255), nullable=True), Column('category_name', TEXT(50), nullable=True), Column('city', TEXT(255), nullable=True), Column('state_code', TEXT(10), nullable=False), Column('zip_code', Integer, nullable=True), Column('country_code', TEXT(10), nullable=False), Column('area_code', Integer, nullable=True), Column('start_date', TIMESTAMP, nullable=True), Column('expire_date', TIMESTAMP, nullable=True), Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now), Column('last_updated_at', TIMESTAMP, onupdate=datetime.datetime.now)) table_ads_data = Table( 'ads_data', meta, Column('ad_id', Integer, primary_key=True, autoincrement=False), Column('dealer_id', Integer, ForeignKey('dealer_master.dealer_id')), Column('make_id', Integer, nullable=True), Column('make_name', TEXT, nullable=True), Column('ad_status', TEXT, nullable=True), Column('created_at', TIMESTAMP, nullable=True), Column('last_updated_at', TIMESTAMP, nullable=True), Column('expire_date', TIMESTAMP, nullable=True), Column('create_date', TIMESTAMP, nullable=True), Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now), Column('last_updated_at', TIMESTAMP, onupdate=datetime.datetime.now)) table_rep_master = Table( 'rep_master', meta, Column('rep_id', Integer, primary_key=True, autoincrement=False), Column('rep_name', TEXT(255), nullable=True), Column('is_active', BOOLEAN, nullable=True), Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now), Column('last_updated_at', TIMESTAMP, onupdate=datetime.datetime.now)) table_ManagerMaster = Table( 'manager_master', meta, Column('manager_id', Integer, primary_key=True, autoincrement=False), Column('manager_name', TEXT(255), nullable=True), Column('is_active', BOOLEAN, nullable=True), Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now), Column('last_updated_at', TIMESTAMP, onupdate=datetime.datetime.now)) table_DealerRepManagerMapping = Table( 'dealer_rep_manager_mapping', meta, Column('id', Integer, primary_key=True, autoincrement=True), Column('dealer_id', Integer, ForeignKey('dealer_master.dealer_id')), Column('rep_id', Integer, ForeignKey('rep_master.rep_id')), Column('manager_id', Integer, ForeignKey('manager_master.manager_id')), Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now), Column('last_updated_at', TIMESTAMP, onupdate=datetime.datetime.now)) table_Billing = Table( 'billing_data', meta, Column('dealer_id', Integer, ForeignKey('dealer_master.dealer_id')), Column('billing_id', Integer, primary_key=True, autoincrement=False), Column('bill_amount', Integer, nullable=True), Column('billing_month', Integer, nullable=True), Column('billing_year', Integer, nullable=True), Column('payment_method', TEXT(255), nullable=True), Column('package_name', TEXT(255), nullable=True), Column('dealer_change_status', TEXT(2), default='A'), Column('billing_change_previous_month', Integer, nullable=True), Column('billing_date', TIMESTAMP, nullable=True), Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now), Column('last_updated_at', TIMESTAMP, onupdate=datetime.datetime.now)) table_CoopData = Table( 'co_op_data', meta, Column('id', Integer, primary_key=True, autoincrement=True), Column('dealer_id', Integer, ForeignKey('dealer_master.dealer_id')), Column('co_op_month', Integer, nullable=True), Column('co_op_year', Integer, nullable=True), Column('co_op_flag', BOOLEAN, nullable=True), Column('created_at', TIMESTAMP, onupdate=datetime.datetime.now), Column('last_updated_at', TIMESTAMP, onupdate=datetime.datetime.now)) meta.create_all(db) except SQLAlchemyError as sqlalchemy_error: logger.error("ERROR: %s", sqlalchemy_error) sys.exit() logger.info("SUCCESS: Tables created") insert_to_realm = table_realm_type.insert() try: insert_to_realm.execute( { 'id': 1, 'name': 'CYCLE', 'category': 'COMMERCIAL', 'is_active': 1 }, { 'id': 2, 'name': 'RV', 'category': 'COMMERCIAL', 'is_active': 1 }, { 'id': 3, 'name': 'TRUCK', 'category': 'RECREATIONAL', 'is_active': 1 }, { 'id': 4, 'name': 'EQUIPMENT', 'category': 'RECREATIONAL', 'is_active': 1 }) except SQLAlchemyError as err: print(err) logger.error("ERROR: %s", err) logger.info("SUCCESS: Inserted values") return meta
#!/usr/bin/python # # Re-encrypts (git checkout) encrypted files for the given environment. # import helpers import traceback import argparse # Check parameters parser = argparse.ArgumentParser(description='Decrypt environment files') parser.add_argument('env_name', type=str, help='Name of the environment') args = parser.parse_args() helpers.logger = helpers.setup_logging('reencrypt_env.log') helpers.log('Re-encrypting environment: %s' % args.env_name, as_banner=True, bold=True) try: helpers.reencrypt_env(args.env_name) except Exception, e: helpers.logger.debug('Failure: ' + str(e) + ", exception: " + traceback.format_exc().replace('\n', ' | ')) raise
# arrival_date = stop_events[0][0] # departure_date = stop_events[-1][0] # if i > 0: # diff = (arrival_date - starting_time[-1]).total_seconds() # if diff < 0: # diff = (arrival_date -starting_time[0]).total_seconds() # arrival_times.append((diff, name)) # starting_time = (arrival_date, departure_date) #pprint(arrival_times) arrival_times_list.append(arrival_times) assert len(segments_list) == len(arrival_times_list) return segments_list, arrival_times_list def is_stop_added(segment_starts, event, index): if not segment_starts: return False if segment_starts[-1][0][-1] == index: return True if "stop.name" in event and segment_starts[-1][1] == event["stop.name"]: return True return False if __name__ == "__main__": logger = setup_logging("forecasting.py", "forecasting.log") BASE_DIR = "forecasting/" main(sys.argv[1:])
parser.add_argument('--tags', type=str, default='', help='Command-separated list of tags to apply', required=False) parser.add_argument('--force', help='Whether to force the operation without prompt', action='store_true') parser.add_argument('--healthcheck', help='Whether to do health check', action='store_true') parser.add_argument('--logdir', type=str, default='/tmp', help='Directory to stash logs', required=False) parser.add_argument('--playbook', type=str, default='site.yml', help='Playbook to execute', required=False) parser.add_argument('--extra_vars', type=str, default='', help='Extra variables for the playbook', required=False) args = parser.parse_args() env_name = args.env_name if not args.force and not helpers.yes_or_no('Are you SURE you want to deploy %s to environment: %s with tags: \'%s\' ?' % (args.playbook, env_name, args.tags)): sys.exit(1) # Global default logger logger = helpers.setup_logging('ansible_deploy_env.log', log_dir=args.logdir) helpers.logger = logger health.logger = logger helpers.log('[%s] Ready to deploy Ansible changes' % env_name, as_banner=True, bold=True) # Populate the environment try: helpers.populate_env(env_name) except Exception, e: logger.error('Failure: ' + str(e) + ", exception: " + traceback.format_exc().replace('\n', ' | ')) raise Exception('[%s] Populate failed' % env_name) try: # Deploy Ansible helpers.log('[%s] Deploying Ansible' % env_name, as_banner=True)
# import argparse import traceback import health import helpers import testhelpers import random, string import json import os import sys import requests import time TESTS_DIR = os.path.abspath(os.path.dirname(__file__)) logger = helpers.setup_logging('integration_tests.log') helpers.logger = logger PHASE_BEFORE = "before" PHASE_AFTER = "after" def get_random_id(): return ''.join([random.choice(string.lowercase) for i in xrange(20)]) def execute_before_phase(phase): return phase is None or phase == PHASE_BEFORE def execute_after_phase(phase): return phase is None or phase == PHASE_AFTER
""" Example code to demonstrate MySQL -> Redshift replication using pandas """ import click from datetime import datetime from db_utils import extract_table_into_dataframe, write_to_redshift from helpers import setup_logging import json import sys logger = setup_logging('etl') with open('config.json', 'r') as f: config = json.load(f) @click.group() def cli(): pass @cli.command() @click.option('--run_date', type=str, default=datetime.today().strftime('%Y-%m-%d'), help='Please enter the date to run the ETL script for') @click.option('--table_name', type=str, default=None, help='Please enter the table name to replicate')
import boto3 from dotenv import load_dotenv import os import pandas as pd from sqlalchemy import create_engine from helpers import setup_logging logger = setup_logging('load') # load dotenv in the base root ROOT_DIR = os.path.dirname( os.path.abspath(__file__)) #refers to application_top dotenv_path = os.path.join(ROOT_DIR, '.env') load_dotenv(dotenv_path) DB_USER = os.getenv('WS_RS_USER') DB_SECRET = os.getenv('WS_RS_SECRET') AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') S3_BUCKET_NAME = os.getenv('S3_BUCKET_NAME', 'dev-bucket') if not DB_USER or not DB_SECRET: raise Exception('Environment Variables missing.') exit(0) def execute_query(source_config, sql): """ Executes SQL query against source database. Used for reading data into ETL process.
for event in events: system_time.set_time(event['timestamp']) getattr(stats, event['event_name'])(**(event['event_data'])) stats.job_ended() if print_stats: print stats return stats.stats class SystemTime(object): def __init__(self): self._t = 0 def set_time(self, t): if t < self._t: raise RuntimeError('Time decreased') self._t = t def get_time(self): return self._t if __name__ == '__main__': if len(sys.argv) != 2: usage() sys.exit(1) setup_logging() analyze_basic(sys.argv[1]) analyze_distn(sys.argv[1])
config = ConfigParser.RawConfigParser() try: config.readfp(open('config')) except IOError: print "Error: No configuration file 'config' found" exit(1) try: download_dir = config.get('General', 'download-dir') quality = config.get('General', 'quality') log_file_path = config.get('Logging', 'file-path') log_file_size = config.get('Logging', 'max-size') except ConfigParser.NoOptionError, e: print "Error:", e exit(1) helpers.setup_logging(log_file_path, log_file_size) if not os.path.exists(download_dir): os.makedirs(download_dir) classes = [{ "name": "Intro to DSA - Why Learning DSA is Important", "videoId": "YEOuqKT-svE", "day": datetime.strptime("19/11/20", '%d/%m/%y').date() }, { "name": "Basic Maths + Practise Questions + Q/A", "videoId": "POehjAlYqNw", "day": datetime.strptime("25/11/20", '%d/%m/%y').date() }, { "name": "Introduction to Arrays and Operations", "videoId": "uFdm_kXGJkU",
import fileinput from collections import OrderedDict SCRIPTS_DIR = helpers.PROJECT_ROOT_DIR + '/scripts' TEMPLATES_DIR = SCRIPTS_DIR + '/templates' PARAMS_JSON = SCRIPTS_DIR + '/create_env_params.json' ANSIBLE_VAULT_CHALLENGE_FILE = SCRIPTS_DIR + '/ansible-vault-challenge.txt' RESUMABLE_FILE_NAME = 'resumeable.txt' TERRAFORM_SHA_FILE = 'terraform-sha.txt' RESUME_SECTION = 'RESUME' K8S_SECTION = 'K8S' DESTROY_FILE_NAME = 'destroy.sh' PREFS_FILE_DEFAULT = os.path.expanduser('~') + '/.k8s/config' NUM_ADS = 3 helpers.logger = helpers.setup_logging('create_env.log') def get_git_branch_name(env_name): (stage_name, team_name) = env_name.split('/') return '%s/create-%s-%s-env' % (getpass.getuser(), stage_name, team_name) def generate_destroy_env_script(args): env_dir = helpers.ENVS_DIR + '/' + args.env_name destroy_file = env_dir + '/' + DESTROY_FILE_NAME f = open(destroy_file, 'w') f.write('set -e\n') f.write('cd %s\n' % env_dir) f.write( 'terragrunt destroy -force -state=`pwd`/terraform.tfstate -var disable_auto_retries=true '
def main_wc_gc_model(): config = ConfigParser.ConfigParser() config.read('config/model_config.config') app_log = setup_logging('wc_gc_logger', config.get('logger', 'log_file_name')) app_log.info('Scoring DUNS number: %d' % user_duns_number) rate_lookup_table = read_rate_lookup( config.get('data_files', 'rate_lookup'), user_is_uslh) input_data['lookup_key'] = input_data['state'] + input_data['class_code'] avg_rates = merge_rate_lookup(input_data, rate_lookup_table) entered_ratios = calc_entered_payroll_ratios(input_data) inputs_valid, reason = check_inputs(input_data, entered_ratios) if not inputs_valid: return (numpy.NaN, numpy.NaN, numpy.NaN, reason) payrolls = calc_payroll_ratio(input_data) calc_allocate_clerical_payroll(payrolls, user_estimated_clerical_payroll) calc_clerical_class_premium(payrolls, rate_lookup_table) calc_standard_premium(payrolls, user_experience_mod) standard_premium_to_allocate = calc_missing_standard_premium( payrolls, avg_rates, user_experience_mod) calc_allocated_standard_premium(payrolls, standard_premium_to_allocate) calc_premium_discount(payrolls, config.getfloat('constants', 'other_loadings'), eval(config.get('data_files', 'ncci_tier_files'))) state_rate_data = merge_wcng_lr_rate_need( payrolls, user_division, user_effective_date, user_is_uslh, config.get('data_files', 'state_rate_need_lookup'), config.get('data_files', 'wcng_lr')) credit_scores = get_dnb_scores( user_duns_number, default_credit_score_pct=config.get('constants', 'default_duns_cs_pct'), default_financial_score_pct=config.get('constants', 'default_duns_fs_pct')) total_class_premium = input_data['class_premium'].sum() predom_state = input_data.groupby( by='state')['class_premium'].sum().idxmax(axis=1) model_inputs = calc_normalized_claim_counts( input_history, predom_state, eval(config.get('aqi', 'aqi_data')), total_class_premium, config.get('data_files', 'cdf_file')) model_inputs['credit_score_pct'] = credit_scores['credit_score_pct'] model_inputs['financial_score_pct'] = credit_scores['financial_score_pct'] model_inputs['payroll'] = user_total_projected_payroll model_inputs['major_group'] = get_sic_major_group(user_sic_code) predicted_lr = run_model( model_inputs, config.get('data_files', 'model_coefficients_file'), eval(config.get('model_rules', 'rules'))) state_rate_data['target_pricing_deviation_factor'] = ( ((predicted_lr / state_rate_data['avg_wcng_loss_ratio']) * state_rate_data['variable_rate_need']) + state_rate_data['fix_rate_need']) state_rate_data['estimated_premium'] = state_rate_data[ 'target_pricing_deviation_factor'] * state_rate_data[ 'manual_rate_pre_model'] output_midpoint = state_rate_data['estimated_premium'].sum() lower_ratio, upper_ratio = calc_diamond_bound_ratios( entered_ratios['clerical'], entered_ratios['non_clerical'], config.get('data_files', 'bound_ratios')) return (output_midpoint * lower_ratio, output_midpoint, output_midpoint * upper_ratio, '')
def parse_journeystartend_event(header_fields, body_fields): """ Header: date | 2 (?), event_type, event_id, status (Normal/Warning/Error), 2793946901 (ref event_id?) Body: 0: vehicle 1: junk 2: gid (vehicle_id) 3: vehicle_id 4: gps 5: gid (line) """ #print(header_fields, body_fields) return { "date": dateutil.parser.parse(header_fields[0]), "event.type": header_fields[2], "event.id": int(header_fields[3]), "ref.id": int(header_fields[5]), "vehicle.type": body_fields[0], "gid": int(body_fields[2]), "vehicle.id": int(body_fields[3]), #"gps": str_gps_to_tuple(body_fields[4]), "line": int(body_fields[5][7:11]), "line.id": int(body_fields[5][11:]), } if __name__ == "__main__": logger = setup_logging("eda.py", "main.log") main(sys.argv[1:])
import datetime from flask import Flask, jsonify, redirect, render_template, url_for from flask_moment import Moment from helpers import (build_schedule, elapsed_percent, ensure_logs_dir, human_readable_time, load_yaml, schedule_expired, setup_logging) from vlc_client.vlc_client import VLCClient app = Flask(__name__) moment = Moment(app) config = load_yaml('config.yaml') ensure_logs_dir(config['LOGGING']['path']) log_file = setup_logging(config['LOGGING']['path']) app.logger.addHandler(log_file) vlc = VLCClient(config['VLC']) SCHEDULE = {"exp": datetime.datetime.now(), "playlist": {}, "current": {}} @app.route('/') def index(): # TODO 6.26.2020: Throw a 404 if vlc is not running instead of breaking app # rebuild the schedule if the cache has expired if schedule_expired(SCHEDULE): try: current = vlc.get_status() playlist = vlc.get_playlist() SCHEDULE.clear() SCHEDULE.update(
unfilled_plot = sns.heatmap( df[unfilled_cols].isnull().sample(frac=1).reset_index(drop=True), cmap=sns.color_palette(colours)) unfilled_plot.figure.savefig("../output/unfilled_plot.png") # Drop based on NAs # Drop based on correlation pass if __name__ == "__main__": get_parser() PATH_TO_CONFIG = "config.json" config = get_config(PATH_TO_CONFIG) setup_logging(config) path_to_dataset = config["dataset"]["path"] df = pd.read_csv(path_to_dataset, infer_datetime_format=True, parse_dates=['timestamp']) logging.info("Dataset loaded") logging.info(f"Dataset has the following shape {df.shape}") main(df, config)
def main(): """Main function """ args_parser = argparse.ArgumentParser() args_parser.add_argument("source", type=argparse.FileType("rb"), help="application XML file") args_parser.add_argument("-t", "--target", type=str, help="target folder") args_parser.add_argument("-v", "--verbosity", action="count", help="be more verbose", default=0) args_parser.add_argument("-e", "--erase", action="store_true", help="erase target folder") args_parser.add_argument("-q", "--quiet", action="store_true", help="no user interaction") args_parser.add_argument("-i", "--ignore-cfg", type=argparse.FileType("rb"), help="ignore config file") args_parser.add_argument("-l", "--libraries", action="store_true", help="parse libraries") args_parser.add_argument("-p", "--pages", action="store_true", help="parse pages") args_parser.add_argument("-d", "--databases", action="store_true", help="parse databases") args_parser.add_argument("-r", "--resources", action="store_true", help="parse resources") args_parser.add_argument("-n", "--info", action="store_true", help="parse information") args_parser.add_argument("-s", "--security", action="store_true", help="parse security") args_parser.add_argument("-u", "--structure", action="store_true", help="parse structure") args_parser.add_argument("-o", "--e2vdom", action="store_true", help="parse e2vdom") args_parser.add_argument("-c", "--app-actions", action="store_true", help="parse application actions") args_parser.add_argument("-ds", "--delete-source", action="store_true", help="delete source .xml file") args = args_parser.parse_args() # Setup logging system and show necessary messages log_level = logging.INFO if not args.verbosity else logging.DEBUG show_module_name = args.verbosity > 1 setup_logging(log_level, module_name=show_module_name) INFO("") INFO("Information logging turned on") DEBUG("Debug logging turned on") INFO("") INFO(BLOCK_END) INFO("") ignore = args.ignore_cfg if ignore: INFO("Parsing: 'ignore' configuration file") ignore = json_load(ignore, critical=True) INFO("Done: 'ignore' configuration file") config = { "target": { "path": args.target or os.path.split(args.source.name)[-1].split(".")[0], "erase": args.erase, "quiet": args.quiet, }, "source": args.source, "ignore": ignore, "delete_source": args.delete_source, "parse": { "app_actions": args.app_actions, "e2vdom": args.e2vdom, "structure": args.structure, "security": args.security, "info": args.info, "resources": args.resources, "databases": args.databases, "pages": args.pages, "libraries": args.libraries }, } parse_all = False for val in config["parse"].values(): parse_all = parse_all or val config["parse_all"] = not parse_all # Main process starting parse(config) if config["delete_source"] and os.path.exists(args.source.name): args.source.close() os.remove(args.source.name) INFO("\nPath to application:\n{}".format(config["target"]["path"]))
for i in range(segments): plt.figure() trajectory_ids = sorted([32, 34, 4, 6, 11, 16, 20, 24, 27]) markers = [""] * len(trajectory_ids) #["o", "s", "p", "*", "+", "x", "1", "2", "3", "4"] for traj, mark in zip(trajectory_ids, markers): result = hlp.load_array("{}{}/{}/predicted".format(BASE_DIR, traj, i)) feature = result["feature"] feature = (feature - np.min(feature)) / np.ptp(feature) truth = result["truth"] pred = result["predicted"] plt.plot(feature, [abs(t - p) for t, p in zip(truth, pred)], "{}--".format(mark), label="Traj. {}".format(traj)) plt.legend(loc='center left', bbox_to_anchor=(1, 0.5)) plt.xlabel("Segment progress") plt.ylabel("Absolute error [s]") plt.title("Predicted arrival time errors for test trajectories (segment {})".format(i)) plt.savefig("forecasting/metrics/segment_{}.png".format(i), bbox_inches="tight") plt.close() #truths.extend(result["truth"]) #predicted.extend(result["predicted"]) #truths = np.array(truths) #predicted = np.array(predicted) #logger.info("RMSE: {}".format(round(np.sqrt(mean_squared_error(truths, predicted)), 2))) #logger.info("MAE: {}".format(round(mean_absolute_error(truths, predicted), 2))) if __name__ == "__main__": logger = setup_logging("file_lengths.py", "file_lengths.log") main(sys.argv[1:])
#!/usr/bin/python # # Unpopulates dynamic files for the given environment. # import helpers import traceback import argparse # Check parameters parser = argparse.ArgumentParser(description='Unpopulate dynamic files for an environment') parser.add_argument('env_name', type=str, help='Name of the environment') args = parser.parse_args() helpers.logger = helpers.setup_logging('unpopulate_env.log') helpers.log('Unopulating environment: %s' % args.env_name, as_banner=True, bold=True) try: helpers.unpopulate_env(args.env_name) except Exception, e: helpers.logger.debug('Failure: ' + str(e) + ", exception: " + traceback.format_exc().replace('\n', ' | ')) raise
plt.clf() plt.close() def plot_gp_grid_region(grid, gp_mean, gp_var): plt.plot(grid, gp_mean, lw=1, color="red") plt.fill_between(grid, gp_mean[:, 0] - 2 * np.sqrt(gp_var[:, 0]), gp_mean[:, 0] + 2 * np.sqrt(gp_var[:, 0]), color="red", alpha=0.2) def train_GP(X_tau, Y, session, number, gp_name): """GP which maps tau -> lng or lat.""" with gpflow.defer_build(): m = gpflow.models.GPR(X_tau, Y, kern=gpflow.kernels.RBF(1)) m.likelihood.variance = 1e-03 m.likelihood.variance.trainable = False m.compile() opt = gpflow.train.ScipyOptimizer() opt.minimize(m) logger.info("{} GP #{} trained.".format(gp_name, number)) session.save(BASE_DIR + "GP/model_{}_{}".format(number, gp_name), m) return m if __name__ == "__main__": logger = setup_logging("gps_var.py", "gps_var.log") BASE_DIR = "gps_var/" main(sys.argv[1:])
#!/usr/bin/python import json import traceback import argparse from argparse import RawTextHelpFormatter from helpers import setup_logging # Global default logger logger = setup_logging('health.log') def test_k8s(kubeconfig): # TODO - add k8s general health check pass def health(healthfile, logger=logger): # here we try to figure out who we want to monitor by looking at the passed in health file # and also to other parameters passed in which may change the behavior endpoints = [] try: logger.info("Checking with health file: %s" % healthfile) f = open(healthfile, "r") health_config = json.load(f) kubeconfig = health_config['k8s']['kubeconfig'] test_k8s(kubeconfig) logger.info('Health Check succeeded.') except Exception, e:
python interact_server.py --max_history 4 --top_p 0.8 --fp16 O2 --model_checkpoint runs/Jul19_14-38-58_ip-172-31-39-133_goood """ import collections import logging import os import random from argparse import ArgumentParser import irc3 from irc3.plugins.command import command os.sys.path.append("..") from interact_server import ModelAPI, TOPICS import logging from helpers import setup_logging setup_logging("irc_bot", level=logging.INFO) logger = logging.getLogger(__file__) logging.getLogger("zmqtest").setLevel(logging.INFO) @irc3.plugin class Plugin: requires = [ "irc3.plugins.core", "irc3.plugins.command", "irc3.plugins.log" ] def __init__(self, bot): self.bot = bot self.model_api = ModelAPI(port=bot.config["model_api"]["port"])