def sendEmail(): if request.json['EmailId'] is not None: symbols = symbol_list() save_attachment(symbols) msg = EmailMessage() msg['To'] = request.json['EmailId'] readConfig = read_config() msg['From'] = readConfig['SENDER_EMAIL'] pswd = readConfig['PASSWORD'] msg.set_content(readConfig['EMAIL_BODY']) msg['Subject'] = readConfig['EMAIL_SUBJECT'] list_of_files = os.listdir() if 'full_figure.pdf' in list_of_files: files = ['full_figure.pdf'] for file in files: with open(file, 'rb') as f: file_data = f.read() file_name = f.name msg.add_attachment(file_data, maintype='application', subtype='octet-stream', filename=file_name) with smtplib.SMTP_SSL('smtp.gmail.com', 465) as smtp: smtp.login(msg['From'], pswd) smtp.send_message(msg) emptyTargetFiles(list_of_files) speech = 'You will soon receive an email with an attached document that shows the performance of the stocks you asked about. Caio' return { "fulfillmentText": speech, "displayText": speech }
def __init__(self): file_paths, configs = read_config() if Camera.detector is None: print('[INFO] loading face detector...') Camera.detector = FaceDetector() if Camera.embedder is None: # load our serialized face embedding model from disk print('[INFO] loading embedder from {}'.format( file_paths['embedder_path'])) Camera.embedder = cv2.dnn.readNetFromTorch( file_paths['embedder_path']) if Camera.recognizer is None: # load the actual face recognition model along with the label encoder print('[INFO] loading face recognizer from {}'.format( file_paths['recognizer_path'])) Camera.recognizer = pickle.loads( open('output/recognizer.pickle', 'rb').read()) if Camera.le is None: print('[INFO] loading le from {}'.format(file_paths['le_path'])) Camera.le = pickle.loads(open('output/le.pickle', 'rb').read()) print('[INFO] Confidence value is set to {}'.format( configs['confidence'])) Camera.confidence = float(configs['confidence']) Camera.max_retry_count = int(configs['max_retry_count'])
def objective(args): config = config_reader.read_config(utils.abs_path_of("config/default.ini")) params = {} params['l1_reg'] = args['l1_reg'] params['l2_reg'] = args['l2_reg'] params['num_layers'] = int(args['num_layers']) params['layer_size'] = int(args['layer_size']) params['learning_rate'] = args['learning_rate'] params['batch_size'] = args['batch_size'] params['dropout_keep_probability'] = args['dropout_keep_probability'] params['validation_window'] = args['validation_window'] trows = csv_reader.read_csv_dataframe( config.get_rel_path("PATHS", "training_file")) vrows = csv_reader.read_csv_dataframe( config.get_rel_path("PATHS", "validation_file")) with open(config.get_rel_path("PATHS", "training_file")) as f: temporary_reader = csv.reader(f, delimiter=',') total_columns = len(next(temporary_reader)) params['total_columns'] = total_columns with tf.Graph().as_default(): loss = run_MLP(params, trows, vrows) return loss
def train_model(): file_paths, configs = read_config() # load the face embeddings print('[INFO] loading face embeddings...') data = pickle.loads(open(file_paths['detector_path'], 'rb').read()) # encode the labels print('[INFO] encoding labels...') le = LabelEncoder() labels = le.fit_transform(data['names']) # train the model used to accept the 128-d embeddings of the face and # then produce the actual face recognition print('[INFO] training model...') recognizer = SVC(C=1.0, kernel='linear', probability=True) recognizer.fit(data['embeddings'], labels) # write the actual face recognition model to disk f = open(file_paths['recognizer_path'], 'wb') f.write(pickle.dumps(recognizer)) f.close() # write the label encoder to disk f = open(file_paths['le_path'], 'wb') f.write(pickle.dumps(le)) f.close()
def run_MLP(params, trows, vrows): config = config_reader.read_config(utils.abs_path_of("config/default.ini")) if not os.path.isdir(config.get_rel_path("PATHS", "checkpoint_dir")): utils.mkdir_recursive(config.get_rel_path("PATHS", "checkpoint_dir")) iris_runner = mlp.FCNRunner(config, params) # trows, vrows, test_rows, config) iris_runner.bind_training_dataqueue_dataframe(trows, params) iris_runner.bind_validation_dataqueue_dataframe(vrows) if "TEST" in config: test_path = config.get_rel_path("TEST", "test_file") with tf.name_scope("test_data"): # TODO check this with Yanli # test_rows = csv_reader.read_test_csv_dataframe(test_path, int(config["TEST"]["batch_size"])) test_rows = csv_reader.read_csv_dataframe(test_path) iris_runner.bind_test_dataqueue_dataframe(test_rows) iris_runner.initialize() if "TRAINING" in config: valid_acc, train_loss, train_auc, valid_loss = iris_runner.run_training_dataframe( trows, vrows) if "TEST" in config: iris_runner.run_test(test_rows) iris_runner.close_session() return 1 - valid_acc, train_loss, train_auc, valid_loss
def main(number, shoulder=None, username=None, password=None, verbose=False, metadata=None): if not shoulder or not username or not password: HOME = os.environ['HOME'] config_file = os.environ.get('DATABASES_XML_EZID', HOME + '/.databases.xml') dbs = read_config(config_file=config_file) USERNAME = dbs['EZID']['USER'] PASSWORD = dbs['EZID']['PASSWORD'] SHOULDER = dbs['EZID']['SHOULDER'] shoulder = shoulder if shoulder else SHOULDER username = username if username else USERNAME password = password if password else PASSWORD ezid = EZIDClient(credentials=dict(username=username, password=password)) new_ids = [] for x in range(0, number): if not metadata: metadata = { '_profile': 'dc', } ez = ezid.mint(shoulder=shoulder, data=metadata) save_new_id(ez) new_ids.append(ez) metadata['_target'] = 'http://content.cdlib.org/' + ez resp = ezid.update(ez, metadata) print(resp) if verbose: print(ez) return new_ids
def optimize(): config = config_reader.read_config(utils.abs_path_of("config/default.ini")) space = { 'learning_rate': hyperopt.hp.choice('learning_rate', [0.0001, 0.001]), 'rnn_num_layers': hyperopt.hp.choice('rnn_num_layers', [2, 3, 4]), 'rnn_num_hidden': hyperopt.hp.choice('rnn_num_hidden', [200, 300, 400]), 'rnn_batch_size': hyperopt.hp.choice('rnn_batch_size', [50, 100, 200]), 'dropout_keep_probability': hyperopt.hp.choice('dropout_keep_probability', [0.5, 0.4, 0.6]), 'validation_window': hyperopt.hp.choice('validation_window', [5]) } best_model = hyperopt.fmin(objective, space, algo=hyperopt.tpe.suggest, max_evals=200) print(best_model) print(hyperopt.space_eval(space, best_model))
def make_all(location = 'components.yaml'): data = read_config(location) components = data.get('components', {}) templates = data.get('templates', {}) for key, value in components.items(): print 'building %s' % key make_component(key, value['destination']) for key, value in templates.items(): print 'building %s' % key make_template(key, value['destination'])
def main(): global trows, vrows config = config_reader.read_config(utils.abs_path_of("config/default.ini")) trows = csv_reader.read_csv_dataframe( config.get_rel_path("PATHS", "training_file")) vrows = csv_reader.read_csv_dataframe( config.get_rel_path("PATHS", "validation_file")) while True: optimize()
def run_RNN(params, trainX, trainY, ValidX, ValidY): config = config_reader.read_config(utils.abs_path_of("config/default.ini")) training_iters = config.getint("RNN_SEQUENCE_CLASSIFICATION", "rnn_training_iters") validation_interval = config.getint("PROCESS", "validation_interval") #train_batchsize = config.getint("RNN_SEQUENCE_CLASSIFICATION", "rnn_batch_size") #keep_prob = config.getfloat("TRAINING", "dropout_keep_probability", fallback=1.0) train_batchsize = params['rnn_batch_size'] keep_prob = params['dropout_keep_probability'] validation_window = params['validation_window'] train_len, rows, row_size = trainX.shape batches = batch_iter( zip(trainX, trainY), train_batchsize, training_iters) _, rows, row_size = trainX.shape num_classes = trainY.shape[1] with tf.name_scope('input'): data = tf.placeholder(tf.float32, [None, rows, row_size]) target = tf.placeholder(tf.float32, [None, num_classes]) dropout = tf.placeholder(tf.float32) model = SequenceClassification(data, target, dropout, params) sess = tf.Session() sess.run(tf.global_variables_initializer()) for i,batch in enumerate(batches): batch_x, batch_y = zip(*batch) _, train_accuracy = sess.run([model.optimize, model.accuracy], {data: batch_x, target: batch_y, dropout: keep_prob}) print('Batch {:2d} Train_accuracy {:3.1f}%'.format(i + 1, 100 * train_accuracy)) val_loss = [] v_count = 0 if (i+1) % validation_interval == 0: accuracy, summary_test = sess.run([model.accuracy, model.summary_op_test], {data: ValidX, target: ValidY, dropout: 1}) print("********************************************") print('Validation_accuracy {:3.1f}%'.format(100 * accuracy)) print("********************************************") loss = -1*accuracy val_loss.append(loss) v_count += 1 if v_count > validation_window: Validation_Loss = np.mean(val_loss[-validation_window:]) else: Validation_Loss = np.mean(val_loss) return Validation_Loss #if __name__ == '__main__': #main()
def main(): global trows, vrows, total_columns config = config_reader.read_config(utils.abs_path_of("config/default.ini")) with open(config.get_rel_path("PATHS", "training_file")) as f: temporary_reader = csv.reader(f, delimiter=',') total_columns = len(next(temporary_reader)) trows = csv_reader.read_csv_dataframe( config.get_rel_path("PATHS", "training_file")) vrows = csv_reader.read_csv_dataframe( config.get_rel_path("PATHS", "validation_file")) while True: optimize()
def run_MLP(params): config = config_reader.read_config(utils.abs_path_of("config/default.ini")) if not os.path.isdir(config.get_rel_path("PATHS", "checkpoint_dir")): utils.mkdir_recursive(config.get_rel_path("PATHS", "checkpoint_dir")) iris_runner = mlp.FCNRunner(config, params) # trows, vrows, test_rows, config) if "TRAINING" in config: with tf.name_scope("train_data"): #train_batch_size = config.getint("TRAINING", "batch_size") train_batch_size = params['batch_size'] stratified_task = config.get("TRAINING", "stratified_sampling", fallback="") trows = csv_reader.read_csv( config.get_rel_path("PATHS", "training_file"), train_batch_size, stratified_task, config) with tf.name_scope("validation_data"): vrows = csv_reader.read_csv( config.get_rel_path("PATHS", "validation_file"), config.getint("TRAINING", "validation_batch_size")) iris_runner.bind_training_dataqueue(trows, params) iris_runner.bind_validation_dataqueue(vrows) ''' if "TEST" in config: test_path = config.get_rel_path("TEST","test_file") with tf.name_scope("test_data"): test_rows = csv_reader.read_test_csv(test_path, int(config["TEST"]["batch_size"])) iris_runner.bind_test_dataqueue(test_rows) ''' iris_runner.initialize() if "TRAINING" in config: valid_loss = iris_runner.run_training() #if "TEST" in config: #iris_runner.run_test() return valid_loss
def grab_components(location = 'components.yaml'): data = read_config(location) if 'codegen' not in data: raise KeyError('codegen not defined in data.yaml') try: shutil.rmtree('.components') except OSError: #Don't worry if the directory doesn't exist pass codegen = data.get('codegen') components = data.get('components', {}) templates = data.get('templates', {}) grab(codegen, 'codegen') for key, value in components.items(): grab(value, key) for key, value in templates.items(): grab(value, key)
def main(): global logger logger = logging.getLogger("dir_batcher.py") logger.setLevel(logging.INFO) ch = logging.StreamHandler() ch.setLevel(logging.INFO) logger.addHandler(ch) if len(sys.argv) != 3: logger.error("Usage: dir_batcher.py <config.ini> <instance_dir>") from config_reader import read_config try: config = read_config(sys.argv[1]) except: logger.exception("Error while reading config file - aborting") exit(1) instance_dir = Path(sys.argv[2]) if not instance_dir.exists(): logger.error('Path "{}" does not exist - aborting'.format(sys.argv[2])) exit(1) batch = Batch(config) batches = Queue() CPU_COUNT = config['cpu_count'] finished_queue = Queue() for filename in instance_dir.glob('**/*'): if not filename.is_dir(): batches.put((batch, filename)) batch_count = batches.qsize() for _ in range(CPU_COUNT): Process(target=worker, args=(batches, finished_queue)).start() for _ in range(batch_count): print("finished instance {}".format(finished_queue.get())) for _ in range(CPU_COUNT): batches.put('STOP')
def objective(args): config = config_reader.read_config(utils.abs_path_of("config/default.ini")) params = {} params['l1_reg'] = args['l1_reg'] params['l2_reg'] = args['l2_reg'] params['num_layers'] = args['num_layers'] params['layer_size'] = args['layer_size'] params['learning_rate'] = args['learning_rate'] params['batch_size'] = args['batch_size'] params['dropout_keep_probability'] = args['dropout_keep_probability'] params['validation_window'] = args['validation_window'] trows = csv_reader.read_csv_dataframe( config.get_rel_path("PATHS", "training_file")) vrows = csv_reader.read_csv_dataframe( config.get_rel_path("PATHS", "validation_file")) with tf.Graph().as_default(): loss = run_MLP(params, trows, vrows) return loss
import logging from config_reader import read_config config_data = read_config() logging.basicConfig(filename='logs.log', level=logging.DEBUG, format='[%(asctime)s] %(levelname)s in %(module)s: %(message)s')
import n64_controller_input_reader as input_controller import asyncio import config_reader from player import Player from socket import error as SocketError import socket, errno import atexit #defines players = [] player = Player # Read Settings config = config_reader.read_config('settings.ini') # Helper to create the number of players defined in the settings def createPlayerFromConfig(): curr_numb = 0 for port in config['Controller']: global player input_port = config['Controller'][port] if input_port: player.number = curr_numb player.device = input_controller.init_input_device(input_port) players.append(player) curr_numb += 1 # Start # create Player async def createPlayerFromConfig():
from batch import Batch from plotter import Plotter if __name__ == "__main__": import sys if len(sys.argv) < 3: print("Usage: run_and_show.py <config_file> [instance_file, ...]") exit(127) from config_reader import read_config config = read_config(sys.argv[1]) batch = Batch(config) for filename in sys.argv[2:]: logfile = batch.run(filename) if logfile is not None: plotter = Plotter(logfile) plotter.prepare_plot() plotter.show_plot()
keras_weights_file = args.model frame_rate = args.frame_rate process_speed = args.process_speed host = args.service_host port = args.service_port post_path = args.post_path out_mp4_path = args.save_mp4_path mirror = args.mirror # load model # The authors of the original model don't use vgg normalization (subtracting mean) on input images model = get_testing_model() model.load_weights(keras_weights_file) # load config params, model_params = read_config() # Video reader cam = cv2.VideoCapture(device) # CV_CAP_PROP_FPS cam.set(cv2.CAP_PROP_FPS, frame_rate) input_fps = cam.get(cv2.CAP_PROP_FPS) print("Running at {} fps.".format(input_fps)) ret_val, orig_image = cam.read() width = orig_image.shape[1] height = orig_image.shape[0] factor = 0.3 out = None
import os import tensorflow as tf import config_reader import csv_reader import mlp import utils import sys, os config = config_reader.read_config(utils.abs_path_of("config/default.ini")) if not os.path.isdir(config.get_rel_path("PATHS", "checkpoint_dir")): utils.mkdir_recursive(config.get_rel_path("PATHS", "checkpoint_dir")) iris_runner = mlp.FCNRunner(config) # trows, vrows, test_rows, config) if "TRAINING" in config: with tf.name_scope("train_data"): train_batch_size = config.getint("TRAINING", "batch_size") stratified_task = config.get("TRAINING", "stratified_sampling", fallback="") trows = csv_reader.read_csv( config.get_rel_path("PATHS", "training_file"), train_batch_size, stratified_task, config) with tf.name_scope("validation_data"): vrows = csv_reader.read_csv( config.get_rel_path("PATHS", "validation_file"), config.getint("TRAINING", "validation_batch_size"))
def get_parts_order(): _, model_params = read_config() return model_params['part_str']
import os import tensorflow as tf import config_reader import csv_reader import mlp import utils config = config_reader.read_config("config/default.ini") if not os.path.isdir(config.get("PATHS", "checkpoint_dir")): utils.mkdir_recursive(config.get("PATHS", "checkpoint_dir")) iris_runner = mlp.FCNRunner(config) # trows, vrows, test_rows, config) if "TRAINING" in config: with tf.name_scope("train_data"): train_batch_size = config.getint("TRAINING", "batch_size") stratified_task = config.get("TRAINING", "stratified_sampling", fallback="") trows = csv_reader.read_csv(config.get("PATHS", "training_file"), train_batch_size, stratified_task, config) with tf.name_scope("validation_data"): vrows = csv_reader.read_csv( config.get("PATHS", "validation_file"), config.getint("TRAINING", "validation_batch_size")) iris_runner.bind_training_dataqueue(trows) iris_runner.bind_validation_dataqueue(vrows)
HOME = os.environ['HOME'] #setup some globals for file locations and names DIR_EXE = HOME + '/branches/production/voro/batch-bin/' DIR_DATA_IN = HOME + '/data/in/oac-ead/' DIR_WORKSPACE = HOME + '/workspace/' DIR_IN = HOME + '/users/in/' DIR_APACHE_WEBDAV_CONF = HOME + '/users/apache/' DIR_EAD_TEST = HOME + '/workspace/test-oac/submission' #subdirs for data, from DIR_DATA_IN root path DIR_SUB_REPO = 'repodata' DIR_SUB_SUBMISSION = DIR_DATA_IN + 'submission' #DIR_SUB_EAD_PRODUCTION = 'prime2002' DIR_LOGGING = HOME + '/log/update_voroEAD/' db = read_config() DSC_DATABASE = os.environ['DSC_DATABASE'] DATABASE_HOST = db[DSC_DATABASE]['HOST'] DATABASE_NAME = db[DSC_DATABASE]['NAME'] DATABASE_USER = db[DSC_DATABASE]['USER'] DATABASE_PASSWORD = db[DSC_DATABASE]['PASSWORD'] DATABASE_PORT = db[DSC_DATABASE]['PORT'] FILE_VOROUSERS = 'voro.users.txt' # HARDCODED in build.users.pl FILE_VOROGROUPS = 'voro.groups.txt' # HARDCODED in build.users.pl FILE_APACHE_DIGEST = 'users.digest' FILE_APACHE_GROUPS = 'groups' # HARDCODED in build.users.pl FILE_APACHE_DAV_CONF = 'DAV.conf' # HARDCODED in build.users.pl FILE_LOGGING = 'voro_update.log'
import os from config_reader import read_config #To make project independent of absolute path: PROJECT_ROOT = PROJECT_PATH = os.path.abspath(os.path.split(__file__)[0]) SEND_BROKEN_LINK_EMAILS = True DATABASES = read_config() # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. # If running in a Windows environment this must be set to the same as your # system time zone. TIME_ZONE = 'America/Los_Angeles PST8PDT SystemV/PST8PDT US/Pacific US/Pacific-New' # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html LANGUAGE_CODE = 'en-us' SITE_ID = 1 # If you set this to False, Django will make some optimizations so as not # to load the internationalization machinery. USE_I18N = True # Absolute path to the directory that holds media. # Example: "/home/media/media.lawrence.com/" MEDIA_ROOT = os.path.join([PROJECT_PATH, 'site_media'])
from base64 import b64encode import datetime import hashlib import inspect import redis import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) import news_api_client import config_reader as reader from cloudAMQP_client import CloudAMQPClient config = reader.read_config() SCRAPE_NEWS_TASK_QUEUE_URL = config.get('PIPELINE', 'SCRAPE_QUEUE_URL') SCRAPE_NEWS_TASK_QUEUE_NAME = config.get('PIPELINE', 'SCRAPE_QUEUE_NAME') SLEEP_TIME_IN_SECONDS = config.getint('PIPELINE', 'MONITOR_SLEEP_TIME') REDIS_HOST = config.get('REDIS', 'REDIS_HOST') REDIS_PORT = config.getint('REDIS', 'REDIS_PORT') # it will expires in 3 days NEWS_TIME_OUT_IN_SECONDS = 3600 * 24 * 3 NEWS_SOURCES = [ 'bbc-news', 'bbc-sport', 'business-insider', 'cnn', 'entertainment-weekly', 'espn', 'fox-news', 'ign', 'techcrunch', 'the-new-york-times', 'the-wall-street-journal', 'the-washington-post' ]
#!/bin/env python import os import os.path import datetime import pysqlite2._sqlite as sqlite import lxml.etree as ET import glob import csv import MySQLdb from config_reader import read_config HOME = os.environ['HOME'] DIR_ROOT = HOME + '/data/in/oac-ead/prime2002/' DB_FILE = HOME + '/indexes/sqlite3/ois.sqlite3' db = read_config() DB_MYSQL_NAME = db['default-ro']['NAME'] DB_MYSQL_USER = db['default-ro']['USER'] DB_MYSQL_PASSWORD = db['default-ro']['PASSWORD'] DB_MYSQL_HOST = db['default-ro']['HOST'] DB_MYSQL_PORT = db['default-ro']['PORT'] DIR_ORPHANS = os.path.realpath(__file__) + '/orphans' def run_samples(): DIR_SAMPLES_ROOT = HOME + '/data/in/oac-ead/prime2002/' samples = [("A","ark:/13030/tf10000759"), ("B","ark:/13030/kt200014h4"), ("C","ark:/13030/kt196nc93r"), ("D","ark:/13030/tf6r29p0kq"),
run_tasks.put(run_task) for _ in range(len(self.algorithm_configs * self.run_count)): lines = finished_queue.get() logger.write(lines) else: for algorithm_config in self.algorithm_configs: runner = Runner(instance, logger, self.iterations, algorithm_config) for run_nr in range(self.run_count): runner.run_algorithm(run_nr) logger.close() return logfile_name if __name__ == "__main__": import sys, argparse parser = argparse.ArgumentParser() parser.add_argument('config_file') parser.add_argument('instance_file') parser.add_argument('--stdout', action='store_true') args = parser.parse_args() from config_reader import read_config batch = Batch(read_config(args.config_file), log_to_stdout=args.stdout) batch.run(args.instance_file)