def user_menu(session, directory): """ Show the menu in the first tmux window. Retrieve the information about the devices to use for the analysis and the folder containig the apps to analyze.""" devices = {} uManagers = [] logger = util.init_logger(config.configFile, 'Manager') logger.info('BareDroid init') devices = initDevice(logger) windows = divide_et_impera(devices, session) #watch log and errors logFolder = os.path.join(os.getcwd(),\ util.read_config(config.configFile, 'Log', 'folder')) windows[1].list_panes()[0].send_keys('tail -F %smanager.log'%logFolder) windows[1].list_panes()[1].send_keys('tail -F %smanager_error.log'%logFolder) ans=True while ans: print '-----------------' print (""" 1) List devices 2) Start Analysis 3) Stop Analysis 4) Select Devices q) Quit m) Menu """) print '' ans=raw_input(">> ") if ans=="1": listDevices(config.deviceInfoFile, logger) elif ans=="2": folder = createFolder(logger) #initialize uManagers = initAnalysis(directory,\ int(util.read_config(config.configFile,'Analysis', 'apps')),\ logger, devices) #start startAnalysis(uManagers, folder, windows, logger) elif ans=="3": stopAnalysis(uManagers) elif ans=="4": #TODO select devices print 'TODO' elif ans=='m' or ans == 'M': print '' elif ans=='q' or ans == 'Q': check(directory, logger) logger.info('exit') #print print_on_window(windows[0], '\nExit') quit() elif ans !="": print("\n Not valid, try again...") ans = True else: print("\n Not valid, try again...") ans = True
def watch_update(self): force, new_version, update_fn, fn_md5 = unpackb(self.sock.recv(), encoding='utf-8') cur_version = read_config()['version'] if force == 0 and new_version == cur_version: log.logger.info('current version is the latest version') return retry = 5 index = 1 while index <= retry: # wget update package and check md5 ## ## try: shutil.copy(update_fn, os.path.join(prog_dir(), '..')) log.logger.info('copy update file done') except Exception, e: log.logger.error(e) index += 1 continue if fn_md5 != get_file_md5(os.path.join(prog_dir(), '..', os.path.basename(update_fn))): log.logger.error("file md5 no match") index += 1 continue else: break
def generate_goal_states(): config = read_config("config.yaml") agent_config = config['Agent'] network_config = agent_config['Network'] agent = DCAAgent(agent_config, network_config) env = DCAEnv() new_goal_states=[] new_goal_path=[] start_time=time.time() print('Start generating new goal states (5 steps away)') for T in range(9, 10): n_games = 10000 num_game = 1 while num_game <= n_games: temp, _ = agent.collect_data(env, T) new_goal_state=temp[0][-1].tolist() if new_goal_state not in new_goal_states: new_goal_states.append(new_goal_state) new_goal_path.append(temp[0]) num_game = num_game + 1 print('Finish generating new goal states (5 steps away) @ %s'%(time.time()-start_time)) print('# of new goal states is %s'%(len(new_goal_states))) return new_goal_states,new_goal_path
def main(self): self.setup_basic_logging() parser = self.create_parser() (options, args) = parser.parse_args() cfg = read_config(options.config) self.setup_logging(cfg) self.handle_args(parser, cfg, options, args)
def launch_lab(conn, user_vpc, lab): """Execute a lab configuration""" path = LAB_DIR + lab + '/scripts/' prompt, answer = _prompt_config(lab, path) # import lab configs labmod = imp.load_source('labmod', path + prompt[answer] + '.py') labmod.pre_process() cfg = util.read_config(LAB_DIR + lab + '/instances.cfg') # prompt for any dynamic configuration options for instance in cfg['instance']: for k, v in instance.iteritems(): if str(v).startswith('PROMPT:'): instance[k] = raw_input('{0}: '.format(v.split(':')[1])) if str(v).startswith('PROMPT#:'): instance[k] = int(raw_input('{0}: '.format(v.split(':')[1]))) for device in instance['device']: for k, v in device.iteritems(): if str(v).startswith('PROMPT:'): device[k] = raw_input('{0}: '.format(v.split(':')[1])) if str(v).startswith('PROMPT#:'): device[k] = int(raw_input('{0}: '.format(v.split(':')[1]))) # connection and required info security_groups = conn.get_all_security_groups( filters={'vpc-id': user_vpc.id}) subnets = conn.get_all_subnets(filters={'vpc-id': user_vpc.id}) # launch inst.launch_instances(conn, user_vpc, prompt[answer], lab, labmod, cfg, security_groups, subnets) labmod.post_process()
def main(): # Read config file cfg = util.read_config('config/digit.yaml') # Load digit data from dataset x_train, x_test, y_train, y_test = load_data(cfg['dataset']) x_train, y_train = util.shuffle_data(x_train, y_train) x_test, y_test = util.shuffle_data(x_test, y_test) # Default model name as loaded from file, overwritten if training model_name = cfg['nn']['model_name'] model_dir = cfg['nn']['model_dir'] with tf.Session() as sess: if cfg['nn']['train']: # Train network on our training data print('[ANN] Training new network...') model, model_name = train_network(sess, x_train, y_train, cfg) else: print('[ANN] Testing network {0}...'.format(model_name)) model = util.load_model( os.path.join(model_dir, model_name + "_model")) # Test network on our testing data results = test_network(sess, model, x_test, y_test, cfg) # TODO: Tristan to reimplement analyse results to get confusion matrix and roc curve conf_mat = {} # conf_mat = util.analyse_results(y_test, results) util.store_results(conf_mat, os.path.join(model_dir, model_name + "_cm"))
def main(): logger.info("Starting google music uploader") pidfile = None if len(sys.argv) > 1: if sys.argv[1] == "--pidfile": if len(sys.argv) < 3: logger.error("Missing pidfile path") return pidfile = sys.argv[2] if pidfile: if not util.make_sure_path_exists(os.path.dirname(pidfile)): logger.warning("Error creating pidfile directory %s" % os.path.dirname(pidfile)) return with open(pidfile, "w+") as f: logger.debug("Writing pidfile to %s" % pidfile) f.write(str(os.getpid())) config = util.read_config(DirInfo.AppConfig) session_opts = { 'session.type': 'memory', 'session.auto': 'true' } app = SessionMiddleware(bottle.app(), session_opts) run(app=app, host='0.0.0.0', port=config['PORT'], debug=True)
def launch_lab(conn, user_vpc, lab): """Execute a lab configuration""" path = LAB_DIR + lab + '/scripts/' response = _prompt_config(lab, path) # import lab configs labmod = imp.load_source('labmod', path + response + '.py') labmod.pre_process() cfg = util.read_config(LAB_DIR + lab + '/instances.cfg') # prompt for any dynamic configuration options for instance in cfg['instance']: for k, v in instance.iteritems(): if str(v).startswith('PROMPT:'): instance[k] = raw_input('{0}: '.format(v.split(':')[1])) if str(v).startswith('PROMPT#:'): instance[k] = int(raw_input('{0}: '.format(v.split(':')[1]))) for device in instance['device']: for k, v in device.iteritems(): if str(v).startswith('PROMPT:'): device[k] = raw_input('{0}: '.format(v.split(':')[1])) if str(v).startswith('PROMPT#:'): device[k] = int(raw_input('{0}: '.format(v.split(':')[1]))) # connection and required info security_groups = conn.get_all_security_groups(filters = {'vpc-id': user_vpc.id}) subnets = conn.get_all_subnets(filters = {'vpc-id': user_vpc.id}) # launch inst.launch_instances(conn, user_vpc, lab, labmod, cfg, security_groups, subnets) labmod.post_process()
def launch_lab(conn, user_vpc, lab): """Execute a lab configuration""" path = LAB_DIR + lab + "/scripts/" prompt, answer = _prompt_config(lab, path) # import lab configs labmod = imp.load_source("labmod", path + prompt[answer] + ".py") labmod.pre_process() cfg = util.read_config(LAB_DIR + lab + "/instances.cfg") # prompt for any dynamic configuration options for instance in cfg["instance"]: for k, v in instance.iteritems(): if str(v).startswith("PROMPT:"): instance[k] = raw_input("{0}: ".format(v.split(":")[1])) if str(v).startswith("PROMPT#:"): instance[k] = int(raw_input("{0}: ".format(v.split(":")[1]))) for device in instance["device"]: for k, v in device.iteritems(): if str(v).startswith("PROMPT:"): device[k] = raw_input("{0}: ".format(v.split(":")[1])) if str(v).startswith("PROMPT#:"): device[k] = int(raw_input("{0}: ".format(v.split(":")[1]))) # connection and required info security_groups = conn.get_all_security_groups(filters={"vpc-id": user_vpc.id}) subnets = conn.get_all_subnets(filters={"vpc-id": user_vpc.id}) # launch inst.launch_instances(conn, user_vpc, prompt[answer], lab, labmod, cfg, security_groups, subnets) labmod.post_process()
def create_amis(conn, user_vpc, lab_tag): """Create all images for lab""" running_labs = get_running_labs(conn, user_vpc) cfg = util.read_config(LAB_DIR + lab_tag.rsplit('-', 1)[0] + '/instances.cfg') instances = inst.get_vpc_instances(conn, user_vpc) lab_instances = [] for instance in instances: if 'Lab' in instance.tags: if instance.tags['Lab'] == lab_tag and instance.tags['AMI-Build'] == 'True': lab_instances.append(instance) print "Creating AMI's for lab: {0}".format(lab_tag) for instance in lab_instances: name_tag = TRAINER + '-{0}-'.format(VPC) + \ '{0}-'.format(lab_tag.rsplit('-', 1)[0]) + \ '{0}-'.format(instance.tags['Script']) + \ '{0}'.format(instance.tags['AMI-Key']) ami = conn.create_image(instance.id, name_tag, '{0} lab AMI'.format(lab_tag.rsplit('-', 1)[0])) # tags tags = {'Name': '{0}'.format(name_tag), 'AMI-Key': '{0}'.format(instance.tags['AMI-Key']), 'Lab': '{0}'.format(lab_tag.rsplit('-', 1)[0])} conn.create_tags(ami, tags) print "AMI build trigger completed successfully ..." print "Note: It could take 15-20 minutes for the AMI build to complete."
def main(): util.read_config() pygame.mixer.pre_init(44100, -16, 1, 1024 * 1) pygame.init() controller = gamecontroller.GameController() pygame.display.update() def process_events(events, controller): for event in events: if event.type == QUIT: sys.exit(0) if event.type == MOUSEMOTION: for b in controller.circles.itervalues(): if (b.is_pressed): b.rect.move_ip(event.rel) if event.type == MOUSEBUTTONDOWN: pos = pygame.mouse.get_pos() clicked = [ s for s in controller.circles.itervalues() if s.rect.collidepoint(pos) ] for b in clicked: b.pressed() if event.type == MOUSEBUTTONUP: for b in controller.circles.itervalues(): b.unpressed() if event.type == USEREVENT + 1: #only redraw needed pass if event.type == USEREVENT + 2: #only redraw needed pass if event.type == KEYDOWN: controller.process_key(event.key) while controller.running: process_events([pygame.event.wait()] + pygame.event.get(), controller) controller.redraw() pygame.display.flip() if controller.playing: controller.stop_fgsegmentation() controller.picam.close() pygame.quit()
def __init__(self, config_file="etc/client.conf", logfile=""): self.config = read_config(app_abs_path(config_file)) if not logfile: logfile = self.config['log_file'] self.loglevel = logging._levelNames[self.config['log_level']] self._setup_logging() if self._no_handlers: self._setup_handlers(logfilepath=app_abs_path(logfile))
def connect_to_database(): # connect logging.info('Connecting to the mindful database') conn = psycopg2.connect(host=util.read_config('postgresql', 'host'), database=util.read_config('postgresql', 'database'), user=util.read_config('postgresql', 'user'), password=util.read_config('postgresql', 'password')) cur = conn.cursor() yield cur # close the connection cur.close() conn.commit() conn.close() logging.info('Database connection closed')
def main(): util.read_config() pygame.mixer.pre_init(44100, -16, 1, 1024 * 1) pygame.init() controller = gamecontroller.GameController() pygame.display.update() def process_events(events, controller): for event in events: if event.type == QUIT: sys.exit(0) if event.type == MOUSEMOTION: for b in controller.circles.itervalues(): if b.is_pressed: b.rect.move_ip(event.rel) if event.type == MOUSEBUTTONDOWN: pos = pygame.mouse.get_pos() clicked = [s for s in controller.circles.itervalues() if s.rect.collidepoint(pos)] for b in clicked: b.pressed() if event.type == MOUSEBUTTONUP: for b in controller.circles.itervalues(): b.unpressed() if event.type == USEREVENT + 1: # only redraw needed pass if event.type == USEREVENT + 2: # only redraw needed pass if event.type == KEYDOWN: controller.process_key(event.key) while controller.running: process_events([pygame.event.wait()] + pygame.event.get(), controller) controller.redraw() pygame.display.flip() if controller.playing: controller.stop_fgsegmentation() controller.picam.close() pygame.quit()
def launcher(): """ Launch all commands """ config = util.read_config() for task in config: if 'cmd' in config[task]: if not 'enable' in config[task]: config[task]['enable'] = True if config[task]['enable']: launch(config[task])
def main(): # Read config file cfg = util.read_config('config/mushroom.yaml') # Load mushroom data from dataset x_train, x_test, y_train, y_test = load_data(cfg['dataset'], cfg['test_ratio_offset']) x_train, y_train = util.shuffle_data(x_train, y_train) x_test, y_test = util.shuffle_data(x_test, y_test) # Default model name as loaded from file, overwritten if training model_name = cfg['nn']['model_name'] model_dir = cfg['nn']['model_dir'] with tf.Session() as sess: if cfg['nn']['train']: # Train network on our training data print('[ANN] Training new network...') model, model_name, train_stats = train_network( sess, x_train, y_train, cfg) else: loaded_results = util.load_results( os.path.join(model_dir, model_name + "_cm")) # Setup our continous plot plt.title('Error vs Epoch') plt.plot(loaded_results['train_stats']['train_errors'], color='r', label='training') plt.plot(loaded_results['train_stats']['valid_errors'], color='b', label='validation') plt.xlabel('Epoch') plt.ylabel('Error') plt.legend() plt.grid() plt.show() print('[ANN] Testing network {0}...'.format(model_name)) model = util.load_model( os.path.join(model_dir, model_name + "_model")) train_stats = loaded_results['train_stats'] # Test network on our testing data results = test_network(sess, model, x_test, y_test, cfg) conf_mat, sk_fpr, sk_tpr, roc_auc = util.analyse_results( y_test, results) print('[ANN] ROC Area Under Curve: {0:.2f}'.format(roc_auc)) plot_roc(sk_fpr, sk_tpr, roc_auc) results_to_save = { 'conf_mat': conf_mat, 'train_stats': train_stats, 'roc_auc': float(roc_auc) } util.store_results(results_to_save, os.path.join(model_dir, model_name + "_cm"))
def interpretArgs(): global debugOutput settings = {} settingsFile = None info = { 'mid': '-', 'output': '-' } lookupMode = False args = sys.argv[:] args.pop(0); while args: val = args.pop(0) if val == '-h' or val == '--help': usage() elif val == '-p': if not args: print('-p requires argument', file=sys.stderr) usage() info['mid'] = args.pop(0) elif val == '-c': if not args: print('-c requires argument', file=sys.stderr) usage() settingsFile = args.pop(0) util.read_config(settings, settingsFile, debugOutput) elif val == '-o': if not args: print('-o requires argument', file=sys.stderr) usage() info['output'] = args.pop(0) elif val == '--lookup': lookupMode = True break elif val == '--debug': debugOutput = True else: print('illegal argument '+val, file=sys.stderr) usage() return (settings, settingsFile, info, lookupMode, args)
def run_tracking(pr_df, commit_df, srcpath, output, repo_name, home_url, recent_authors): config = util.read_config('summary') email = config['email'] day = config['day'] today = datetime.datetime.today() is_today = today.strftime('%A') == day objectives = config['objectives'] authors = config['authors'] monitors_str = '<tr>' \ '<td data-label="Report to">{email}</td>' \ '<td data-label="Day">{day}</td>' \ '<td data-label="Objectives">{objectives}</td>' \ '<td data-label="Authors">{authors}</td>' \ '</tr>'.format(email=email, day=day, objectives=objectives, authors=authors) with open(os.path.join(srcpath, 'templates', 'tracking.html'), 'r') as f: tracking_text = f.read() target_path = os.path.join(output, 'tracking.html') with open(target_path, 'w') as f: f.write(tracking_text.format(name=repo_name, nav=htmls.compute_nav(home_url, view='', timeframe='', recent_authors=recent_authors), home_url=home_url, timeframe='', view='', author='', monitors=monitors_str)) if is_today: month_start = today - datetime.timedelta(days=14) week_start = today - datetime.timedelta(days=7) last_month_prs = pr_df[(pr_df.index >= month_start) & (pr_df.index < week_start)] last_week_prs = pr_df[pr_df.index >= week_start] last_week_commits = commit_df[commit_df.index >= week_start] last_mean = last_month_prs.no_reviews.mean() this_mean = last_week_prs.no_reviews.mean() review_text = 'The mean reviews per pull request was {avg_review_week:.2f}, ' \ '{status} previous week\'s which saw a mean rate of {avg_review_month:.2f}'.format( avg_review_week=this_mean, avg_review_month=last_mean, status='about the same as' if last_mean - 0.1 < this_mean < last_mean + 0.1 else 'higher than' if this_mean > last_mean else 'lower than' ) with open(os.path.join(srcpath, 'templates', 'summary.html'), 'r') as f: page_text = f.read() page_text = page_text.format( no_prs=last_week_prs.shape[0], no_commits=last_week_commits.shape[0], no_lines=last_week_commits.insertions.sum() + last_week_commits.deletions.sum(), no_code_lines=last_week_commits.code_changes.sum(), review_text=review_text, name=repo_name, link=home_url ) subject = 'git-quality weekly summary' email_summary(email, page_text, subject)
def notification_message(song): chosen_str = " chosen by {username}".format( username=song.username) if song.username != "Unknown" else "" server_url = SERVER_URL if SERVER_URL != "None" else "http://{hostname}:{port}".format( hostname=socket.gethostname(), port=util.read_config( 'server', 'port', type=int, default=util.DEFAULT_PORT)) msg = "The song of the day is: {song_name}{chosen_str}: {url} \\n" \ "To add your songs please visit {server_url}, " \ "or to provide bug reports or feature requests please visit {repo_url}" \ "\\n{release_notes}".format(song_name=song.title, chosen_str=chosen_str, url=song.url, server_url=server_url, repo_url=REPO_URL, release_notes="") return msg
def make_submission(inference_config="config/infer.yml", run_base_dir="wandb", save_local=False): submission_folder = pathlib.Path(datetime.now().strftime("%Y%m%d-%H%M")) submission_dir = "submissions" / submission_folder submission_dir.mkdir(exist_ok=True) dir_util.copy_tree('src', str(submission_dir / 'src')) dir_util.copy_tree('prodia', str(submission_dir / 'src' / 'prodia')) with open(submission_dir / 'dataset-metadata.json', 'w') as f: json.dump( { "title": str(submission_folder), "id": f"xvrhlt/{submission_folder}", "licenses": [{ "name": "CC0-1.0" }] }, f) shutil.copy(inference_config, submission_dir) config = util.read_config(inference_config) run_base_dir = pathlib.Path(run_base_dir) dst_run_dir = submission_dir / "runs" dst_run_dir.mkdir() for run in config['runs']: if isinstance(run, str): run_id = run run_kwargs = {} elif isinstance(run, dict): run_id = run['run_id'] run_kwargs = run src_path = run_base_dir / run_id dst_run_dir = dst_run_dir / run_id dst_run_dir.mkdir() shutil.copy(str((src_path / "config.yaml")), str(dst_run_dir / "config.yaml")) if 'model_pth' in run_kwargs: model = src_path / run_kwargs['model_pth'] else: model = max(src_path.rglob('*.ckpt')) (dst_run_dir / 'model.ckpt').write_bytes(model.read_bytes()) kaggle.api.dataset_create_new(submission_dir, dir_mode='zip') if save_local: shutil.make_archive(submission_folder, 'zip', root_dir='submissions') dir_util.remove_tree(submission_dir)
def lidar_logger(output_directory): """ LIDAR Logger """ global LIDAR_STATUS, LIDAR_DATA port_name = '/dev/lidar' lidar = None # Configure config = util.read_config() while not util.DONE: try: lidar = RPLidar(port_name) print(lidar.get_info()) print(lidar.get_health()) # Open the output file timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M") with open(os.path.join(output_directory,timestamp+"_lidar.csv"), "w") as lidar_output: lidar_output.write("%s %s %s *\n" % (config['time'], "VERSION", json.dumps({"class": "VERSION", "version": util.DATA_API}))) lidar_output.write("%s %s %s *\n" % (config['time'], config['class'], json.dumps(config))) for _, scan in enumerate(lidar.iter_scans(max_buf_meas=1500)): lidartime = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ") data = [] for (_, angle, distance) in scan: if distance > 0: data.append((int(angle)%360, int(distance))) lidar_data = { 'class': 'LIDAR', 'device': 'A1M8', 'time': lidartime, 'scan': data, } lidar_output.write("%s %s %s *\n" % (lidar_data['time'], lidar_data['class'], json.dumps(lidar_data))) LIDAR_DATA = lidar_data LIDAR_STATUS = True if util.DONE: break except KeyboardInterrupt: util.DONE = True except Exception as ex: print("LIDAR Logger Exception: %s" % ex) time.sleep(util.ERROR_DELAY) if lidar is not None: lidar.stop() lidar.stop_motor() lidar.disconnect() LIDAR_STATUS = False time.sleep(util.ERROR_DELAY)
def email_summary(email_address, content, subject): """ Emails award winners to the given email address :param str email_address: the address to email """ from_address = util.read_config('email')['from'] msg = MIMEMultipart('alternative') msg['Subject'] = subject msg['From'] = from_address msg['To'] = email_address msg.attach(MIMEText(html2text.html2text(content), 'plain')) msg.attach(MIMEText(content, 'html')) s = smtplib.SMTP('localhost') for e in email_address.split(','): s.sendmail(from_address, [e], msg.as_string()) s.close()
def lpcm_logger(output_directory): """ LPCM Capture Wrapper """ global LPCM_DATA # Configure config = util.read_config() timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M") with open(os.path.join(output_directory, timestamp + "_lpcm.csv"), "w") as lpcm_output: lpcm_output.write("%s %s %s *\n" % (config['time'], "VERSION", json.dumps({ "class": "VERSION", "version": util.DATA_API }))) lpcm_output.write( "%s %s %s *\n" % (config['time'], config['class'], json.dumps(config))) while not util.DONE: try: timestamp = datetime.datetime.now() filename = timestamp.strftime("%Y%m%d%H%M") LPCM_DATA = { "class": "LPCM", "time": timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ") } for channel in ["left", "right"]: capture_file = filename + "_" + channel + ".wav" if os.path.isfile( os.path.join(output_directory, capture_file)): LPCM_DATA[channel] = capture_file if os.system("./lpcm_collect.sh %s %s \"%s\"" % (output_directory, filename, config['lpcm']['arecord'])) != 0: time.sleep(util.ERROR_DELAY) else: lpcm_output.write("%s %s %s *\n" % ( LPCM_DATA['time'], LPCM_DATA['class'], json.dumps(LPCM_DATA), )) except KeyboardInterrupt: util.DONE = True except Exception as ex: print("LPCM Logger Exception: %s" % ex) time.sleep(util.ERROR_DELAY)
def create_leaderboard(config_path, output_path): """ Create a leaderboard for the given ``config`` file. :param config_path: `str` representing path to config file. """ config = read_config(config_path) title = config.get('title', '') if not title: print("Leaderboard title is mandatory. Please provide the same in " "your config file") sys.exit(1) tabs = config.get('tabs', []) pills = [] for i, tab in enumerate(tabs): pill = { 'pill_id': 'pill%d' % i, 'status': 'active' if i == 0 else '', 'nav_id': 'tab%d' % i, 'title': tab['title'] } pills.append(pill) tables = [] for i, tab in enumerate(tabs): table = { 'status': 'show active' if i == 0 else '', 'nav_id': 'tab%d' % i, 'pill_id': 'pill%d' % i, 'type': tab['type'], 'table': '' } data = _read_csv(tab['data']) print(len(data)) tmpl = Template(_read_template(tab['type'])) tbl_html = tmpl.render(data=data) table['table'] = tbl_html tables.append(table) index_tmpl = Template(_read_template('index')) index_html = index_tmpl.render(title=title, pills=pills, tables=tables) with open(output_path, 'w') as f: f.write(index_html)
def main(): from torch.nn import init config = util.read_config('../configs/process.yaml') # get_temp_vocab(config) vocab = data.Vocab(config.vocab_path, max_size=config.max_size) # vocab.build_vectors(config.pre_word_embedding_path, 300, unk_init=init.xavier_uniform) if config.save: torch.save(vocab, config.vocab_path_50) val_data = DocDataset(config.val_path, vocab, config) test_data = DocDataset(config.test_path, vocab, config) if config.save: torch.save(val_data, config.val_data_path) torch.save(test_data, config.test_data_path) train_data = DocDataset(config.train_path, vocab, config) if config.save: torch.save(train_data, config.train_data_path)
def init_db(type): result = None if type == 'MYSQL': host = util.read_config('MYSQL', 'host') port = int(util.read_config('MYSQL', 'port')) user = util.read_config('MYSQL', 'username') password = util.read_config('MYSQL', 'password') database = util.read_config('MYSQL', 'database') result = MySQL(host=host, port=port, user=user, password=password, database=database) elif type == 'SQLSERVER': host = util.read_config('SQLSERVER', 'host') user = util.read_config('SQLSERVER', 'username') password = util.read_config('SQLSERVER', 'password') database = util.read_config('SQLSERVER', 'database') result = MSSQL(host=host, user=user, password=password, database=database) return result
def univ_collection(target_alias=None): ''' this function use api defined in util.py, collect faculty lists of universities provided in config.json and covert the Chinese to name represent by pinyin followed by the full name of this university params: target_alias: str, list of str, or None. the institutions we would like to find connection for, if set to None find for all institutions supported by config.json return value: defaultdict, each key is the university's name, and the value is a list of all faculty members' pinyin names with this alias ''' configs = util.read_config(filename='config/institutions.json') print('find connection for {}'.format( target_alias if target_alias else 'all')) univ_faculty_collection = util.crawl_faculty_list(configs, target_alias) univ_faculty_collection = util.extract_name(univ_faculty_collection) univ_faculty_collection = util.name_to_pinyin(univ_faculty_collection) return univ_faculty_collection
def upload_file(): if not weibo.check_login_status(): if not util.check_config_file(): util.generate_config() config = util.read_config() if not config: util.alert('请先设置你的微博账号') util.open_with_editor() sys.exit(0) username = config['username'] password = config['password'] try: weibo.login_with_username_and_password(username, password) except: util.alert('登录失败,请重试!') sys.exit(0) if weibo.check_login_status(): # util.delete_config() util.alert('登录成功!') else: util.alert('登录失败,请重试!') sys.exit(0) img_file = get_paste_img_file() if img_file: try: url = weibo.request_image_url(img_file.name) return url except: util.delete_cookie() util.alert('Cookie 过期,请重新登录!') sys.exit(0) return None else: util.alert('您的剪切板里没有图片!') sys.exit(0)
def upload_file(): if not weibo.check_login_status(): if not util.check_config_file(): util.generate_config() config = util.read_config() if not config: util.alert('请先设置你的微博账号') util.open_with_editor() sys.exit(0) username = config['username'] password = config['password'] try: weibo.login_with_username_and_password(username, password) except: util.alert('登录失败,请重试!') sys.exit(0) if weibo.check_login_status(): util.delete_config() else: util.alert('登录失败,请重试!') sys.exit(0) img_file = get_paste_img_file() if img_file: try: url = weibo.request_image_url(img_file.name) return url except: util.delete_cookie() util.alert('Cookie 过期,请重新登录!') sys.exit(0) return None else: util.alert('您的剪切板里没有图片!') sys.exit(0)
def create_amis(conn, user_vpc, lab_tag): """Create all images for lab""" running_labs = get_running_labs(conn, user_vpc) cfg = util.read_config(LAB_DIR + lab_tag.rsplit('-', 1)[0] + '/instances.cfg') instances = inst.get_vpc_instances(conn, user_vpc) lab_instances = [] for instance in instances: if 'Lab' in instance.tags: if instance.tags['Lab'] == lab_tag and instance.tags[ 'AMI-Build'] == 'True': lab_instances.append(instance) print "Creating AMI's for lab: {0}".format(lab_tag) for instance in lab_instances: name_tag = TRAINER + '-{0}-'.format(VPC) + \ '{0}-'.format(lab_tag.rsplit('-', 1)[0]) + \ '{0}-'.format(instance.tags['Script']) + \ '{0}'.format(instance.tags['AMI-Key']) ami = conn.create_image( instance.id, name_tag, '{0} lab AMI'.format(lab_tag.rsplit('-', 1)[0])) # tags tags = { 'Name': '{0}'.format(name_tag), 'AMI-Key': '{0}'.format(instance.tags['AMI-Key']), 'Lab': '{0}'.format(lab_tag.rsplit('-', 1)[0]) } conn.create_tags(ami, tags) print "AMI build trigger completed successfully ..." print "Note: It could take 15-20 minutes for the AMI build to complete."
def email_awards(email_address, awards_df, repo_name, srcpath=os.getcwd()): """ Emails award winners to the given email address :param str email_address: the address to email :param pd.DataFrame awards_df: the awards dataframe """ report_filename = os.path.join(srcpath, 'templates', 'email_report.html') with open(report_filename, 'r') as f: content = f.read() month = awards_df.index[0].split()[1] table = awards_df.to_html(col_space=5) content = content.format(repo=repo_name, month=month, table=table) msg = MIMEMultipart('alternative') msg['Subject'] = '{repo} quality stats awards for {month}'.format(repo=repo_name, month=month) from_address = util.read_config('email')['from'] msg['From'] = from_address msg['To'] = email_address msg.attach(MIMEText(content, 'html')) msg.attach(MIMEText(html2text.html2text(content), 'plain')) s = smtplib.SMTP('localhost') for e in email_address.split(','): s.sendmail(from_address, [e], msg.as_string()) s.close()
def main(): cfg = util.read_config('config/digit.yaml') x_train, x_test, y_train, y_test = load_data(cfg['dataset']) x_train, y_train = util.shuffle_data(x_train, y_train) x_test, y_test = util.shuffle_data(x_test, y_test) i=0 y_test3 = [] while i<len(y_test): y_test3.append(y_test[i][0]) i += 1 y_test3 = np.array(y_test3) model_name = cfg['svm']['model_name'] model_dir = cfg['svm']['model_dir'] # svm = train_svm_multi(x_train, y_train, cfg) # svm = train_svm(x_train, y_train, cfg) # y_test2 = test_svm(svm, x_test) # plot_confusion_matrix(y_test3, y_test2) train_test_svm_multi_roc_ver(x_train, y_train, x_test, y_test3, cfg)
def create_amis(conn, user_vpc, lab_tag): """Create all images for lab""" running_labs = get_running_labs(conn, user_vpc) cfg = util.read_config(LAB_DIR + lab_tag.rsplit("-", 1)[0] + "/instances.cfg") instances = inst.get_vpc_instances(conn, user_vpc) lab_instances = [] for instance in instances: if "Lab" in instance.tags: if instance.tags["Lab"] == lab_tag and instance.tags["AMI-Build"] == "True": lab_instances.append(instance) print "Creating AMI's for lab: {0}".format(lab_tag) for instance in lab_instances: name_tag = ( TRAINER + "-{0}-".format(VPC) + "{0}-".format(lab_tag.rsplit("-", 1)[0]) + "{0}-".format(instance.tags["Script"]) + "{0}".format(instance.tags["AMI-Key"]) ) ami = conn.create_image(instance.id, name_tag, "{0} lab AMI".format(lab_tag.rsplit("-", 1)[0])) # tags tags = { "Name": "{0}".format(name_tag), "AMI-Key": "{0}".format(instance.tags["AMI-Key"]), "Lab": "{0}".format(lab_tag.rsplit("-", 1)[0]), } conn.create_tags(ami, tags) print "Completed ..."
def friction(agi, extension, now, context): config = util.read_config(config_filename) if config: config_map = util.relevant_config(config, extension, now, context) if config_map: action(action_map, config_map)(agi)
return log.logger.info('update zops_agent done') def backup_agent(self): app_dir = os.path.join(prog_, '..') src = "%s/agent" % app_dir dst = "%s/agent.%s" % (app_dir, int(time.time())) backup_list = glob.glob('%s/agent.[0-9]*' % app_dir) max_backup = int(self.opts['max_backup']) if len(backup_list) >= max_backup: for bak in sorted(backup_list, key=lambda x: -int(os.path.basename(x).split('.')[1]))[max_backup - 1:]: shutil.rmtree(bak) shutil.copytree(src, dst) if __name__ == '__main__': opts = read_config(config_file='conf/client.conf') wd = WatchDog(opts) #apscheduler.events def err_listener(ev): if ev.exception: aps_log.info('%s error.', str(ev.job)) else: aps_log.info('%s miss', str(ev.job)) from apscheduler.schedulers.blocking import BlockingScheduler from apscheduler.events import EVENT_JOB_ERROR, EVENT_JOB_MISSED from apscheduler.triggers.interval import IntervalTrigger sched = BlockingScheduler() sched.add_listener(err_listener, EVENT_JOB_ERROR | EVENT_JOB_MISSED) sched.add_job(wd.watch_process, IntervalTrigger(seconds=10)) sched.add_job(wd.watch_update, IntervalTrigger(seconds=10)) sched.start()
from os import path from datetime import datetime from RQ3_H1 import format_as_csv from util import read_config from utils import utils project_config = read_config(['project_details.properties']) def compute(): random_number = "{:%Y_%m_%d_%H_%M_%S}".format(datetime.now()) file_path = 'RQ3_H1/results/does_coverage_increase__' + random_number result = for_list_of_projects() formatted_result = format_as_csv.format_contents(result) utils.write_list_as_csv(formatted_result, file_path + '.csv') utils.write_json_file(result, file_path + '.json') def for_list_of_projects(): project_list = project_config.get('projects', 'project_list').split(",") result = [] for project in project_list: result.append({ 'project_name': project, 'result': for_each_project(project) }) return result
def interpretArgs(): settings = { 'delim': ',', 'quote': '"', 'hdr_split': '|', 'row_split': '|', 'database': 'db/members.db', 'header_elig': 'code/headers/elig.hdr', 'header_encs': 'code/headers/encs.hdr', 'header_lab_rsl': 'code/headers/lab_rsl.hdr', 'header_med_clms': 'code/headers/med_clms.hdr', 'header_rx_clms': 'code/headers/rx_clms.hdr', 'join_id': 'MEMBER_ID', 'shelve_id_files': [ 'code/db/set_myeloma.txt', 'code/db/set_diabetes.txt' ], 'anonymize': { 'do': False, 'date_columns': [ 'ELIG_EFFECTIVE_DATE', 'ELIG_TERMINATION_DATE', 'ENCS_SERVICE_DATE', 'ENCS_PAID_DATE', 'ENCS_ADMIT_DATE', 'ENCS_DISCHARGE_DATE', 'LAB_RSL_SERVICE_DATE', 'MED_CLMS_SERVICE_DATE', 'MED_CLMS_PAID_DATE', 'MED_CLMS_ADMIT_DATE', 'MED_CLMS_DISCHARGE_DATE', 'RX_CLMS_SERVICE_DATE', 'RX_CLMS_PAID_DATE', 'RX_CLMS_PRESCRIPTION_DATE' ], 'age_columns': [ 'ELIG_AGE', 'LAB_RSL_AGE', 'RX_CLMS_AGE' ], 'redact_columns': [ 'ELIG_PATIENT_KEY', 'ELIG_OLD_MEMBER_ID', 'ELIG_SUBSCRIBER_ID', 'ELIG_ZIP', 'ELIG_COUNTRY_CODE', 'ELIG_PCP_ID', 'ELIG_GROUP_ID', 'ELIG_SUB_GROUP_ID', 'ELIG_PLAN_ID', 'LAB_RSL_SUBSCRIBER_ID' ] } } info = { 'pid': '', 'output': '-' } args = sys.argv[:] args.pop(0); do_list = False while args: val = args.pop(0) if val == '-h' or val == '--help': usage() if val == '-l' or val == '--list': do_list = True elif val == '-p': if not args: print('-p requires argument', file=sys.stderr) usage() info['pid'] = args.pop(0) elif val == '--all': info['pid'] = '--all' elif val == '-c': if not args: print('-c requires argument', file=sys.stderr) usage() util.read_config(settings, args.pop(0)) elif val == '-o': if not args: print('-o requires argument', file=sys.stderr) usage() info['output'] = args.pop(0) elif arg == '--seed': if not len(args): print('--seed requires integer seed', file=sys.stderr) usage() try: seed = int(args.pop(0)) random.seed(seed) except: print('--seed requires integer seed', file=sys.stderr) usage() else: print('illegal argument '+val, file=sys.stderr) usage() if do_list: printList(settings) sys.exit(0) if info['pid'] == '': print('patient id required', file=sys.stderr) usage() return (settings, info)
schema = '' host = '' port = None user = '' password = '' import util import os _config = os.environ.get('SOCA_CONFIG') or util.find_file('.socarc') if _config: locals().update(util.read_config(_config))
def __init__(self): self.config = read_config(self.source_name) self.log = config_logging(self.config).getLogger(self.source_name) self.db = DB.from_config(self.config)
import time from subprocess import run import util if __name__ == '__main__': config = util.read_config('config.yaml') runners = {} for k, v in config['runners'].items(): runners[k] = v['cmd'] for r in config['runs']: if r in runners: cmd = runners[r] print('\n\n[RUN COMMAND]: {}'.format(cmd)) cmds = cmd.split(' ') run(cmds) time.sleep(0.25) else: err_str = "Invalid runner found in runs section of config.yaml: {}".format( r) raise RuntimeError(err_str)
try: body = self.serial.loads(self.job.recv()) except: log.logger.error(traceback.format_exc()) continue self.job.send(self.serial.dumps({'ret':'ok'})) job_handler = JobHandler(self.opts, body) job_handler.start() def sigterm_clean(signum, frame): clean_proc(job_handler) try: os.kill(os.getpid(),signal.SIGKILL) except OSError: pass signal.signal(signal.SIGTERM, sigterm_clean) def destroy(self): if self.job.closed is False: self.job.setsockopt(zmq.LINGER, 1) self.job.close() if self.context.closed is False: self.context.term() def __del__(self): self.destroy() if __name__ == '__main__': opts = read_config() jobsvr = JobServer(opts) jobsvr.run()
elif arg == '-f': if not args or args[0] == '--': print('-f requires format file', file=sys.stderr) usage() util.read_format(args.pop(0), cms_get_patient.input_format, usage) elif arg == '-o': if not args or args[0] == '--': print('-o requires output file', file=sys.stderr) usage() output = args.pop(0) elif arg == '-c': if not args or args[0] == '--': print('-c requires argument', file=sys.stderr) usage() settingsFile = args.pop(0) util.read_config(settings, settingsFile, build_dictionary.debugOutput) elif arg == '--debug': build_dictionary.debugOutput = True else: print('unrecognized argument: ' + arg, file=sys.stderr) usage() if not len(query): print('query is required', file=sys.stderr) usage() build_dictionary.init(settings, settingsFile) allPaths = [] while args: path = args.pop(0)
usage() if arg == '--path': if not args: print('--path requires path', file=sys.stderr) usage() path = args.pop(0) elif arg == '-f': if not args: print('-f requires format file', file=sys.stderr) usage() util.read_format(args.pop(0), input_format, usage) elif arg == '-c': if not args: print('-c requires argument', file=sys.stderr) usage() util.read_config(settings, args.pop(0)) else: print('unrecognized argument: ' + arg, file=sys.stderr) usage() if path is None: print('need to specify path with --path', file=sys.stderr) usage() if not len(input_format.keys()): print('need to specify non-empty format file with -f', file=sys.stderr) usage() out = { 'delim': settings['delim'], 'quote': settings['quote'] } util.process_burst_directory(path, lambda root, file: processFile(root, path, file, out))
def start_server(max_num, settings_file, format_file, class_file, line_file, cms_path, addr, port, debug): settings = {} util.read_config(settings, settings_file, True) use_cache = settings.get('use_cache', True) all_paths = [] input_format = {} use_db = False omop = None if settings.get('omop_use_db', False): use_db = True omop = OMOP(settings, True) else: util.convert_paths([ cms_path ], all_paths) util.read_format(format_file, input_format, usage) cms_analyze.input_format = input_format cms_get_patient.input_format = input_format build_dictionary.debugOutput = True build_dictionary.init(settings, settings_file) dictionary_file = os.path.join(json_dir, dictionary_bind) patients = set() def save_patients(): if not use_cache: return with open(patients_list, 'w') as pf: pf.write('\n'.join(sorted(list(patients)))) pf.flush() if not os.path.isfile(patients_list) or not use_cache: if use_db: omop.list_patients(patients, prefix=json_dir, limit=max_num, show_old_ids=True) else: tf = StringIO() cms_analyze.compute(all_paths, {}, False, tf, filter_zero=True) tf.flush() tf.seek(0) lines = tf.readlines()[-max_num:] if max_num is not None else tf.readlines() for line in lines: patients.add(json_dir + line.strip() + '.json') save_patients() dict = {} if use_cache: if os.path.isfile(dictionary_file): with open(dictionary_file, 'r') as input: dict = json.loads(input.read()) else: os.makedirs(json_dir) # write the initial empty dictionary # also ensures that the folder is writeable with open(dictionary_file, 'w') as output: output.write("{}") server = create_server((addr, port)) server.bind_path('/', '..') prefix = '/' + os.path.basename(os.path.normpath(server.base_path)) server.add_default_white_list() server.add_file_patterns([ prefix + '/' + json_dir + '*', prefix + '/' + patients_list ], True) server.favicon_fallback = 'favicon.ico' server.report_slow_requests = True if debug: server.suppress_noise = True @server.text_get(prefix + '/' + patients_list, 0) def get_list(req, args): return '\n'.join(sorted(list(patients))) @server.json_get(prefix + '/' + json_dir, 1) def get_patient(req, args): pid = args['paths'][0] if pid.endswith('.json') and use_db: pid = pid[:-len('.json')] pid = omop.get_person_id(pid) cache_file = os.path.join(json_dir, pid) p_name = json_dir + pid.strip() if p_name not in patients: patients.add(p_name) save_patients() if pid.endswith('.json'): pid = pid[:-len('.json')] if not os.path.isfile(cache_file) or not use_cache: if use_db: patient = omop.get_patient(pid, dict, line_file, class_file) else: patient = cms_get_patient.process(all_paths, line_file, class_file, pid) build_dictionary.extractEntries(dict, patient) if use_cache: with open(cache_file, 'w') as pf: pf.write(json_dumps(patient)) pf.flush() if use_cache: with open(dictionary_file, 'w') as output: output.write(json_dumps(dict)) output.flush() return patient with open(cache_file, 'r') as pf: return json.loads(pf.read()) @server.json_get(prefix + '/' + dictionary_file) def get_dictionary(req, args): return dict msg("starting server at {0}:{1}", addr if addr else 'localhost', port) server.serve_forever() msg("shutting down..") server.server_close()
"key_category": category['key_category'] } def run_scrapers(limit_sources=None): """ Collect and run scrapers if limit_names is None, all scrapers will run """ scrapers = set() for root, dirs, files in os.walk(script_dir()): for file_name in glob.fnmatch.filter(files, 'scraper_*.py'): file_path = os.path.join(root, file_name) mod_name = os.path.splitext(file_name)[0] mod = imp.load_source(mod_name, file_path) for (cls_name, cls) in inspect.getmembers(mod, inspect.isclass): if glob.fnmatch.fnmatch(cls_name, 'Scraper*') and\ (not limit_sources or \ (getattr(cls, 'source_name') in limit_sources)): scrapers.add(cls) for scraper in scrapers: scraper().run() if __name__ == '__main__': config = read_config('scraper') limit_sources = None if config.limit_sources: limit_sources = map(string.strip, config.limit_sources.split(',')) run_scrapers(limit_sources)
print "Can't run Candy! Need wxWidgets v2.8+, but got v" + wxVersion sys.exit(0) import wx.stc as stc import wx.lib.pubsub as pubsub import keyboard import util from status_line import StatusLine import data from constants import * project_dir = os.path.join(os.path.dirname(__file__), '..') general_conf_path = os.path.join(project_dir, u'general.conf') general_config = util.read_config(general_conf_path) color_conf = os.path.join(project_dir, u'colorscheme-default.conf') color_scheme = util.read_config(color_conf) class VisualItem(object): """ An item to hold visual representation. E.g. if the external item is a filename, this one will hold things like justified name, startChar in the ViewWindow coords and the like. Number of these objects is the number RawItems that actually fit on screen (at least partially). """ def __init__(self): # Character on the row-representing string, which is the str[0]-th # char of this item's visual representation. Will not be negative, # since objects of this class represent only visible things, even
venues = os.path.join(api_v1, 'venues'), venue = os.path.join(api_v1, 'venues/<id:int>'), venue_nearby = os.path.join(api_v1, 'venues/<id:int>/nearby'), categories = os.path.join(api_v1, 'categories'), category = os.path.join(api_v1, 'categories/<id:int>'), category_venues = os.path.join(api_v1, 'categories/<id:int>/venues'), zips = os.path.join(api_v1, 'zips'), zip = os.path.join(api_v1, 'zips/<zip>'), zip_venues = os.path.join(api_v1, 'zips/<zip>/venues'), ) ### setup config = util.read_config("api") log = util.config_logging(config).getLogger("server") # connection pool pool = psycopg2.pool.ThreadedConnectionPool( minconn=1, maxconn=int(config.max_connections or 10), dsn=config.db, cursor_factory=psycopg2.extras.RealDictCursor ) # install plugin install(bottle_pgpool.PgSQLPoolPlugin(pool)) ### helpers
# -*- coding: utf-8 -*- import os from qiniu import Auth, put_file from util import read_config config = read_config() def upload_qiniu(path, upload_name): ''' upload file to qiniu''' q = Auth(config['ak'], config['sk']) key = upload_name # upload to qiniu's markdown dir token = q.upload_token(config['bucket'], key) ret, info = put_file(token, key, path, check_crc=True) return ret != None and ret['key'] == key
} def run_scrapers(limit_sources=None): """ Collect and run scrapers if limit_names is None, all scrapers will run """ scrapers = set() for root, dirs, files in os.walk(script_dir()): for file_name in glob.fnmatch.filter(files, 'scraper_*.py'): file_path = os.path.join(root, file_name) mod_name = os.path.splitext(file_name)[0] mod = imp.load_source(mod_name, file_path) for (cls_name, cls) in inspect.getmembers(mod, inspect.isclass): if glob.fnmatch.fnmatch(cls_name, 'Scraper*') and\ (not limit_sources or \ (getattr(cls, 'source_name') in limit_sources)): scrapers.add(cls) for scraper in scrapers: scraper().run() if __name__ == '__main__': config = read_config('scraper') limit_sources = None if config.limit_sources: limit_sources = map(string.strip, config.limit_sources.split(',')) run_scrapers(limit_sources)
venues=os.path.join(api_v1, "venues"), venue=os.path.join(api_v1, "venues/<id:int>"), venue_nearby=os.path.join(api_v1, "venues/<id:int>/nearby"), categories=os.path.join(api_v1, "categories"), category=os.path.join(api_v1, "categories/<id:int>"), category_venues=os.path.join(api_v1, "categories/<id:int>/venues"), zips=os.path.join(api_v1, "zips"), zip=os.path.join(api_v1, "zips/<zip>"), zip_venues=os.path.join(api_v1, "zips/<zip>/venues"), ) ### setup config = util.read_config("api") log = util.config_logging(config).getLogger("server") # connection pool pool = psycopg2.pool.ThreadedConnectionPool( minconn=1, maxconn=int(config.max_connections or 10), dsn=config.db, cursor_factory=psycopg2.extras.RealDictCursor ) # install plugin install(bottle_pgpool.PgSQLPoolPlugin(pool)) ### helpers def fq_url(*path, **subst): """ Builds fully qualified url. Uses bottle request object for scheme and loc
import util import models import os from data import * import time import warnings warnings.filterwarnings("ignore") import torch import torch.nn as nn # config config = util.read_config('./configs/predict.yaml') torch.manual_seed(config.seed) # log log_path = './data/results/' # log_path = config.log + '2018-10-20-12:17:22/' if not os.path.exists(log_path): os.mkdir(log_path) logging = util.logging(log_path+'res_log.txt') # 记录本次运行的记录 # checkpoint if config.checkpoint_restore: print('loading checkpoint from {}...'.format(config.checkpoint_restore)) checkpoints = torch.load(config.checkpoint_restore) # cuda use_cuda = torch.cuda.is_available() and len(config.gpus) > 0
def _read_config(self): return util.read_config(os.path.join(self.app_data_dir, self.email, CFG_FILE_NAME))
def parse_config(self): self.cfg_dict_1 = util.read_config(self.config_file_1) self.cfg_dict_2 = util.read_config(self.config_file_2)
#!/usr/bin/env python import gzip from pipeline import * import util config = util.read_config('import') log = util.config_logging(config) table = 'area.area' columns=('area', 'zip', 'po_name', 'geom') filename = 'bayareadata.gz' area = 'sfbayarea' db = util.DB.from_config(config) log.info('importing file %r to table %r' % (filename, table)) # compose import pipeline cat(gzip.open(filename)) | skip(head=2, tail=2) | split(sep='|') |\ transform([lambda r: area, 0, 1, 2]) |\ join('\t') | load_data(db, table, columns=columns, clean=True)