def read_info(self): info_config = configparser.ConfigParser()
def main(): """ Parse arguments """ parser = argparse.ArgumentParser(description='Calculate db overlaps') parser.add_argument('--config', help='Config file') parser.add_argument('--input', help='Input file') parser.add_argument('--output', help='Output file') parser.add_argument('--columns', help='Columns file') parser.add_argument('--verbose', help='increase output verbosity', action='store_true') args = parser.parse_args() """ Read config file """ if args.verbose: print("Reading config file") config = configparser.ConfigParser() config.read(args.config) """ Read input file and define variables """ if args.verbose: print("Reading input file and defining variables") my_dict_data = {} # input_file_path = config.get('DEFAULT', 'input') input_file_path = args.input input_file = open(input_file_path) line = input_file.readline() header = line.split('\t') overlapping = Overlap() overlap_threshold = float(config.get('DEFAULT', 'overlap_threshold')) # 0.7 bnd_window = float(config.get('DEFAULT', 'bnd_window')) # 200 """ Get list of fields to be extracted """ if args.verbose: print("Getting list of fields to be extracted") unique_dbs_str = config.get('DEFAULT', 'unique_dbs') unique_dbs = [x.strip() for x in unique_dbs_str.split(',') ] # ['gnomad', 'ngc38', 'wgs13k', 'ngclov', 'bkl'] dbs_list = [] for e in unique_dbs: e_list = list( filter( lambda x: e in x and not x.startswith('left') and not x. startswith('right'), header)) dbs_list.append(e_list) dbs = list(chain.from_iterable( dbs_list)) # eg: ['gnomad_s', 'gnomad_e', 'gnomad_ov_id'...] fields_list = [['CHROM', 'POS', 'END', 'SVTYPE'], dbs] fields = list(chain.from_iterable(fields_list)) # unlist """ Build list of indices, since this may change. You can stick with the column names from above """ if args.verbose: print("Building list of indices") index = [] for f in fields: index.append(header.index(f)) """ Build dict elements """ if args.verbose: print("Building dictionary elements") dic_element = 0 for line in input_file: information = line.split('\t') my_dict_data[dic_element] = {} my_dict_data[dic_element]['CHROM'] = information[header.index('CHROM')] my_dict_data[dic_element]['POS'] = information[header.index('POS')] my_dict_data[dic_element]['END'] = information[header.index('END')] my_dict_data[dic_element]['SVTYPE'] = information[header.index( 'SVTYPE')] my_dict_data[dic_element]['dbs'] = {} # Let's first initialize all the dictionaries to avoid overwriting them for db in unique_dbs: if information[header.index(db + '_s')] != "": my_dict_data[dic_element]['dbs'][db] = {} my_start_split = information[header.index(db + '_s')].split(',') id_counter = 0 for start in my_start_split: my_dict_data[dic_element]['dbs'][db][id_counter] = {} id_counter = id_counter + 1 for db in dbs: if information[header.index(db)] != "": # split to get db name my_split_db_str = db.split("_") db_name = my_split_db_str[0] my_db_element = my_split_db_str[1] my_split = information[header.index(db)].split(',') id_counter = 0 if my_db_element == 's': for start in my_split: my_dict_data[dic_element]['dbs'][db_name][ id_counter].update({'START': start}) id_counter = id_counter + 1 elif my_db_element == 'e': for end in my_split: my_dict_data[dic_element]['dbs'][db_name][ id_counter].update({'END': end}) id_counter = id_counter + 1 else: # wgs13k has different ID # db_id = '' # if db_name in ['wgs13k', 'bkl']: # if db_name in ['wgs13k']: # db_id = my_split_db_str[1] # else: # db_id = my_split_db_str[2] # my_id_split = information[header.index(db)].split(',') for id in my_split: my_values_split = id.split('|') if db == 'gnomad_ov_id': db_type = my_values_split[1] SN_value = my_values_split[2] else: db_type = my_values_split[0] SN_value = re.sub('SN=', '', my_values_split[1]) my_dict_data[dic_element]['dbs'][db_name][ id_counter].update({'SN': SN_value}) my_dict_data[dic_element]['dbs'][db_name][ id_counter].update({'db_type': db_type}) id_counter = id_counter + 1 dic_element = dic_element + 1 """ Read dataframe in pandas """ if args.verbose: print("Reading data frame in pandas") # df_pd = pd.read_csv(input_file_path, delimiter='\t') tmplist = [] # pool = mp.Pool(4) # TO DO multiprocessing for chunk in pd.read_csv(input_file_path, delimiter='\t', chunksize=2000): # c = pool.apply_async(process_frame, [chunk]) tmplist.append(chunk) df_pd = pd.concat(tmplist, axis=0) del tmplist """ For each db, calculate overlaps and add to pandas dataframe """ if args.verbose: print("Calculating overlaps") for db in unique_dbs: # for each db db_results = [] for dic_element in my_dict_data.keys(): # for each row dic_element_value = [] query_start = my_dict_data[dic_element]['POS'] query_end = my_dict_data[dic_element]['END'] query_type = my_dict_data[dic_element]['SVTYPE'] if query_type == 'TRA' or query_type == 'INS': query_start = int(query_start) - bnd_window query_end = int(query_start) + bnd_window if query_start == query_end: query_end = int(query_end) + 1 # print("Query: " + str(dic_element), query_start, query_end, query_type) if db in my_dict_data[dic_element]['dbs'].keys(): for id_counter in my_dict_data[dic_element]['dbs'][db].keys(): db_start = my_dict_data[dic_element]['dbs'][db][ id_counter]['START'] db_end = my_dict_data[dic_element]['dbs'][db][id_counter][ 'END'] if db == 'bkl': try: ovl = overlapping.overlap( [int(query_start), int(query_end)], [int(db_start), int(db_end)]) db_size = (int(ovl[1]) - int(ovl[0])) + 1 except Exception: db_size = 0 dic_element_value.append(db_size) else: sn = my_dict_data[dic_element]['dbs'][db][id_counter][ 'SN'] db_type = my_dict_data[dic_element]['dbs'][db][ id_counter]['db_type'] if db_type == query_type: MRPO = overlapping.reciprocal_percentage_overlap( [int(query_start), int(query_end)], [int(db_start), int(db_end)]) if query_type == 'TRA' or query_type == 'INS': dic_element_value.append(sn) else: if MRPO >= overlap_threshold: dic_element_value.append(sn) else: dic_element_value = ['0'] # print("Query: " + str(dic_element_value), query_start, query_end, query_type) ##If bkl sum all values, else return comma separated if db == 'bkl': bkl_lst = list(map(int, dic_element_value)) db_results.append(sum(bkl_lst)) else: db_results.append(';'.join(dic_element_value)) db_results_ref = [x if x != '' else '0' for x in db_results] df_pd[db] = db_results_ref """ Define columns to export """ if args.verbose: print("Defining columns to export") columns_path = args.columns with open(columns_path, 'r') as colnames: cols_list = yaml.safe_load(colnames) fixed_cols = cols_list['fixed'] samples_cols = cols_list['samples'] cols = list(chain.from_iterable([fixed_cols, samples_cols])) """ Melt data frame """ if args.verbose: print("Melting data frame") outfile = df_pd[cols] outfile_melt = pd.melt(outfile, id_vars=fixed_cols, value_vars=samples_cols, var_name='SAMPLE', value_name='GT') """ Filtering and reformatting """ if args.verbose: print("Filtering and reformatting") # outfile_alt = outfile_melt.loc[-outfile_melt['GT'].isin(['0/0', './.', './0', '0/.', '0'])] outfile_alt = outfile_melt.loc[ outfile_melt['GT'] != './.:NaN:0:0,0:--:NaN:NaN:NaN:NAN:NAN:NAN'] ##add column for sample ID outfile_alt['ID'] = outfile_alt.GT.str.split(':').str[7] # outfile_alt['ID_sample'] = outfile_alt.GT.str.split(':').str[7] ##keep only GT from GT column outfile_alt['GT'] = outfile_alt.GT.str.split(':').str[0] # outfile_alt.replace(to_replace='GT', value=outfile_alt.GT.str.split(':').str[0]) #not working, to fix ##reorder by input columns file cols_ordered = list( chain.from_iterable([['SAMPLE', 'GT'], list(fixed_cols)])) # cols_ordered = list(chain.from_iterable([['SAMPLE', 'GT', 'ID_SAMPLE'], list(fixed_cols)])) outfile_out = outfile_alt[cols_ordered] """ Save to file """ if args.verbose: print("Saving to file") out_file_path = args.output out_file_melt_path = re.sub('tab', 'melt.tab', out_file_path) outfile.to_csv(out_file_path, sep='\t', index=False) outfile_out.to_csv(out_file_melt_path, sep='\t', index=False)
from scipy.interpolate import InterpolatedUnivariateSpline import numpy as np import time import pandas as pd import plotly.graph_objs as go from collections import deque import os import json import configparser import logging import io import base64 import string config = configparser.ConfigParser() config.read('./config/config.cfg') def create_logger(): logger = logging.getLogger() logger.setLevel(logging.INFO) log_file_handler = logging.FileHandler(config.get('LOG', 'LOG_FILE_PATH')) fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' formatter = logging.Formatter(fmt) log_file_handler.setFormatter(formatter) logger.addHandler(log_file_handler)
def format_file( filename: str, *, min_py3_version: Tuple[int, int], max_py_version: Tuple[int, int], ) -> bool: with open(filename) as f: contents = f.read() cfg = configparser.ConfigParser() cfg.read_string(contents) _clean_sections(cfg) # normalize names to underscores so sdist / wheel have the same prefix cfg['metadata']['name'] = cfg['metadata']['name'].replace('-', '_') # if README.md exists, set `long_description` + content type readme = _first_file(filename, 'readme') if readme is not None: long_description = f'file: {os.path.basename(readme)}' cfg['metadata']['long_description'] = long_description tags = identify.tags_from_filename(readme) if 'markdown' in tags: cfg['metadata']['long_description_content_type'] = 'text/markdown' elif 'rst' in tags: cfg['metadata']['long_description_content_type'] = 'text/x-rst' else: cfg['metadata']['long_description_content_type'] = 'text/plain' # set license fields if a license exists license_filename = _first_file(filename, 'licen[sc]e') if license_filename is not None: cfg['metadata']['license_file'] = os.path.basename(license_filename) license_id = identify.license_id(license_filename) if license_id is not None: cfg['metadata']['license'] = license_id if license_id in LICENSE_TO_CLASSIFIER: cfg['metadata']['classifiers'] = ( cfg['metadata'].get('classifiers', '').rstrip() + f'\n{LICENSE_TO_CLASSIFIER[license_id]}') requires = _python_requires(filename, min_py3_version=min_py3_version) if requires is not None: if not cfg.has_section('options'): cfg.add_section('options') cfg['options']['python_requires'] = requires install_requires = _requires(cfg, 'install_requires') if install_requires: cfg['options']['install_requires'] = '\n'.join(install_requires) setup_requires = _requires(cfg, 'setup_requires') if setup_requires: cfg['options']['setup_requires'] = '\n'.join(setup_requires) if cfg.has_section('options.extras_require'): for key in cfg['options.extras_require']: group_requires = _requires(cfg, key, 'options.extras_require') cfg['options.extras_require'][key] = '\n'.join(group_requires) py_classifiers = _py_classifiers(requires, max_py_version=max_py_version) if py_classifiers: cfg['metadata']['classifiers'] = ( cfg['metadata'].get('classifiers', '').rstrip() + f'\n{py_classifiers}') imp_classifiers = _imp_classifiers(filename) if imp_classifiers: cfg['metadata']['classifiers'] = ( cfg['metadata'].get('classifiers', '').rstrip() + f'\n{imp_classifiers}') # sort the classifiers if present if 'classifiers' in cfg['metadata']: classifiers = sorted(set(cfg['metadata']['classifiers'].split('\n'))) classifiers = _trim_py_classifiers( classifiers, requires, max_py_version=max_py_version, ) cfg['metadata']['classifiers'] = '\n'.join(classifiers) sections: Dict[str, Dict[str, str]] = {} for section, key_order in KEYS_ORDER: if section not in cfg: continue entries = {k.replace('-', '_'): v for k, v in cfg[section].items()} new_section = {k: entries.pop(k) for k in key_order if k in entries} # sort any remaining keys new_section.update(sorted(entries.items())) sections[section] = new_section cfg.pop(section) for section in cfg.sections(): sections[section] = dict(cfg[section]) cfg.pop(section) for k, v in sections.items(): cfg[k] = v sio = io.StringIO() cfg.write(sio) new_contents = sio.getvalue().strip() + '\n' new_contents = new_contents.replace('\t', ' ') new_contents = new_contents.replace(' \n', '\n') if new_contents != contents: with open(filename, 'w') as f: f.write(new_contents) return new_contents != contents
def void_island_grind(window): time.sleep(1) config = configparser.ConfigParser() try: config.read('config.ini', encoding='utf-8') except: config.read('config.ini', encoding='utf-8-sig') restart = int(config['Void_Island']['Restart']) level = int(config['Void_Island']['Level']) melee_kaki_index = config['Void_Island']['Melee_Index'].split(',') range_kaki_index = config['Void_Island']['Range_Index'].split(',') adv_start_diff = [1619 - 245, 887 - 123] pyautogui.click(adv_start_diff[0] + window[0], adv_start_diff[1] + window[1], duration=0.25) # check if inventory full full_inventory_img_diff = [727 - 245, 351 - 123, 1190 - 245, 397 - 123] full_inventory_img = ImageGrab.grab(bbox=(window[0] + full_inventory_img_diff[0], window[1] + full_inventory_img_diff[1], window[0] + full_inventory_img_diff[2], window[1] + full_inventory_img_diff[3])) full_inventory_img.save('inventory_check.jpg', 'JPEG') im_hash = imagehash.average_hash(Image.open('inventory_check.jpg')) im_hash_ref = imagehash.average_hash(Image.open('Ref\\inventory_check_ref.jpg')) if abs(im_hash - im_hash_ref) > 4: void_island_diff = [328 - 245, 560 - 123] pyautogui.click(void_island_diff[0] + window[0], void_island_diff[1] + window[1], duration=0.5) level_inc_diff = [1149 - 245, 716 - 123] time.sleep(0.8) if level > 1: pyautogui.click(level_inc_diff[0] + window[0], level_inc_diff[1] + window[1], clicks=level, interval=0.5) grind_start_diff = [1103 - 245, 831 - 123] pyautogui.click(grind_start_diff[0] + window[0], grind_start_diff[1] + window[1], duration=0.8) team_select_diff = [859 - 245, 748 - 123] pyautogui.click(team_select_diff[0] + window[0], team_select_diff[1] + window[1], duration=0.8) melee_group_diff = [596 - 245, 624 - 123] first_kaki_diff = [365 - 245, 766 - 123] pyautogui.click(melee_group_diff[0] + window[0], melee_group_diff[1] + window[1], duration=0.8) time.sleep(0.2) move_counter = 0 for i in range(len(melee_kaki_index)): # max 13 kaki, in case of new kaki, need to modify the if statement if (int(melee_kaki_index[i]) - 6 * move_counter) > 7: pyautogui.moveTo(first_kaki_diff[0] + 150 * 6 + window[0], first_kaki_diff[1] + window[1]) pyautogui.mouseDown() time.sleep(0.5) pyautogui.dragRel(xOffset=-150 * 6, yOffset=0, duration=3, mouseDownUp=False) time.sleep(0.5) pyautogui.mouseUp() move_counter += 1 pyautogui.click(first_kaki_diff[0] + 150 * (int(melee_kaki_index[i]) - 6 * move_counter - 1) + window[0], first_kaki_diff[1] + window[1], duration=0.8) elif (int(melee_kaki_index[i]) - 6 * move_counter) <= 7: pyautogui.click(first_kaki_diff[0] + 150 * (int(melee_kaki_index[i]) - 6 * move_counter - 1) + window[0], first_kaki_diff[1] + window[1], duration=0.8) range_group_diff = [797 - 245, 622 - 123] pyautogui.click(range_group_diff[0] + window[0], range_group_diff[1] + window[1], duration=0.8) for i in range(len(range_kaki_index)): # max 17 kaki, in case of new kaki, need to modify if statement if (int(range_kaki_index[i]) - 6 * move_counter) > 13: pyautogui.moveTo(first_kaki_diff[0] + 150 * 6 + window[0], first_kaki_diff[1] + window[1]) pyautogui.mouseDown() time.sleep(0.5) pyautogui.dragRel(xOffset=-150 * 6, yOffset=0, duration=3, mouseDownUp=False) time.sleep(0.5) pyautogui.mouseUp() pyautogui.moveTo(first_kaki_diff[0] + 150 * 6 + window[0], first_kaki_diff[1] + window[1]) pyautogui.mouseDown() time.sleep(0.5) pyautogui.dragRel(xOffset=-150 * 6, yOffset=0, duration=3, mouseDownUp=False) time.sleep(0.5) pyautogui.mouseUp() move_counter += 2 pyautogui.click(first_kaki_diff[0] + 150 * (int(range_kaki_index[i]) - 6 * move_counter) + window[0] - 3, first_kaki_diff[1] + window[1], duration=0.8) elif (int(range_kaki_index[i]) - 6 * move_counter) > 7: pyautogui.moveTo(first_kaki_diff[0] + 150 * 6 + window[0], first_kaki_diff[1] + window[1]) pyautogui.mouseDown() time.sleep(0.5) pyautogui.dragRel(xOffset=-150 * 6, yOffset=0, duration=3, mouseDownUp=False) time.sleep(0.5) pyautogui.mouseUp() move_counter += 1 pyautogui.click(first_kaki_diff[0] + 150 * (int(range_kaki_index[i]) - 6 * move_counter - 1) + window[0], first_kaki_diff[1] + window[1], duration=0.8) elif 0 < (int(range_kaki_index[i]) - 6 * move_counter) <= 7: pyautogui.click(first_kaki_diff[0] + 150 * (int(range_kaki_index[i]) - 6 * move_counter - 1) + window[0], first_kaki_diff[1] + window[1], duration=0.8) elif (int(range_kaki_index[i]) - 6 * move_counter) <= 0: pyautogui.moveTo(first_kaki_diff[0] + window[0], first_kaki_diff[1] + window[1]) pyautogui.mouseDown() time.sleep(0.5) pyautogui.dragRel(xOffset=150 * 6, yOffset=0, duration=3, mouseDownUp=False) time.sleep(0.5) pyautogui.mouseUp() move_counter -= 1 pyautogui.click(first_kaki_diff[0] + 150 * (int(range_kaki_index[i]) - 6 * move_counter - 1) + window[0], first_kaki_diff[1] + window[1], duration=0.8) confirm_team_diff = [1493 - 245, 872 - 123] pyautogui.click(window[0] + confirm_team_diff[0], window[1] + confirm_team_diff[1], duration=0.5) go_button_diff = [1460 - 245, 839 - 123] pyautogui.click(window[0] + go_button_diff[0], window[1] + go_button_diff[1], duration=0.5) time.sleep(6) toggle_auto_path_finding(window) time.sleep(6) select_blessing_diff = [1047 - 245, 740 - 123] pyautogui.click(window[0] + select_blessing_diff[0], window[1] + select_blessing_diff[1], duration=0.5) time.sleep(60*restart - 10) success_continue_diff = [957 - 245, 874 - 123] pyautogui.click(window[0] + success_continue_diff[0], window[1] + success_continue_diff[1], duration=0.5) time.sleep(1) # miracle_stone experience completion pyautogui.click(window[0] + success_continue_diff[0], window[1] + success_continue_diff[1], duration=0.5) time.sleep(10) # confirm pyautogui.click(window[0] + success_continue_diff[0], window[1] + success_continue_diff[1], duration=0.5) time.sleep(10) else: send_email('Inventory Full!') sys.exit() pass
def main(): # Parse arguments and pass through unrecognised args parser = argparse.ArgumentParser( add_help=False, usage='%(prog)s [test_runner.py options] [script options] [scripts]', description=__doc__, epilog=''' Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( '--combinedlogslen', '-c', type=int, default=0, help= 'print a combined log (of length n lines) from all test nodes and test framework to the console on failure.' ) parser.add_argument( '--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') parser.add_argument( '--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument( '--extended', action='store_true', help='run the extended test suite in addition to the basic tests') parser.add_argument( '--force', '-f', action='store_true', help= 'run tests even on platforms where they are disabled by default (e.g. windows).' ) parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument( '--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') parser.add_argument( '--keepcache', '-k', action='store_true', help= 'the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.' ) parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") parser.add_argument('--failfast', action='store_true', help='stop execution after the first test failure') args, unknown_args = parser.parse_known_args() # args to be passed on always start with two dashes; tests are the remaining unknown args tests = [arg for arg in unknown_args if arg[:2] != "--"] passon_args = [arg for arg in unknown_args if arg[:2] == "--"] # Read config generated by configure. config = configparser.ConfigParser() configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" config.read_file(open(configfile, encoding="utf8")) passon_args.append("--configfile=%s" % configfile) # Set up logging logging_level = logging.INFO if args.quiet else logging.DEBUG logging.basicConfig(format='%(message)s', level=logging_level) # Create base test directory tmpdir = "%s/test_runner_Ł_🏃_%s" % ( args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) os.makedirs(tmpdir) logging.debug("Temporary test directory at %s" % tmpdir) enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") if config["environment"]["EXEEXT"] == ".exe" and not args.force: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 print( "Tests currently disabled on Windows by default. Use --force option to enable" ) sys.exit(0) if not enable_bitcoind: print("No functional tests to run.") print("Rerun ./configure with --with-daemon and then make") sys.exit(0) # Build list of tests test_list = [] if tests: # Individual tests have been specified. Run specified tests that exist # in the ALL_SCRIPTS list. Accept the name with or without .py extension. tests = [re.sub("\.py$", "", test) + ".py" for test in tests] for test in tests: if test in ALL_SCRIPTS: test_list.append(test) else: print("{}WARNING!{} Test '{}' not found in full test list.". format(BOLD[1], BOLD[0], test)) elif args.extended: # Include extended tests test_list += ALL_SCRIPTS else: # Run base tests only test_list += BASE_SCRIPTS # Remove the test cases that the user has explicitly asked to exclude. if args.exclude: exclude_tests = [ re.sub("\.py$", "", test) + ".py" for test in args.exclude.split(',') ] for exclude_test in exclude_tests: if exclude_test in test_list: test_list.remove(exclude_test) else: print("{}WARNING!{} Test '{}' not found in current test list.". format(BOLD[1], BOLD[0], exclude_test)) if not test_list: print( "No valid test scripts specified. Check that your test is in one " "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests" ) sys.exit(0) if args.help: # Print help for test_runner.py, then print help of the first script (with args removed) and exit. parser.print_help() subprocess.check_call([ sys.executable, os.path.join(config["environment"]["SRCDIR"], 'test', 'functional', test_list[0].split()[0]), '-h' ]) sys.exit(0) check_script_list(config["environment"]["SRCDIR"]) check_script_prefixes() if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) run_tests( test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], tmpdir, jobs=args.jobs, enable_coverage=args.coverage, args=passon_args, combined_logs_len=args.combinedlogslen, failfast=args.failfast, )
def token(): config = configparser.ConfigParser() config.read("config.ini") return config.get("bot", "token")
def __init__(self): self._conf = configparser.ConfigParser() if os.path.exists(cloudstack_file): self._conf.read(cloudstack_file)
def saveDefaults(self, Dic=DefaultDic): config = configparser.ConfigParser() config.read_dict(Dic) with open('./configuration.ini', 'w') as configfile: config.write(configfile)
import logging import sys import matplotlib.pyplot as plt import pandas as pd import psycopg2.errors import sqlalchemy as sa from sqlalchemy.dialects import postgresql from sqlalchemy.exc import IntegrityError from wotstats.api import TIME_FIELDS, Realm, account_info from wotstats.plotting import create_plot from wotstats.sql import statistics from wotstats.utils import flatten, timestamps_to_datetime config = configparser.ConfigParser(allow_no_value=True) config.read_dict( { "db": {"url": "postgresql://wotstats@localhost/wotstats"}, "api": {"realm": "EU", "application-id": ""}, "accounts": {}, "plots": {}, "logging": {"log-level": "INFO"}, } ) parser = argparse.ArgumentParser() parser.add_argument("--config", default="/etc/wotstats/wotstats.ini") def main(args=None):
def process_file(path, **kwargs): if not osp.isfile(path): import errno raise IOError(errno.ENOENT, "No such file", path) verboseprint("Processing file %s...." % path) if kwargs.get('backup', True): import shutil shutil.copyfile(path, path + '.bak') cpt = configparser.ConfigParser() # gem5 is case sensitive with paramaters cpt.optionxform = str # Read the current data cpt_file = open(path, 'r') cpt.read_file(cpt_file) cpt_file.close() change = False # Make sure we know what we're starting from if cpt.has_option('root','cpt_ver'): cpt_ver = cpt.getint('root','cpt_ver') # Legacy linear checkpoint version # convert to list of tags before proceeding tags = set([]) for i in range(2, cpt_ver+1): tags.add(Upgrader.legacy[i].tag) verboseprint("performed legacy version -> tags conversion") change = True cpt.remove_option('root', 'cpt_ver') # @todo The 'Globals' option is deprecated, and should be removed in the # future elif cpt.has_option('Globals','version_tags'): tags = set((''.join(cpt.get('Globals','version_tags'))).split()) elif cpt.has_option('root.globals','version_tags'): tags = set((''.join(cpt.get('root.globals','version_tags'))).split()) else: print("fatal: no version information in checkpoint") exit(1) verboseprint("has tags", ' '.join(tags)) # If the current checkpoint has a tag we don't know about, we have # a divergence that (in general) must be addressed by (e.g.) merging # simulator support for its changes. unknown_tags = tags - (Upgrader.tag_set | Upgrader.untag_set) if unknown_tags: print("warning: upgrade script does not recognize the following " "tags in this checkpoint:", ' '.join(unknown_tags)) # Apply migrations for tags not in checkpoint and tags present for which # downgraders are present, respecting dependences to_apply = (Upgrader.tag_set - tags) | (Upgrader.untag_set & tags) while to_apply: ready = set([ t for t in to_apply if Upgrader.get(t).ready(tags) ]) if not ready: print("could not apply these upgrades:", ' '.join(to_apply)) print("update dependences impossible to resolve; aborting") exit(1) for tag in ready: Upgrader.get(tag).update(cpt, tags) change = True to_apply -= ready if not change: verboseprint("...nothing to do") return cpt.set('root.globals', 'version_tags', ' '.join(tags)) # Write the old data back verboseprint("...completed") cpt.write(open(path, 'w'))
from threading import local from .cursors import * from .format import FormatterMixin from .query import QueryMixin from .schema import SchemaMixin from .sync import SyncMixin logger = logging.getLogger(__name__) _ROOT_ID_SQL = 'SELECT id FROM nodes WHERE name IS NULL AND type == "folder" ORDER BY created' _SETTINGS_FILENAME = 'cache.ini' _def_conf = configparser.ConfigParser() _def_conf['sqlite'] = dict(filename='nodes.db', busy_timeout=30000, journal_mode='wal') _def_conf['blacklist'] = dict(folders= []) def _get_conf(path='') -> configparser.ConfigParser: conf = configparser.ConfigParser() conf.read_dict(_def_conf) conffn = os.path.join(path, _SETTINGS_FILENAME) try: with open(conffn) as cf: conf.read_file(cf) except OSError: pass
def getConfig(self): self.cf = configparser.ConfigParser() self.cf.read(self.configLocation) return self.cf
def _main(args): config_path = os.path.expanduser(args.config_path) weights_path = os.path.expanduser(args.weights_path) assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format( config_path) assert weights_path.endswith( '.weights'), '{} is not a .weights file'.format(weights_path) output_path = os.path.expanduser(args.output_path) assert output_path.endswith( '.h5'), 'output path {} is not a .h5 file'.format(output_path) output_root = os.path.splitext(output_path)[0] # Load weights and config. print('Loading weights.') weights_file = open(weights_path, 'rb') weights_header = np.ndarray( shape=(5, ), dtype='int32', buffer=weights_file.read(20)) print('Weights Header: ', weights_header) # TODO: Check transpose flag when implementing fully connected layers. # transpose = (weight_header[0] > 1000) or (weight_header[1] > 1000) print('Parsing Darknet config.') unique_config_file = unique_config_sections(config_path) cfg_parser = configparser.ConfigParser() cfg_parser.read_file(unique_config_file) print('Creating Keras model.') if args.fully_convolutional: image_height, image_width = None, None else: image_height = int(cfg_parser['net_0']['height']) image_width = int(cfg_parser['net_0']['width']) prev_layer = Input(shape=(image_height, image_width, 3)) all_layers = [prev_layer] outputs = [] weight_decay = float(cfg_parser['net_0']['decay'] ) if 'net_0' in cfg_parser.sections() else 5e-4 count = 0 for section in cfg_parser.sections(): print('Parsing section {}'.format(section)) if section.startswith('convolutional'): filters = int(cfg_parser[section]['filters']) size = int(cfg_parser[section]['size']) stride = int(cfg_parser[section]['stride']) pad = int(cfg_parser[section]['pad']) activation = cfg_parser[section]['activation'] batch_normalize = 'batch_normalize' in cfg_parser[section] # Setting weights. # Darknet serializes convolutional weights as: # [bias/beta, [gamma, mean, variance], conv_weights] prev_layer_shape = K.int_shape(prev_layer) # TODO: This assumes channel last dim_ordering. weights_shape = (size, size, prev_layer_shape[-1], filters) darknet_w_shape = (filters, weights_shape[2], size, size) weights_size = np.product(weights_shape) print('conv2d', 'bn' if batch_normalize else ' ', activation, weights_shape) conv_bias = np.ndarray( shape=(filters, ), dtype='float32', buffer=weights_file.read(filters * 4)) count += filters if batch_normalize: bn_weights = np.ndarray( shape=(3, filters), dtype='float32', buffer=weights_file.read(filters * 12)) count += 3 * filters # TODO: Keras BatchNormalization mistakenly refers to var # as std. bn_weight_list = [ bn_weights[0], # scale gamma conv_bias, # shift beta bn_weights[1], # running mean bn_weights[2] # running var ] conv_weights = np.ndarray( shape=darknet_w_shape, dtype='float32', buffer=weights_file.read(weights_size * 4)) count += weights_size # DarkNet conv_weights are serialized Caffe-style: # (out_dim, in_dim, height, width) # We would like to set these to Tensorflow order: # (height, width, in_dim, out_dim) # TODO: Add check for Theano dim ordering. conv_weights = np.transpose(conv_weights, [2, 3, 1, 0]) conv_weights = [conv_weights] if batch_normalize else [ conv_weights, conv_bias ] # Handle activation. act_fn = None if activation == 'leaky': pass # Add advanced activation later. elif activation != 'linear': raise ValueError( 'Unknown activation function `{}` in section {}'.format( activation, section)) padding = 'same' if pad == 1 and stride == 1 else 'valid' # Adjust padding model for darknet. if stride == 2: prev_layer = ZeroPadding2D(((1, 0), (1, 0)))(prev_layer) # Create Conv2D layer conv_layer = (Conv2D( filters, (size, size), strides=(stride, stride), kernel_regularizer=l2(weight_decay), use_bias=not batch_normalize, weights=conv_weights, activation=act_fn, padding=padding))(prev_layer) if batch_normalize: conv_layer = (BatchNormalization( weights=bn_weight_list))(conv_layer) prev_layer = conv_layer if activation == 'linear': all_layers.append(prev_layer) elif activation == 'leaky': act_layer = LeakyReLU(alpha=0.1)(prev_layer) prev_layer = act_layer all_layers.append(prev_layer) elif section.startswith('maxpool'): size = int(cfg_parser[section]['size']) stride = int(cfg_parser[section]['stride']) all_layers.append( MaxPooling2D( padding='same', pool_size=(size, size), strides=(stride, stride))(prev_layer)) prev_layer = all_layers[-1] elif section.startswith('avgpool'): if cfg_parser.items(section) != []: raise ValueError('{} with params unsupported.'.format(section)) all_layers.append(GlobalAveragePooling2D()(prev_layer)) prev_layer = all_layers[-1] elif section.startswith('route'): ids = [int(i) for i in cfg_parser[section]['layers'].split(',')] if len(ids) == 2: for i, item in enumerate(ids): if item != -1: ids[i] = item + 1 layers = [all_layers[i] for i in ids] if len(layers) > 1: print('Concatenating route layers:', layers) concatenate_layer = concatenate(layers) all_layers.append(concatenate_layer) prev_layer = concatenate_layer else: skip_layer = layers[0] # only one layer to route all_layers.append(skip_layer) prev_layer = skip_layer elif section.startswith('shortcut'): ids = [int(i) for i in cfg_parser[section]['from'].split(',')][0] activation = cfg_parser[section]['activation'] shortcut = add([all_layers[ids], prev_layer]) if activation == 'linear': shortcut = Activation('linear')(shortcut) all_layers.append(shortcut) prev_layer = all_layers[-1] elif section.startswith('upsample'): stride = int(cfg_parser[section]['stride']) all_layers.append( UpSampling2D( size=(stride, stride))(prev_layer)) prev_layer = all_layers[-1] elif section.startswith('yolo'): classes = int(cfg_parser[section]['classes']) # num = int(cfg_parser[section]['num']) # mask = int(cfg_parser[section]['mask']) n1, n2 = int(prev_layer.shape[1]), int(prev_layer.shape[2]) n3 = 3 n4 = (4 + 1 + classes) yolo = Reshape((n1, n2, n3, n4))(prev_layer) all_layers.append(yolo) prev_layer = all_layers[-1] outputs.append(len(all_layers) - 1) elif (section.startswith('net')): pass # Configs not currently handled during model definition. else: raise ValueError( 'Unsupported section header type: {}'.format(section)) # Create and save model. model = Model(inputs=all_layers[0], outputs=[all_layers[i] for i in outputs]) print(model.summary()) model.save('{}'.format(output_path)) print('Saved Keras model to {}'.format(output_path)) # Check to see if all weights have been read. remaining_weights = len(weights_file.read()) / 4 weights_file.close() print('Read {} of {} from Darknet weights.'.format(count, count + remaining_weights)) if remaining_weights > 0: print('Warning: {} unused weights'.format(remaining_weights)) plot(model, to_file='{}.png'.format(output_root), show_shapes=True) print('Saved model plot to {}.png'.format(output_root))
def get_config(path: str) -> configparser.ConfigParser: config_obj = configparser.ConfigParser( interpolation=configparser.ExtendedInterpolation()) with open(path) as f: config_obj.read_file(f) return config_obj
def create_config(self, rl_time_steps=3000000, garden_time_steps=40, garden_x=10, garden_y=10, sector_width=2, sector_height=2, num_plant_types=2, num_plants_per_type=1, step=1, action_low=0.0, action_high=sim_globals.MAX_WATER_LEVEL, obs_low=0, obs_high=1000, ent_coef=0.01, n_steps=40000, nminibatches=4, noptepochs=4, learning_rate=1e-8, cc_coef=0, water_coef=0, cnn_args=None, dir_path=""): config = configparser.ConfigParser() config.add_section('rl') config['rl']['time_steps'] = str(rl_time_steps) config['rl']['ent_coef'] = str(ent_coef) config['rl']['n_steps'] = str(n_steps) config['rl']['nminibatches'] = str(nminibatches) config['rl']['noptepochs'] = str(noptepochs) config['rl']['learning_rate'] = str(learning_rate) if cnn_args: config.add_section('cnn') config['cnn']['output_x'] = str(cnn_args["OUTPUT_X"]) config['cnn']['output_y'] = str(cnn_args["OUTPUT_Y"]) config['cnn']['num_hidden_layers'] = str( cnn_args["NUM_HIDDEN_LAYERS"]) config['cnn']['num_filters'] = str(cnn_args["NUM_FILTERS"]) config['cnn']['num_convs'] = str(cnn_args["NUM_CONVS"]) config['cnn']['filter_size'] = str(cnn_args["FILTER_SIZE"]) config['cnn']['stride'] = str(cnn_args["STRIDE"]) config.add_section('garden') config['garden']['time_steps'] = str(garden_time_steps) config['garden']['X'] = str(garden_x) config['garden']['Y'] = str(garden_y) config['garden']['sector_width'] = str(sector_width) config['garden']['sector_height'] = str(sector_height) config['garden']['num_plant_types'] = str(num_plant_types) config['garden']['num_plants_per_type'] = str(num_plants_per_type) config['garden']['step'] = str(step) config.add_section('action') config['action']['low'] = str(action_low) config['action']['high'] = str(action_high) config.add_section('obs') config['obs']['low'] = str(obs_low) config['obs']['high'] = str(obs_high) config.add_section('reward') config['reward']['cc_coef'] = str(cc_coef) config['reward']['water_coef'] = str(water_coef) pathlib.Path('gym_config').mkdir(parents=True, exist_ok=True) with open('gym_config/config.ini', 'w') as configfile: config.write(configfile)
def open_config_file(configfile_path): parser = configparser.ConfigParser() parser.optionxform = str parser.read([configfile_path]) return parser
def main(self): """Main function. This should not be overridden by the subclass test scripts.""" parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave worldcoinds and test.* datadir on exit or error") parser.add_argument("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop worldcoinds after the test execution") parser.add_argument("--cachedir", dest="cachedir", default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), help="Directory for caching pregenerated datadirs (default: %(default)s)") parser.add_argument("--tmpdir", dest="tmpdir", help="Root directory for datadirs") parser.add_argument("-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") parser.add_argument("--tracerpc", dest="trace_rpc", default=False, action="store_true", help="Print out all RPC calls as they are made") parser.add_argument("--portseed", dest="port_seed", default=os.getpid(), type=int, help="The seed to use for assigning port numbers (default: current process id)") parser.add_argument("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") parser.add_argument("--configfile", dest="configfile", default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../config.ini"), help="Location of the test framework config file (default: %(default)s)") parser.add_argument("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true", help="Attach a python debugger if test fails") parser.add_argument("--usecli", dest="usecli", default=False, action="store_true", help="use worldcoin-cli instead of RPC for all commands") parser.add_argument("--perf", dest="perf", default=False, action="store_true", help="profile running nodes with perf for the duration of the test") self.add_options(parser) self.options = parser.parse_args() PortSeed.n = self.options.port_seed check_json_precision() self.options.cachedir = os.path.abspath(self.options.cachedir) config = configparser.ConfigParser() config.read_file(open(self.options.configfile)) self.config = config self.options.bitcoind = os.getenv("LITECOIND", default=config["environment"]["BUILDDIR"] + '/src/worldcoind' + config["environment"]["EXEEXT"]) self.options.bitcoincli = os.getenv("LITECOINCLI", default=config["environment"]["BUILDDIR"] + '/src/worldcoin-cli' + config["environment"]["EXEEXT"]) os.environ['PATH'] = os.pathsep.join([ os.path.join(config['environment']['BUILDDIR'], 'src'), os.path.join(config['environment']['BUILDDIR'], 'src', 'qt'), os.environ['PATH'] ]) # Set up temp directory and start logging if self.options.tmpdir: self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: self.options.tmpdir = tempfile.mkdtemp(prefix=TMPDIR_PREFIX) self._start_logging() self.log.debug('Setting up network thread') self.network_thread = NetworkThread() self.network_thread.start() success = TestStatus.FAILED try: if self.options.usecli: if not self.supports_cli: raise SkipTest("--usecli specified but test does not support using CLI") self.skip_if_no_cli() self.skip_test_if_missing_module() self.setup_chain() self.setup_network() self.run_test() success = TestStatus.PASSED except JSONRPCException as e: self.log.exception("JSONRPC error") except SkipTest as e: self.log.warning("Test Skipped: %s" % e.message) success = TestStatus.SKIPPED except AssertionError as e: self.log.exception("Assertion failed") except KeyError as e: self.log.exception("Key error") except Exception as e: self.log.exception("Unexpected exception caught during testing") except KeyboardInterrupt as e: self.log.warning("Exiting after keyboard interrupt") if success == TestStatus.FAILED and self.options.pdbonfailure: print("Testcase failed. Attaching python debugger. Enter ? for help") pdb.set_trace() self.log.debug('Closing down network thread') self.network_thread.close() if not self.options.noshutdown: self.log.info("Stopping nodes") if self.nodes: self.stop_nodes() else: for node in self.nodes: node.cleanup_on_exit = False self.log.info("Note: worldcoinds were not stopped and may still be running") should_clean_up = ( not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED and not self.options.perf ) if should_clean_up: self.log.info("Cleaning up {} on exit".format(self.options.tmpdir)) cleanup_tree_on_exit = True elif self.options.perf: self.log.warning("Not cleaning up dir {} due to perf data".format(self.options.tmpdir)) cleanup_tree_on_exit = False else: self.log.warning("Not cleaning up dir {}".format(self.options.tmpdir)) cleanup_tree_on_exit = False if success == TestStatus.PASSED: self.log.info("Tests successful") exit_code = TEST_EXIT_PASSED elif success == TestStatus.SKIPPED: self.log.info("Test skipped") exit_code = TEST_EXIT_SKIPPED else: self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) exit_code = TEST_EXIT_FAILED logging.shutdown() if cleanup_tree_on_exit: shutil.rmtree(self.options.tmpdir) sys.exit(exit_code)
_config_file_name = 'kiskadee.conf' _sys_config_file = os.path.join('/etc', _config_file_name) _dev_config_file = os.path.join(os.path.dirname(_my_path), # go up a dir 'util', _config_file_name) _doc_config_file = os.path.join(os.path.dirname(_my_path), 'util', _config_file_name) _defaults = {} if not os.path.exists(_sys_config_file): # log _sys_config_file not found # raise ValueError("No such file or directory: %s" % _sys_config_file) pass if not os.path.exists(_dev_config_file): # log _dev_config_file not found pass config = configparser.ConfigParser(defaults=_defaults) _read = config.read([_dev_config_file, _sys_config_file, _doc_config_file]) if len(_read) < 1: raise ValueError("Invalid config files. Should be either %s or %s or %s" % (_sys_config_file, _dev_config_file, _doc_config_file)) # log no config files were loaded pass elif len(_read) == 2 or _read[0] == _sys_config_file: # log _sys_config_file found loaded pass else: # log _read[0] loaded pass log_file = config['DEFAULT']['log_file']
def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" config = configparser.ConfigParser() config.read_file(open(self.options.configfile)) return config["components"].getboolean("ENABLE_CLI")
def __init__(self, app, **kwargs): ttk.Frame.__init__(self, app, **kwargs) # ----------Globals Variables---------- # self.map_lst = {} self.addon_lst = {} self.maps_namelst = [] self.addon_namelst = [] self.c_map = [] self.map_selected = [] self.filever = "0.0.6" # actual version of EpicWorkshop self.ew_cfg = configparser.ConfigParser() self.updaterVal = True # ------------------------------------ # app.title("Epic Workshop " + self.filever) app.geometry("475x420") app.minsize(475, 400) #-----------------------------------------------------------------------------------------------------------# if getattr(sys, 'frozen', False): # if the app is compiled app.iconbitmap(resource_path("epicworkshop.ico")) elif __file__: # if the app is executed with python app.iconbitmap(sys.path[0] + "/img/epicworkshop.ico") #-----------------------------------------------------------------------------------------------------------# app.config(bg="#272727") app.protocol("WM_DELETE_WINDOW", self.on_closing) self.frame = Frame(app, bg="#272727", highlightthickness=0) self.frame.pack(fill=BOTH, anchor=CENTER, expand=YES) self.tmp_can = Canvas(self.frame, bg="#272727", height=40, width=200, highlightthickness=0) self.tmp_can.place(anchor=CENTER, relx=0.5, rely=0.6) self.cfg_label = Label(self.frame, text="Hi !", bg="#272727", font=("Consolas", 12), fg="white", highlightthickness=0) self.cfg_label.place(anchor=CENTER, relx=0.5, rely=0.5) # check if the version of the "version" file is the same as the app if os.path.exists(appdata("version")): with open(appdata("version"), "r") as data: ver = data.read() if ver != self.filever: patch = requests.get("https://api.github.com/repos/Naaikho/epicworkshop-compiled/releases/latest") patch = patch.json()["body"] showinfo("Patch v" + self.filever, patch) with open(appdata("version"), "w") as data: data.write(self.filever) else: # or just create the "version" file with open(appdata("version"), "w") as data: data.write(self.filever) # start the init thread for show the live infos self.initthread = threading.Thread(target=self.init) self.initthread.daemon = True self.initthread.start()
def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" config = configparser.ConfigParser() config.read_file(open(self.options.configfile)) return config["components"].getboolean("ENABLE_ZMQ")
def load_configfile(filename): config_file_parser = configparser.ConfigParser(converters={'list': lambda x: [i.strip() for i in x.split(',')]}) config_file_parser.defaults() config_file_parser.read(filename) return config_file_parser
def read_token_from_config_file(self, settings): parser = cfg.ConfigParser() parser.read(settings) return parser.get('creds', 'token')
import os, configparser from PyQt5.QtWidgets import (QFileDialog, QLabel, QLineEdit, QSpinBox, QDoubleSpinBox, QCheckBox, QGroupBox, QComboBox, QPushButton) from libpport import utilities config = configparser.ConfigParser(strict=False) config.optionxform = str def openini(parent, fileName=''): parent.tabWidget.setCurrentIndex(0) parent.machinePTE.clear() if not fileName: if os.path.isdir(os.path.expanduser('~/linuxcnc/configs')): configsDir = os.path.expanduser('~/linuxcnc/configs') else: configsDir = os.path.expanduser('~/') fileName = QFileDialog.getOpenFileName( parent, caption="Select Configuration INI File", directory=configsDir, filter='*.ini', options=QFileDialog.DontUseNativeDialog, ) if fileName: parent.machinePTE.appendPlainText(f'Loading {fileName[0]}') iniFile = (fileName[0]) else:
def popen_webhost(*, stdout, stderr, script_root=FUNCS_PATH, port=None): testconfig = None if WORKER_CONFIG.exists(): testconfig = configparser.ConfigParser() testconfig.read(WORKER_CONFIG) dll = os.environ.get('PYAZURE_WEBHOST_DLL') if not dll and testconfig and testconfig.has_section('webhost'): dll = testconfig['webhost'].get('dll') if dll: # Paths from environment might contain trailing or leading whitespace. dll = dll.strip() if not dll: dll = DEFAULT_WEBHOST_DLL_PATH if not dll or not pathlib.Path(dll).exists(): raise RuntimeError('\n'.join([ f'Unable to locate Azure Functions Host binary.', f'Please do one of the following:', f' * run the following command from the root folder of the', f' project:', f'', f' $ {sys.executable} setup.py webhost', f'', f' * or download or build the Azure Functions Host and then write', f' the full path to WebHost.dll into the `PYAZURE_WEBHOST_DLL`', f' environment variable. Alternatively, you can create the', f' {WORKER_CONFIG.name} file in the root folder of the project', f' with the following structure:', f'', f' [webhost]', f' dll = /path/Microsoft.Azure.WebJobs.Script.WebHost.dll', ])) worker_path = os.environ.get('PYAZURE_WORKER_DIR') if not worker_path: worker_path = WORKER_PATH else: worker_path = pathlib.Path(worker_path) if not worker_path.exists(): raise RuntimeError(f'Worker path {worker_path} does not exist') # Casting to strings is necessary because Popen doesn't like # path objects there on Windows. extra_env = { 'AzureWebJobsScriptRoot': str(script_root), 'workers:config:path': str(worker_path), 'workers:python:path': str(worker_path / 'python' / 'worker.py'), 'host:logger:consoleLoggingMode': 'always', } if testconfig and 'azure' in testconfig: st = testconfig['azure'].get('storage_key') if st: extra_env['AzureWebJobsStorage'] = st cosmos = testconfig['azure'].get('cosmosdb_key') if cosmos: extra_env['AzureWebJobsCosmosDBConnectionString'] = cosmos if port is not None: extra_env['ASPNETCORE_URLS'] = f'http://*:{port}' return subprocess.Popen(['dotnet', str(dll)], cwd=script_root, env={ **os.environ, **extra_env, }, stdout=stdout, stderr=stderr)
def __init__(self): self.config = configparser.ConfigParser() self.config.read(os.path.abspath('../example.ini'))
def getDefaults(): # Hardcoded defaults dflt = {'system': {'user': '******', 'group': 'sks', 'compress': 'xz', 'svcs': ['sks-db', 'sks-recon'], 'logfile': '/var/log/sksdump.log', 'days': 1, 'dumpkeys': 15000}, 'sync': {'throttle': 0}, 'paths': {'basedir': '/var/lib/sks', 'destdir': '/srv/http/sks/dumps', 'rsync': ('[email protected]:' + '/srv/http/sks/dumps'), 'sksbin': '/usr/bin/sks'}, 'runtime': {'nodump': None, 'nocompress': None, 'nosync': None}} ## Build out the default .ini. dflt_b64 = ("""IyBJTVBPUlRBTlQ6IFRoaXMgc2NyaXB0IHVzZXMgY2VydGFpbiBwZXJtaXNz aW9ucyBmdW5jdGlvbnMgdGhhdCByZXF1aXJlIHNvbWUKIyBmb3JldGhvdWdo dC4KIyBZb3UgY2FuIGVpdGhlciBydW4gYXMgcm9vdCwgd2hpY2ggaXMgdGhl ICJlYXN5IiB3YXksIE9SIHlvdSBjYW4gcnVuIGFzIHRoZQojIHNrcyB1c2Vy IChvci4uLiB3aGF0ZXZlciB1c2VyIHlvdXIgU0tTIGluc3RhbmNlIHJ1bnMg YXMpLgojIEl0IGhhcyB0byBiZSBvbmUgb3IgdGhlIG90aGVyOyB5b3UnbGwg U0VSSU9VU0xZIG1lc3MgdGhpbmdzIHVwIG90aGVyd2lzZS4KIyBJZiB5b3Ug cnVuIGFzIHRoZSBza3MgdXNlciwgTUFLRSBTVVJFIHRoZSBmb2xsb3dpbmcg aXMgc2V0IGluIHlvdXIgc3Vkb2VycwojICh3aGVyZSBTS1NVU0VSIGlzIHRo ZSB1c2VybmFtZSBza3MgcnVucyBhcyk6CiMJQ21uZF9BbGlhcyBTS1NDTURT ID0gL3Vzci9iaW4vc3lzdGVtY3RsIHN0YXJ0IHNrcy1kYixcCiMJICAgICAg ICAgICAgICAgICAgICAgL3Vzci9iaW4vc3lzdGVtY3RsIHN0b3Agc2tzLWRi LFwKIyAgICAgICAgICAgICAgICAgICAgICAgIC91c3IvYmluL3N5c3RlbWN0 bCBzdGFydCBza3MtcmVjb24sXAojCQkgICAgICAgICAgICAgICAgIC91c3Iv YmluL3N5c3RlbWN0bCBzdG9wIHNrcy1yZWNvbgojCVNLU1VTRVIgQUxMID0g Tk9QQVNTV0Q6IFNLU0NNRFMKCiMgVGhpcyB3YXMgd3JpdHRlbiBmb3Igc3lz dGVtZCBzeXN0ZW1zIG9ubHkuIFR3ZWFraW5nIHdvdWxkIGJlIG5lZWRlZCBm b3IKIyBub24tc3lzdGVtZCBzeXN0ZW1zIChzaW5jZSBldmVyeSBub24tc3lz dGVtZCB1c2VzIHRoZWlyIG93biBpbml0IHN5c3RlbQojIGNhbGxhYmxlcy4u LikKCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMj IyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMKCiMgVGhp cyBzZWN0aW9uIGNvbnRyb2xzIHZhcmlvdXMgc3lzdGVtIGNvbmZpZ3VyYXRp b24uCltzeXN0ZW1dCgojIFRoaXMgc2hvdWxkIGJlIHRoZSB1c2VyIFNLUyBy dW5zIGFzLgp1c2VyID0gc2tzCgojIFRoaXMgaXMgdGhlIGdyb3VwIHRoYXQg U0tTIHJ1bnMgYXMuCmdyb3VwID0gc2tzCgojIElmIGVtcHR5LCBkb24ndCBj b21wcmVzcyBkdW1wcy4KIyBJZiBvbmUgb2Y6IHh6LCBneiwgYnoyLCBvciBs cnogKGZvciBscnppcCkgdGhlbiB1c2UgdGhhdCBjb21wcmVzc2lvbiBhbGdv LgojIE5vdGUgdGhhdCBscnppcCByZXF1aXJlcyBleHRyYSBpbnN0YWxsYXRp b24uCmNvbXByZXNzID0geHoKCiMgVGhlc2Ugc2VydmljZXMgd2lsbCBiZSBz dG9wcGVkL3N0YXJ0ZWQsIGluIG9yZGVyLCBiZWZvcmUvYWZ0ZXIgZHVtcHMu IElmIG1vcmUKIyB0aGFuIG9uZSwgc2VwZXJhdGUgYnkgY29tbWFzLgpzdmNz ID0gc2tzLWRiLHNrcy1yZWNvbgoKIyBUaGUgcGF0aCB0byB0aGUgbG9nZmls ZS4KbG9nZmlsZSA9IC92YXIvbG9nL3Nrc2R1bXAubG9nCgojIFRoZSBudW1i ZXIgb2YgZGF5cyBvZiByb3RhdGVkIGtleSBkdW1wcy4gSWYgZW1wdHksIGRv bid0IHJvdGF0ZS4KZGF5cyA9IDEKCiMgSG93IG1hbnkga2V5cyB0byBpbmNs dWRlIGluIGVhY2ggZHVtcCBmaWxlLgpkdW1wa2V5cyA9IDE1MDAwCgoKIyBU aGlzIHNlY3Rpb24gY29udHJvbHMgc3luYyBzZXR0aW5ncy4KW3N5bmNdCgoj IFRoaXMgc2V0dGluZyBpcyB3aGF0IHRoZSBzcGVlZCBzaG91bGQgYmUgdGhy b3R0bGVkIHRvLCBpbiBLaUIvcy4gSWYgZW1wdHkgb3IKIyAwLCBwZXJmb3Jt IG5vIHRocm90dGxpbmcuCnRocm90dGxlID0gMAoKCiMgVGhpcyBzZWN0aW9u IGNvbnRyb2xzIHdoZXJlIHN0dWZmIGdvZXMgYW5kIHdoZXJlIHdlIHNob3Vs ZCBmaW5kIGl0LgpbcGF0aHNdCgojIFdoZXJlIHlvdXIgU0tTIERCIGlzLgpi YXNlZGlyID0gL3Zhci9saWIvc2tzCgojIFRoaXMgaXMgdGhlIGJhc2UgZGly ZWN0b3J5IHdoZXJlIHRoZSBkdW1wcyBzaG91bGQgZ28uCiMgVGhlcmUgd2ls bCBiZSBhIHN1Yi1kaXJlY3RvcnkgY3JlYXRlZCBmb3IgZWFjaCBkYXRlLgpk ZXN0ZGlyID0gL3Nydi9odHRwL3Nrcy9kdW1wcwoKIyBUaGUgcGF0aCBmb3Ig cnN5bmNpbmcgdGhlIGR1bXBzLiBJZiBlbXB0eSwgZG9uJ3QgcnN5bmMuCnJz eW5jID0gcm9vdEBtaXJyb3Iuc3F1YXJlLXIwMHQubmV0Oi9zcnYvaHR0cC9z a3MvZHVtcHMKCiMgVGhlIHBhdGggdG8gdGhlIHNrcyBiaW5hcnkgdG8gdXNl Lgpza3NiaW4gPSAvdXNyL2Jpbi9za3MKCgojIFRoaXMgc2VjdGlvbiBjb250 cm9scyBydW50aW1lIG9wdGlvbnMuIFRoZXNlIGNhbiBiZSBvdmVycmlkZGVu IGF0IHRoZQojIGNvbW1hbmRsaW5lLiBUaGV5IHRha2Ugbm8gdmFsdWVzOyB0 aGV5J3JlIG1lcmVseSBvcHRpb25zLgpbcnVudGltZV0KCiMgRG9uJ3QgZHVt cCBhbnkga2V5cy4KIyBVc2VmdWwgZm9yIGRlZGljYXRlZCBpbi10cmFuc2l0 L3ByZXAgYm94ZXMuCjtub2R1bXAKCiMgRG9uJ3QgY29tcHJlc3MgdGhlIGR1 bXBzLCBldmVuIGlmIHdlIGhhdmUgYSBjb21wcmVzc2lvbiBzY2hlbWUgc3Bl Y2lmaWVkIGluCiMgdGhlIFtzeXN0ZW06Y29tcHJlc3NdIHNlY3Rpb246ZGly ZWN0aXZlLgo7bm9jb21wcmVzcwoKIyBEb24ndCBzeW5jIHRvIGFub3RoZXIg c2VydmVyL3BhdGgsIGV2ZW4gaWYgb25lIGlzIHNwZWNpZmllZCBpbiBbcGF0 aHM6cnN5bmNdLgo7bm9zeW5j""") realcfg = configparser.ConfigParser(defaults = dflt, allow_no_value = True) if not os.path.isfile(cfgfile): with open(cfgfile, 'w') as f: f.write(base64.b64decode(dflt_b64).decode('utf-8')) realcfg.read(cfgfile) return(realcfg)
from ml_seq2seq import seq2seq_train from tokenizer import Tokenizer from data_utils import get_training_data if __name__ == "__main__": """ Main driver program to launch the parallel pipeline for processing github commits. """ parser = argparse.ArgumentParser(description='cmod') parser.add_argument('-c', '--config', help='Config file path', required=True) parser.add_argument('-e', '--erase', help='Clean previous run data', action='store_true') cfg_parser = configparser.ConfigParser() args = parser.parse_args() cfg_parser.read(args.config) cfg = config_ml.ConfigML(cfg_parser) for section_name in cfg_parser: print('Section:', section_name) section = cfg_parser[section_name] for name in section: print(' {} = {}'.format(name, section[name])) print()
def __parse_settings(self, string): config = configparser.ConfigParser() config.read_string("[DEFAULT]\n" + string) return config.defaults()