def main(argv): parser = argparse.ArgumentParser( description='Visual multilingual information denoising bottleneck') parser.add_argument('CONFIG', type=str) args = parser.parse_args(argv) torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = True config = ConfigFactory.parse_file(args.CONFIG) if not config.dset_dir: config.dset_dir = "/ws/ifp-53_2/hasegawa/lwang114/data/zerospeech2021-dataset/phonetic" seed = config.seed torch.manual_seed(seed) torch.cuda.manual_seed(seed) np.random.seed(seed) np.set_printoptions(precision=4) torch.set_printoptions(precision=4) print() print('[CONFIGS]') print(config) print() net = Solver(config) save_embedding = config.get('save_embedding', False) if config.mode == 'train': net.train(save_embedding=save_embedding) elif config.mode == 'test': net.load_checkpoint() net.test(save_embedding=save_embedding) else: return 0
def load_config(file_dir, config_file='config.conf'): config = ConfigFactory.parse_file(os.path.join(file_dir, f'{config_file}')) if 'airflow' in config: if 'start_date' in config['airflow']: config['airflow']['start_date'] = pd.to_datetime( config['airflow']['start_date']).to_pydatetime() return config
def main(): config_path = os.path.join('web', 'rhasspy.conf') logging.info('Loading configuration from %s' % config_path) config = ConfigFactory.parse_file(config_path) train_speech_recognizer(config) train_intent_recognizer(config)
def get_config(self): try: conf = ConfigFactory.parse_file('./conf/config.conf') except Exception as err: raise err("Configuration Failed !") else: return conf
def convert(input_file=None, output_file=None, format='json'): """Convert to json, properties or yaml :param format: json, properties or yaml :type format: basestring :return: json, properties or yaml string representation """ if input_file is None: content = sys.stdin.read() config = ConfigFactory.parse_string(content) else: config = ConfigFactory.parse_file(input_file) if format.lower() == 'json': res = HOCONConverter.to_json(config) elif format.lower() == 'properties': res = HOCONConverter.to_properties(config) elif format.lower() == 'yaml': res = HOCONConverter.to_yaml(config) else: raise Exception("Format must be 'json', 'properties' or 'yaml'") if output_file is None: print(res) else: with open(output_file, "w") as fd: fd.write(res)
def main(): # parse arguments. get config file path. parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', help='config file', required=True) parser.add_argument('-o', '--overrides', type=json.loads, help='a JSON string with configuration overrides', required=False, default="{}" ) args = parser.parse_args() # the configuration file can be provided in json as much as in hocon # ConfigFactory will pick up the format from the file extension config = ConfigFactory.parse_file(args.config) # we like to keep the config as a dict from here on config = config.as_plain_ordered_dict() # now we override file config with command line provided config config.update(args.overrides) # go on processing sync(config)
def single_run(policy, trace, size=4, changes={}, name=None, save=True, reuse=False, verbose=False, readonly=False): name = name if name else policy policy = Policy[policy] trace = Trace[trace] if 0 < size < 9: size = trace.typical_caches()[size - 1] conf_path = caffeine_root + 'simulator{0}src{0}main{0}resources{0}'.format( os.sep) conf_file = conf_path + 'application.conf' if not os.path.exists(output_csvs_path): os.makedirs(output_csvs_path) run_simulator = './gradlew simulator:run -x caffeine:compileJava -x caffeine:compileCodeGenJava' # run_simulator = './gradlew simulator:run' if os.path.exists(conf_file): conf = ConfigFactory.parse_file(conf_file) else: conf = ConfigFactory.parse_string(""" caffeine { simulator { } } """) simulator = conf['caffeine']['simulator'] simulator.put('files.paths', [resources_path + trace.format() + os.sep + trace.file()]) simulator.put('files.format', trace.value['format']) simulator.put('maximum-size', size) simulator.put('policies', [policy.value]) simulator.put('admission', [Admission.ALWAYS.value]) simulator.put('report.format', 'csv') simulator.put( 'report.output', output_csvs_path + '{}-{}-{}.csv'.format(trace.name, size, name)) for k, v in changes.items(): simulator.put(k, v) with open(conf_file, 'w') as f: f.write(HOCONConverter.to_hocon(conf)) if (not reuse or not os.path.isfile( simulator['report']['output'])) and not readonly: call(run_simulator, shell=True, cwd=caffeine_root, stdout=subprocess.DEVNULL if not verbose else None) with open(simulator['report']['output'], 'r') as csvfile: reader = csv.DictReader(csvfile) results = {line['Policy']: float(line['Hit rate']) for line in reader} if not save: os.remove(simulator['report']['output']) return results if len(results) != 1 else results[0]
def from_file(cls, path, soup=None, savae_config=(10, 0.0002, 0.3)): config_tree = ConfigFactory.parse_file(path) if soup: config_tree['seed'] = soup config_tree['num_iter'], config_tree['eps'], config_tree[ 'momentum'] = savae_config return cls(config_tree)
def from_conf(cls, conf_path): conf = ConfigFactory.parse_file(str(conf_path)) data_path = Path( os.path.abspath(get_conf(conf, group=cls.GROUP, key="data_path"))) if get_conf(conf, group=cls.GROUP, key="output_dir") is None: output_dir = data_path.parent.joinpath("output") else: output_dir = Path(get_conf(conf, group=cls.GROUP, key="output_dir")) return cls(data_path=data_path, output_dir=output_dir, n_evals=get_conf(conf, group=cls.GROUP, key="n_evals", default=50), n_bootstraps=get_conf(conf, group=cls.GROUP, key="n_bootstraps", default=100), n_folds=get_conf(conf, group=cls.GROUP, key="n_folds", default=6), search_method=get_conf(conf, group=cls.GROUP, key="search_method", default="particle swarm"))
def get_custom_settings(args): custom_settings_file = vars(args).get('custom_settings_file') if custom_settings_file and os.path.exists(custom_settings_file): print('Loading custom settings {}'.format(custom_settings_file)) return ConfigFactory.parse_file(custom_settings_file) else: return None
def __init__(self, conf_path): super(Option, self).__init__() self.conf = ConfigFactory.parse_file(conf_path) # ------------ General options ---------------------------------------- self.save_path = self.conf['save_path'] self.dataPath = self.conf['dataPath'] # path for loading data set self.dataset = self.conf[ 'dataset'] # options: imagenet | cifar10 | cifar100 | imagenet100 | mnist self.nGPU = self.conf['nGPU'] # number of GPUs to use by default self.GPU = self.conf['GPU'] # default gpu to use, options: range(nGPU) self.visible_devices = self.conf['visible_devices'] # ------------- Data options ------------------------------------------- self.nThreads = self.conf['nThreads'] # number of data loader threads # ---------- Optimization options -------------------------------------- self.nEpochs = self.conf[ 'nEpochs'] # number of total epochs to train 400 self.batchSize = self.conf['batchSize'] # mini-batch size 128 self.momentum = self.conf['momentum'] # momentum 0.9 self.weightDecay = float(self.conf['weightDecay']) # weight decay 1e-4 self.ori_opt_type = self.conf['ori_opt_type'] self.quan_opt_type = self.conf['quan_opt_type'] # lr master for optimizer 1 (mask vector d) self.ori_lr = self.conf['ori_lr'] # initial learning rate self.quan_lr = self.conf['quan_lr'] self.lrPolicy = self.conf[ 'lrPolicy'] # options: multi_step | linear | exp | const | step self.power = self.conf['power'] # power for inv policy (lr_policy) self.step = self.conf[ 'step'] # step for linear or exp learning rate policy self.decayRate = self.conf['decayRate'] # lr decay rate self.endlr = self.conf['endlr'] # ---------- Model options --------------------------------------------- self.netType = self.conf[ 'netType'] # options: ResNet | PreResNet | GreedyNet | NIN | LeNet5 | LeNet500300 | DenseNet_Cifar | AlexNet self.experimentID = self.conf['experimentID'] self.depth = self.conf['depth'] # resnet depth: (n-2)%6==0 self.nClasses = self.conf[ 'nClasses'] # number of classes in the dataset self.wideFactor = self.conf[ 'wideFactor'] # wide factor for wide-resnet self.drawNetwork = self.conf['drawNetwork'] # ---------- Quantization options --------------------------------------------- self.quantization_k = self.conf['quantization_k'] self.alpha = float(self.conf['alpha']) self.beta = float(self.conf['beta']) self.gamma = float(self.conf['gamma']) # ---------- Resume or Retrain options --------------------------------------------- self.resume = None if len(self.conf['resume']) == 0 else self.conf[ 'resume'] # "./checkpoint_064.pth" self.ori_retrain = None if len( self.conf['ori_retrain']) == 0 else self.conf['ori_retrain'] self.quan_retrain = None if len( self.conf['quan_retrain']) == 0 else self.conf['quan_retrain']
def parse_m5o_file(self, file_path): try: parsed = ConfigFactory.parse_file(file_path) parsed["_file_path"] = str(file_path) return parsed except Exception as e: raise MeltanoAnalysisFileParserError(str(e), str(file_path.parts[-1]))
def load(file: str, clazz): try: conf = ConfigFactory.parse_file(file) return __parse(conf, clazz, "") except pyparsing.ParseSyntaxException as e: raise MalformedConfigException( f'parsing failure line {e.lineno} character {e.col}, got "{e.line}"' )
def agentset(): agentset_conf = ConfigFactory.parse_file('conf/agentset.conf') agentset_prim = agentset_conf.get_list('primitives') html = load_html(agentset_prim) return render_template('agentset.html', task=Markup(html))
def __init__(self, conf_path="configs/eval.hocon"): self.eval_conf = ConfigFactory.parse_file(conf_path) self.plot = self.eval_conf['plot'] self.num_epochs = self.eval_conf['num_epochs'] self.lr = self.eval_conf['lr'] self.weight_decay = self.eval_conf['weight_decay'] self.batch_size = self.eval_conf['batch_size'] self.show_plots = self.eval_conf['show_plots']
def gen_conf(scale, app_name, query): conf = ConfigFactory.parse_file('../conf/application.conf') conf.put("all.query-num", query) conf.put("all.data-scale", scale) conf.put("all.app-suffix", app_name) with open('../conf/application-run.conf', 'w') as f: f.write(HOCONConverter.convert(conf, 'hocon'))
def test_get_config_item(): config = ConfigFactory.parse_file( 'default_configuration\\application.conf') actual = apic.get_config_item(config, 'non_existing_item') assert actual is None actual = apic.get_config_item(config, 'sso_service_base_url') assert actual == 'https://sso.moodysanalytics.com'
def add_dataset(self, dataset_path, name): data_conf_obj = dict() data_conf = ConfigFactory.parse_file(dataset_path) data_conf_obj['data_path'] = os.path.expanduser(data_conf['data_path']) data_conf_obj['label1'] = os.path.expanduser(data_conf['label1']) data_conf_obj['label2'] = os.path.expanduser(data_conf['label2']) data_conf_obj['meta_path'] = os.path.expanduser(data_conf['meta_path']) self.datasets[name] = data_conf_obj
def load(cls, config_name="reference.conf"): frame = inspect.stack()[1] module = inspect.getmodule(frame[0]) loader = pkgutil.get_loader(module.__name__) base = Path(loader.path).parent config = ConfigFactory.parse_file(str(base / config_name)) return cls(config)
def __init__(self, conf_path): super(Option, self).__init__() self.conf = ConfigFactory.parse_file(conf_path) # ------------- general options ---------------------------------------- self.save_path = self.conf['save_path'] # log path self.data_path = self.conf['data_path'] # path for loading data set self.dataset = self.conf['dataset'] # options: imagenet | cifar10 self.seed = self.conf['seed'] # manually set RNG seed self.gpu = self.conf['gpu'] # GPU id to use, e.g. "0,1,2,3" self.n_gpus = len( self.gpu.split(',')) # number of GPUs to use by default # ------------- data options ------------------------------------------- self.n_threads = self.conf[ 'n_threads'] # number of threads used for data loading self.n_classes = self.conf[ 'n_classes'] # number of classes in the dataset self.multi_frame = self.conf['multi_frame'] # ------------- discrimination-aware options --------------------------- self.n_losses = self.conf['n_losses'] # number of additional losses # self.pruning_rate = self.conf['pruning_rate'] # pruning rate # change by liuxu rates = self.conf['pruning_rate'].split(',') self.pruning_rate = [1 - float(i) for i in rates] self.softmax_weight = self.conf[ 'softmax_weight'] # weight of the softmax loss self.mse_weight = self.conf[ 'mse_weight'] # weight of the mean square loss self.max_samples = self.conf[ 'max_samples'] # maximum sample size used for channel selection, -1 means using whole data set self.warmstart = self.conf[ 'warmstart'] # whether to use warmstart in subproblem # ------------- common optimization options ---------------------------- self.batch_size = self.conf['batch_size'] # mini-batch size self.momentum = self.conf['momentum'] # momentum self.weight_decay = self.conf['weight_decay'] # weight decay self.lr = 0.1 self.layer_wise_lr = self.conf[ 'layer_wise_lr'] # initial learning rate # ------------- model options ------------------------------------------ self.net_type = self.conf[ 'net_type'] # options: resnet | preresnet | vgg self.experiment_id = self.conf[ 'experiment_id'] # identifier for experiment self.depth = self.conf['depth'] # resnet depth: (n-2)%6==0 # ---------- resume or pretrained options --------------------------------- # path to model to pretrained with, load model state_dict only self.pretrained = None if len( self.conf['pretrained']) == 0 else self.conf['pretrained'] # path to directory containing checkpoint, load state_dicts of model and optimizer, as well as training epoch self.resume = None if len( self.conf['resume']) == 0 else self.conf['resume']
def _load_custom_config(run_config): """Load custom configuration input HOCON file for cromwell. """ from pyhocon import ConfigFactory, HOCONConverter, ConfigTree conf = ConfigFactory.parse_file(run_config) out = {} if "database" in conf: out["database"] = HOCONConverter.to_hocon(ConfigTree({"database": conf.get_config("database")})) return out
def __init__(self, config='./config'): reader = ConfigFactory.parse_file(config).get('reader') self.input_path = '/dev/input/event{}'.format(reader) self.device = evdev.InputDevice(self.input_path) self.device.grab() self.prefix = "KEY_" self.enter = "ENTER"
def test_factory_persist_local(): source_path = f"{BASE_PATH}/Brodmann/BIrodmann/test/unit_test_files/factory_persist_local/source/groupby_limit.csv" conf_file = f"{BASE_PATH}/Brodmann/BIrodmann/test/unit_test_files/factory_persist_local/conf/conf.conf" conf = ConfigFactory.parse_file(conf_file) source_df = pd.read_csv(source_path) FactoryPersistDF(source_df, "local", conf, logging.getLogger(__name__)).build().persist() target_df = pd.read_csv(conf.get_string("App.Reports.persist.path")) assert_frame_equal(target_df, source_df)
def load_default_config(path: Union[Path, str] = None): """Load custom configuration from specified file. Modifies global constants!""" # pylint: disable=global-statement global DEFAULT_CONFIG if path: DEFAULT_CONFIG = ConfigTree.merge_configs( DEFAULT_CONFIG, ConfigFactory.parse_file(str(path))) else: DEFAULT_CONFIG = default_config(RESOURCES_PATH, "graph.conf")
def open(self): raw_input = ConfigFactory.parse_file( os.path.abspath('../config/formed_signal_input.conf')) with ProcessPoolExecutor( max_workers=ProjParaShop.PARALLEL_SIZE) as executor: print( list( executor.map(SignalFeature4Fog().process, raw_input["FORMED_SIGNAL_MSG"])))
def read_config(path): from pyhocon import ConfigFactory as HoconConfigFactory config = HoconConfigFactory.parse_file(path.absolute()) global canvas_width, canvas_height, hex_palette, default_color_idx canvas_width = config.get('board.width') canvas_height = config.get('board.height') hex_palette = config.get('board.palette') default_color_idx = config.get('board.defaultColor')
def __init__(self): conf = ConfigFactory.parse_file("../config/ml.conf") self.mode = conf["NN.mode"] self.input_file = conf["NN.input_file"] self.model = conf["NN.model"] self.output = conf["NN.output"] self.batch_size = conf["NN.batch_size"] self.epochs = conf["NN.epochs"] self.cluster_size = conf["NN.cluster_size"] self.steps = conf["NN.steps"]
def provide(config_file_path=None) -> "Config": if config_file_path is None: if "NOX__CONFIG_FILE_PATH" in os.environ: config_file_path = os.environ["NOX__CONFIG_FILE_PATH"] else: config_file_path = "nox.conf" config = ConfigFactory.parse_file(config_file_path) return Config(config)
def read_conf(filename): conf = ConfigFactory.parse_file(filename) config_list = conf.get_config("config_list") for exchange in config_list: config = {} exchange_conf = config_list.get_config(exchange) config["api_key"] = exchange_conf["api_key"] config["api_secret"] = exchange_conf["api_secret"] config_list[exchange] = config return config_list
def gen_conf(t1, t2, scale, app_name, query): conf = ConfigFactory.parse_file('../conf/application.conf') conf.put("Q23.table-list", [t1, t2]) conf.put("all.data-scale", scale) conf.put("all.hdfs", 'hdfs://%s:8020/'%HDFS) conf.put("all.app-suffix", app_name) conf.put("Q23.query", QUERYS[query]) with open('../conf/application-run.conf', 'w') as f: f.write(HOCONConverter.convert(conf, 'hocon'))
def load_config(file_path): """ Wrapper over pyhocon that returns a python dict rather than pyhocon's structure, and also corrects None values :param file_path: path of the config file to parse :return: config dict """ conf = ConfigFactory.parse_file(file_path) _load_config(conf) return dict(conf)
def parse(hocon_file): """ This function takes a hocon file and returns a dictionary. :param hocon_file: Path to hocon file. :returns conf: Dictionary version of passed hocon file """ conf = ConfigFactory.parse_file(hocon_file) conf = json.loads(HOCONConverter.convert(conf, 'json')) return conf
def main(): parent_parser = argparse.ArgumentParser(add_help=False) parent_parser.add_argument('-d', '--database', help='database', required=True, action='store') parent_parser.add_argument('-s', '--schema', help='schema', required=False, action='store', default=None) parent_parser.add_argument('-t', '--table', help='table filter (using % as a wildcard)', required=False, action='store') parent_parser.add_argument('-dr', '--dry-run', dest='dry_run', help='dry run', required=False, action='store_true') argparser = argparse.ArgumentParser(description='export') subparsers = argparser.add_subparsers(help='sub-command help', dest='subparser_name') ddl_parser = subparsers.add_parser('ddl', help='ddl', parents=[parent_parser]) ddl_parser.add_argument('-e', '--export', dest='export_file', help='export', required=False, action='store') ddl_parser.add_argument('-i', '--import', dest='import_file', help='import', required=False) data_parser = subparsers.add_parser('data', help='data', parents=[parent_parser]) data_parser.add_argument('-e', '--export', dest='export_file', help='export', required=False, action='store') data_parser.add_argument('-i', '--import', dest='import_file', help='import', required=False) subparsers.add_parser('list', help='list', parents=[parent_parser]) args = argparser.parse_args() home_dir = os.environ['HOME'] config_path = os.path.join(home_dir, '.catdb') if not os.path.exists(config_path): sys.stderr.write( 'File {config_path} not found. Go to https://github.com/chimpler/catdb for more details\n'.format( config_path=config_path)) sys.exit(1) config = ConfigFactory.parse_file(config_path) db_config = config['databases.' + args.database] db = DbManager.get_db(db_config['type'], db_config) if args.subparser_name == 'list': print '\n'.join(db.list_tables(args.table, args.schema)) elif args.subparser_name == 'ddl': if args.export_file: ddl_str = json.dumps(db.get_ddl(args.table, args.schema), sort_keys=True, indent=config['ddl-format.indent'], separators=(',', ': ')) with open_output_file(args.export_file) as fd: fd.write(ddl_str) elif args.import_file: with open_input_file(args.import_file) as fd: ddl = json.loads(fd.read()) table_statement = db.create_database_statement(ddl, args.database, args.schema) if args.dry_run: print table_statement else: db.execute(table_statement) elif args.subparser_name == 'data': if args.export_file: db.export_to_file(args.export_file, args.table, args.schema, config['data-format.delimiter'], config['data-format.null']) elif args.import_file: db.import_from_file(args.import_fileport_file, args.table, args.schema, config['data-format.delimiter'], config['data-format.null'])
def from_file(cls, path, fmt='hocon'): if fmt == 'hocon': config_tree = ConfigFactory.parse_file(path) elif fmt == 'json': with open(path, 'r') as f: d = json.load(f) config_tree = ConfigFactory.from_dict(d) else: raise ValueError('Invalid format: {}'.format(fmt)) return cls(config_tree)
def __init__(self, routers=None, **kwargs): self.extensions = {} self.kwargs = kwargs default_config_file = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), 'application.conf') config_file = self.kwargs.pop('config', default_config_file) if os.path.exists(config_file): self.config = ConfigFactory.parse_file(config_file) else: self.config = ConfigTree() if routers is None: routers = [] self.routers = routers self.filters = []
def __init__(self, args): name_conf_file = "./config/%s.conf" % args[1] conf = ConfigFactory.parse_file(name_conf_file) kafka_servers = conf.get('scraper.kafka.servers') self.kafka = KafkaListing(kafka_servers) self.producer = self.kafka.producer() redis_conf = conf.get('scraper.redis') self.visited_pages = redis.StrictRedis(host=redis_conf['host'], port=redis_conf['port'], db=redis_conf['db']) job_conf = conf.get('scraper.job') self.timeout = int(job_conf['timeout']) self.job_name = job_conf['name'] self.mode = job_conf['mode'] print('starting job %s' % job_conf)
def load_config(config_file, fallback_config_files): """ Load configuration from a HOCON configuration file, with an optional fallback chain @param config_file: the primary configuration file @param fallback_config_files: an optional list of fallback configuration files @rtype: ConfigTree @return: configuration """ config = ConfigFactory.parse_file(config_file) if fallback_config_files: for fallback_config_file in fallback_config_files: if isfile(fallback_config_file): config = config.with_fallback(fallback_config_file) else: print 'Warn: "%s" not found or not a file' % fallback_config_file return config
def convert_from_file(cls, input_file=None, output_file=None, output_format='json', indent=2, compact=False): """Convert to json, properties or yaml :param input_file: input file, if not specified stdin :param output_file: output file, if not specified stdout :param output_format: json, properties or yaml :return: json, properties or yaml string representation """ if input_file is None: content = sys.stdin.read() config = ConfigFactory.parse_string(content) else: config = ConfigFactory.parse_file(input_file) res = cls.convert(config, output_format, indent, compact) if output_file is None: print(res) else: with open(output_file, "w") as fd: fd.write(res)
print(directory_name + " was not a directory. Skipping.") continue directory_path = os.path.join(opts.template_directory, directory_name) template_path = os.path.join(directory_path, "template.json") config_path = os.path.join(directory_path, "template.conf") if not os.path.isfile(template_path): print("%s is not a template directory (template file is missing)" % directory_path) continue if not os.path.isfile(config_path): print("%s is not a template directory (config file is missing)" % directory_path) continue template = open(template_path).read() config = ConfigFactory.parse_file(config_path) if "parameters" not in config: config["parameters"] = {} template_variables = get_template_variables(template) complete_parameters = update_parameters(config["parameters"], template_variables, template_path, opts.parameter_type) parameters_changed = False if config["parameters"] != complete_parameters: parameters_changed = True config["parameters"] = complete_parameters template_without_dashes, parameters_without_dashes = convert_dashes_to_underscores(template, config["parameters"], template_path)
def test_include_dict_from_samples(self): config = ConfigFactory.parse_file("samples/animals.conf") assert config.get('cat.garfield.say') == 'meow' assert config.get('dog.mutt.hates.garfield.say') == 'meow'
def __init__(self, config = "development.conf"): self.conf = ConfigFactory.parse_file(config)
def get_test_config(): return ConfigFactory.parse_file(pkg_resources.resource_filename('tests', 'test_data/test.conf'))
def read_config(cfgpath): return ConfigFactory.parse_file(cfgpath)
def from_file(cls, path): config_tree = ConfigFactory.parse_file(path) return cls(config_tree)
def __init__(self, defaults, config_path): file_config = ConfigFactory.parse_file(config_path) default_config = ConfigFactory.parse_string(defaults) self._config = file_config.with_fallback(default_config)
f.write(" nGramGranularity = 5,\n") f.write(" addNGramFeatures = true,\n") f.write(" inputFile = \"file:///scratch/network/alexeys/bills/lexs/bills_combined_3_COILNJ_new.json\",\n") if cat != "": f.write(" outputFile = \"/user/alexeys/valid_pairs_\""+cat+",\n") f.write(" outputParquetFile = \"/user/alexeys/bills_combined_\""+cat+",\n") else: f.write(" outputFile = \"/user/alexeys/valid_pairs\",\n") f.write(" outputParquetFile = \"/user/alexeys/bills_combined\",\n") f.write(" nPartitions = 40,\n") f.write(" numTextFeatures = 1048576,\n") f.write(" kval = 40,\n") f.write(" numConcepts = 1500\n") f.write("}\n") for cat in strictCats: createOneConfig(conf,cat) if __name__=='__main__': base_config_path = 'src/main/resources/workflow1_makeCartesian.conf' conf = ConfigFactory.parse_file(base_config_path) nFolders = 14 nPartitions = 120 nCPartitions = 3000 measureName = "maxasymjaccard" inputParquetFile = sys.argv[1] #/user/alexeys/bills_combined inputPairsFile = sys.argv[2] #/user/alexeys/valid_pairs outputMainFile = sys.argv[3] #/user/alexeys/output_sample createConfigsA(conf, nPartitions, nCPartitions, measureName, inputParquetFile, inputPairsFile, outputMainFile, nFolders)
def __init__(self, default_path, config_path): default_config = ConfigFactory.parse_file(default_path) file_config = ConfigFactory.parse_file(config_path) self._config = file_config.with_fallback(default_config)
def prepareAndImportConf(options): logging.info('Parsing base config ...') conf = ConfigFactory.parse_file(DEFAULT_BASE_CONF_NAME) logging.info('Parsing base config ... Successful') logging.info('Assigning parameters ...') name = options.env region = options.region subscriptionId = options.subId tenantId = options.tenantId clientId = options.clientId clientSecret = options.clientSecret username = options.username keyFileName = DEFAULT_BASE_DIR + "/" + username + "/" + options.keyFileName generateKeyToFile(keyFileName, username) networkSecurityGroupResourceGroup = options.networkSecurityGroupResourceGroup networkSecurityGroup = options.networkSecurityGroup virtualNetworkResourceGroup = options.virtualNetworkResourceGroup virtualNetwork = options.virtualNetwork subnetName = options.subnetName computeResourceGroup = options.computeResourceGroup hostFqdnSuffix = options.hostFqdnSuffix dbHostOrIP = options.dbHostOrIP dbUsername = options.dbUsername dbPassword = options.dbPassword masterType = options.masterType.upper() workerType = options.workerType.upper() edgeType = options.edgeType.upper() dirUsername = options.dirUsername dirPassword = options.dirPassword logging.info('Assigning parameters ... Successful') logging.info('Modifying config ...') conf.put('name', name) conf.put('provider.region', region) conf.put('provider.subscriptionId', subscriptionId) conf.put('provider.tenantId', tenantId) conf.put('provider.clientId', clientId) conf.put('provider.clientSecret', clientSecret) conf.put('ssh.username', username) conf.put('ssh.privateKey', keyFileName) setInstanceParameters(conf, 'instances.master', masterType, networkSecurityGroupResourceGroup, networkSecurityGroup, virtualNetworkResourceGroup, virtualNetwork, subnetName, computeResourceGroup, hostFqdnSuffix) setInstanceParameters(conf, 'instances.worker', workerType, networkSecurityGroupResourceGroup, networkSecurityGroup, virtualNetworkResourceGroup, virtualNetwork, subnetName, computeResourceGroup, hostFqdnSuffix) setInstanceParameters(conf, 'instances.edge', edgeType, networkSecurityGroupResourceGroup, networkSecurityGroup, virtualNetworkResourceGroup, virtualNetwork, subnetName, computeResourceGroup, hostFqdnSuffix) setInstanceParameters(conf, 'cloudera-manager.instance', edgeType, networkSecurityGroupResourceGroup, networkSecurityGroup, virtualNetworkResourceGroup, virtualNetwork, subnetName, computeResourceGroup, hostFqdnSuffix) setInstanceParameters(conf, 'cluster.masters.instance', masterType, networkSecurityGroupResourceGroup, networkSecurityGroup, virtualNetworkResourceGroup, virtualNetwork, subnetName, computeResourceGroup, hostFqdnSuffix) setInstanceParameters(conf, 'cluster.workers.instance', masterType, networkSecurityGroupResourceGroup, networkSecurityGroup, virtualNetworkResourceGroup, virtualNetwork, subnetName, computeResourceGroup, hostFqdnSuffix) conf.put('databaseServers.mysqlprod1.host', dbHostOrIP) conf.put('databaseServers.mysqlprod1.user', dbUsername) conf.put('databaseServers.mysqlprod1.password', dbPassword) logging.info('Modifying config ... Successful') confLocation = DEFAULT_BASE_DIR + "/" + username + "/" + DEFAULT_CONF_NAME logging.info('Writing modified config to %s ...' % confLocation) with open(confLocation, "w") as text_file: text_file.write(tool.HOCONConverter.to_hocon(conf)) logging.info('Writing modified config to %s ... Successful' % confLocation) logging.info('Importing config to Cloudera Director server ...') command = "python setup-default.py --admin-username '%s' --admin-password '%s' '%s'" % ( dirUsername, dirPassword, confLocation) execAndLog(command) logging.info('Importing config to Cloudera Director server ... Successful')
def parse_config(): print("Parsing config file") return ConfigFactory.parse_file('../conf/application.conf')
from pyhocon import ConfigFactory conf = ConfigFactory.parse_file("application.conf") index_dir = conf.get("index_dir")
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from pyhocon import ConfigFactory import argparse import os import subprocess def get_args(): parser = argparse.ArgumentParser(description='Find controller.') parser.add_argument("-c" , "--conf", dest="conf_path", help="get controller from the config file") return parser.parse_args() def start(host, port, log_dir): if not os.path.exists(log_dir): os.makedirs(log_dir) cmd = 'nohup sbt "runMain psrs.Controller --host {0} --port {1}" > {2}/controller_{0}_{1}.log 2>&1 &'.format(host, port, log_dir) subprocess.Popen(cmd, shell=True) if __name__ == '__main__': args = get_args() if args.conf_path is not None: conf = ConfigFactory.parse_file(args.conf_path) host = conf['akka.remote.netty.tcp.hostname'] port = conf['akka.remote.netty.tcp.port'] log_dir = conf['psrs.log-dir'] start(host, port, log_dir) else: print "error: application.conf is not supplied!"
def post_helpdesk(): desc = request.form['desc'] captcha = request.form['hmm'] if captcha == 'scrop': status = "success" if send_email(request.form) else "fail" else: status = "fail" # should we bother restoring other fields beside desc? return redirect('/helpdesk?status={}&desc={}'.format(status, desc)) if __name__ == '__main__': app.config['DEBUG'] = True # tension between this and cfg function... conf = ConfigFactory.parse_file('cfg.conf') logfile = conf.get('logfile', '/tmp/cgi.log') logging.basicConfig(filename=logfile, level=logging.DEBUG) app.config['DATA_DIR'] = conf['guestbook_dir'] app.config['TRELLO_EMAIL'] = conf['trello_email'] app.config['MAILGUN_URL'] = conf['mailgun_url'] app.config['MAILGUN_KEY'] = conf['mailgun_key'] logging.debug("Running with data_dir=", app.config['DATA_DIR']) logging.debug(app.config) app.run()
def load_from_file(args): custom_settings_file = vars(args).get('custom_settings_file') if custom_settings_file and os.path.exists(custom_settings_file): return ConfigFactory.parse_file(custom_settings_file) else: return None
conn.request("GET", url + "/api/search/usage?notUsedSince=" + str(date) + "&repos=" + repo, headers=headers) res = conn.getresponse() data = res.read() json_object = json.loads(data) if 'results' in json_object: for value in json_object["results"]: if not_exclude(value["uri"], exclude): updated = value["uri"].replace("api/storage/", "") substr = "/artifactory/" pos = updated.index(substr) path = updated[pos:len(updated)] print path clean_path(conn, path, headers) config = ConfigFactory.parse_file("cleaner.conf") artifactory_url = urlparse(config.get("artifactory.url")) port = artifactory_url.port if artifactory_url.port else 80 if artifactory_url.scheme == 'http' else 443 conn = httplib.HTTPConnection(artifactory_url.hostname, port) if artifactory_url.scheme == 'http' else httplib.HTTPSConnection(artifactory_url.hostname, port) user_pass = b64encode(config.get("artifactory.auth")).decode("ascii") headers = { 'Authorization' : 'Basic %s' % user_pass } for value in config.get("repos"): outdated(conn, artifactory_url.path, headers, value.get('name'), value.get_int('interval'), config.get("exclude"))
from lib import helpers helpers.printCurrentTime("start ./amazon_statistics.py") import logging logging.basicConfig( format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) # CONFIG from pyhocon import ConfigFactory config = ConfigFactory.parse_file('./application.conf') amazon_dump_dir = config.get_string('amazon-dump.dir') movie_reviews_file = config.get_string('amazon-dump.files.reviews.movies') text_min_len = config.get_int('amazon-dump.statistics.review.text.min-len') statistic_measures_cache_dir = config.get_string( 'amazon-dump.statistics.measures.cache-dir') reviews_count_file = config.get_string( 'amazon-dump.statistics.measures.reviews-count.cache-file') number_of_reviews_by_asin_file = config.get_string( 'amazon-dump.statistics.measures.number-of-reviews-by-asin.cache-file') number_of_reviews_by_person_file = config.get_string( 'amazon-dump.statistics.measures.number-of-reviews-by-person.cache-file') bow_by_asin_file = config.get_string( 'amazon-dump.statistics.measures.bow-by-asin.cache-file') import pickle import os
#! /usr/bin/env python # Legacy support for sentences table in DeepDive. # The script reads the table from the database and stores it in the new column format. from pyhocon import ConfigFactory import json import psycopg2 import psycopg2.extras import sys import pipe conf = ConfigFactory.parse_file("../view.conf") docs = conf.get("view.docs") def find_token_offsets(s): # split on whitespace pos = [-1] + [i for i, ltr in enumerate(s) if ltr == " "] + [len(s)] offsets = [[pos[i] + 1, pos[i + 1]] for i in range(0, len(pos) - 1)] return offsets def write_docs(): # write extractions to json file dbconf = conf.get("view.db.default") conn_string = "host='%s' dbname='%s' user='******' password='******'" % ( dbconf.get("host"), dbconf.get("dbname"), dbconf.get("user"),
#!/usr/bin/env python ES_HOST = {"host" : "localhost", "port" : 9200} INDEX_NAME = 'view' TYPE_ANNOTATORS_NAME = 'annotators' TYPE_ANNOTATIONS_NAME = 'annotations' N = 1000 from pyhocon import ConfigFactory from elasticsearch import Elasticsearch import json import sys conf = ConfigFactory.parse_file('../view.conf') conf_annotations = conf.get_list('view.annotations') es = Elasticsearch(hosts = [ES_HOST]) # create a small table that only contains the names of all available extractors def index_annotators(): es.delete_by_query(index = INDEX_NAME, doc_type = TYPE_ANNOTATORS_NAME, body = { "query": { "match_all": {} } }) for ann in conf_annotations: es.index(index = INDEX_NAME, doc_type = TYPE_ANNOTATORS_NAME, body = { "name" : ann.get('name') }, refresh = False) es.indices.refresh(index = INDEX_NAME)
import pymongo from pyhocon import ConfigFactory from math import sin, cos, sqrt, atan2, radians from text import TextUtils from datetime import datetime # load the configuration conf = ConfigFactory.parse_file("conf/app.conf") def mongoClient(): # TODO make singleton / add other properties return pymongo.MongoClient() class Properties(object): # fields used for computing distances textFields = set(conf["data.textFields"]) # fields derived from textFields tokenizedTextFields = set(map(lambda _ : "tokenized_" + _, conf["data.textFields"])) class LatLong(object): """ Represents a (latitude, longitude) tuple. """ @staticmethod def distance(l1, l2): # Source: http://stackoverflow.com/questions/4913349/haversine-formula-in-python-bearing-and-distance-between-two-gps-points # Radius of earth in kilometers R = 6373.0 # convert to radians lat1 = radians(l1.latitude) lon1 = radians(l1.longitude)