def parse_config(self, config_file_path): """Parses the configuration file Args: config_file_path (string): path to the configuration file """ # Read main logfeeder configuration file staticconf.YamlConfiguration(config_file_path) self.aws_config_filepath = staticconf.read( 'logfeeder.aws_config_filepath') self.domain = staticconf.read('logfeeder.domain') app_file = staticconf.read('{0}.file'.format(self.APP_NAME)) # Read app specific configuration file contents = staticconf.YamlConfiguration(app_file) self.api_creds_filepath = staticconf.read('api_creds_filepath') if 'rate_limiter_num_calls_per_timeunit' in contents: self.rate_limiter = RateLimiter( calls_per_timeunit=staticconf.read_int( 'rate_limiter_num_calls_per_timeunit'), seconds_per_timeunit=staticconf.read_int( 'rate_limiter_num_seconds_per_timeunit'), ) self.sub_apis = {} for key in contents: if key.startswith('enable_'): name_of_subapi = key.split('enable_', 1)[1] self.sub_apis[name_of_subapi] = staticconf.read_bool(key) # If an API doesn't have any sub_apis, then set set its APP_NAME to self.sub_apis for code compatibility if not self.sub_apis: self.sub_apis = {self.APP_NAME: True}
def _load_module_configs(env_config_path: str): staticconf.YamlConfiguration(env_config_path) for config in staticconf.read_list('module_config', default=[]): if 'file' in config: staticconf.YamlConfiguration(config['file'], namespace=config['namespace']) staticconf.DictConfiguration(config.get('config', {}), namespace=config['namespace']) if 'initialize' in config: path = config['initialize'].split('.') function = path.pop() module_name = '.'.join(path) module = __import__(module_name, globals(), locals(), [path[-1]]) getattr(module, function)()
def load_package_config(filename, field='module_config', flatten=True): """Load the contents of a yaml configuration file using :func:`staticconf.loader.YamlConfiguration`, and use :func:`configure_packages` to load configuration at ``field``. Usage: .. code-block:: python from yelp_servlib import config_util config_util.load_package_config('config.yaml') :param filename: file path to a yaml config file with a `meta-config` section :param field: field in the file `filename`, which contains a `meta-config` section, which can be read by :func:`configure_packages`. Defaults to `module_config`. :param flatten: boolean for whether staticconf should load each module config with a flattened dictionary. Defaults to True. """ config = staticconf.YamlConfiguration(filename) configs = config.get(field) if configs is None: log.warning('Field: {field} was not found in {filename}'.format( field=field, filename=filename)) else: configure_packages(configs, flatten=flatten) return config
def main(): staticconf.YamlConfiguration(CONFIG_FILE) auth = OAuthHandler( staticconf.read_string('twitter.consumer_key'), staticconf.read_string('twitter.consumer_secret'), ) auth.set_access_token( staticconf.read_string('twitter.access_token'), staticconf.read_string('twitter.access_token_secret'), ) api = API(auth) big_ben_ids = [ 723360691457945600, 723375789467553793, 723390890664824834, 723405988221489154, 723421087703261186, 723436186644025344, 723451541563138052, 723466386304057344, 723481486737985536, 723497089410457600, 723511939465392128, 723528048931430400, 723541884208091137, 723556981991202816, 723572081485615104, 723587184276721665, 723602282374414338, 723617381017374720, 723632480964759553, 723647581516124160, 723662932664524800, 723678284538589184, 723693384272121857, 723709493939453952, 723723076614164480 ] for startid, endid in zip(big_ben_ids, big_ben_ids[1:]): for tweet in limit_handled( Cursor( api.search, q=' OR '.join( 'qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM1234567890' ), since_id=str(startid), max_id=str(endid), lang="en").items(2500)): print(json.dumps(tweet._json))
def main(): staticconf.YamlConfiguration(CONFIG_FILE) auth = OAuthHandler( staticconf.read_string('twitter.consumer_key'), staticconf.read_string('twitter.consumer_secret'), ) auth.set_access_token( staticconf.read_string('twitter.access_token'), staticconf.read_string('twitter.access_token_secret'), ) api = API(auth) big_ben_ids = [ 727256357607464960, 727271714187522048, 727287317912817664, 727302414039158785, 727317768509480960, 727332108876705794, 727347714380419072, 727362055750176768, 727377660742123520, 727393264060534784, 727407354162122753, 727422705876762624, 727437555210293248, 727452651210809344, 727468761842876416, 727483610119413760, 727498961741856768, 727513051440762881, 727528910452305921, 727543248458149888, 727559107612422144, 727574712830857221, 727588550133288961, 727603646221914113, 727619000348348416 ] for startid, endid in zip(big_ben_ids[13:], big_ben_ids[14:]): for tweet in limit_handled( Cursor( api.search, q=' OR '.join( 'qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM1234567890' ), since_id=str(startid), max_id=str(endid), lang="en").items(2500)): print(json.dumps(tweet._json))
def init(self): for file in self.files: try: staticconf.YamlConfiguration(file) except Exception as e: logging.warning( {'message': f'Can\'t open config file - {file} - {e}'})
def load_package_config(filename): """Load the contents of a yaml configuration file using :func:`staticconf.loader.YamlConfiguration`, and use :param filename: filename and path to a yaml config file :returns: the contents of ``filename`` as a dict """ return staticconf.YamlConfiguration(filename)
def get_clusterman_metrics(): try: import clusterman_metrics import clusterman_metrics.util.costs staticconf.YamlConfiguration( CLUSTERMAN_YAML_FILE_PATH, namespace="clusterman", ) staticconf.YamlConfiguration( CLUSTERMAN_METRICS_YAML_FILE_PATH, namespace="clusterman_metrics", ) except (ImportError, FileNotFoundError): clusterman_metrics = None return clusterman_metrics
def test_status_record_get_insert_string(): staticconf.YamlConfiguration("config.yaml") sr = StatusRecord(data_date="2014/10/13", et_status="wip", table_versions="search: 1") output_under_test = get_insert_string(sr.minimized_record()) expected_value = "INSERT INTO sdw_pipeline_status (data_date, et_status, "\ "table_versions) VALUES (%(data_date)s, %(et_status)s, "\ "%(table_versions)s)" assert expected_value == output_under_test
def test_status_record_get_update_string(): staticconf.YamlConfiguration("config.yaml") sr = StatusRecord(et_status="complete", et_runtime=1000) expected_value = "UPDATE sdw_pipeline_status SET et_runtime = "\ "%(et_runtime)s, et_status = %(et_status)s WHERE "\ "data_date = %(data_date)s AND "\ "table_versions = %(table_versions)s" output_under_test = get_update_string(sr.get_set_clause()) assert expected_value == output_under_test
def setup_class(cls): """SmartStack needs to be configured to use service discovery, so doing that for the entire class. """ if IS_OPEN_SOURCE_MODE: return staticconf.YamlConfiguration( '/nail/etc/services/services.yaml', namespace='smartstack_services' )
def load_config(config_path): """Reads the configuration from a YAML file located at `config_path`. """ config = {} staticconf.YamlConfiguration(config_path) # Get all brands for the ELG talk config['brands'] = staticconf.read('brands') return config
def main(): """TODO DESCRIBE THIS """ staticconf.YamlConfiguration(CONFIG) generate_biased_data( policy=policy.get_policy( complexity=staticconf.read('generate_data.policy.complexity'), num_features=staticconf.read('generate_data.num_features'), mean=staticconf.read('generate_data.policy.mean'), std=staticconf.read('generate_data.policy.std'), ), )
def get_clusterman_metrics(): try: import clusterman_metrics clusterman_yaml = CLUSTERMAN_YAML_FILE_PATH staticconf.YamlConfiguration(CLUSTERMAN_METRICS_YAML_FILE_PATH, namespace='clusterman_metrics') except (ImportError, FileNotFoundError): # our cluster autoscaler is not currently open source, sorry! clusterman_metrics = None clusterman_yaml = None return clusterman_metrics, clusterman_yaml
def load_cluster_pool_config(cluster: str, pool: str, scheduler: str, signals_branch_or_tag: Optional[str]) -> None: pool_namespace = POOL_NAMESPACE.format(pool=pool, scheduler=scheduler) pool_config_file = get_pool_config_path(cluster, pool, scheduler) staticconf.YamlConfiguration(pool_config_file, namespace=pool_namespace) if signals_branch_or_tag: staticconf.DictConfiguration( {'autoscale_signal': { 'branch_or_tag': signals_branch_or_tag }}, namespace=pool_namespace, )
def main(): """Connects to the stream and starts threads to write them to a file.""" staticconf.YamlConfiguration(CONFIG_FILE) listener = QueueListener() auth = OAuthHandler( staticconf.read_string('twitter.consumer_key'), staticconf.read_string('twitter.consumer_secret'), ) auth.set_access_token( staticconf.read_string('twitter.access_token'), staticconf.read_string('twitter.access_token_secret'), ) writer_thread = threading.Thread(target=worker, args=(listener,)) writer_thread.start() stream = Stream(auth, listener) print_status(listener) try: while True: try: # stream.sample() # blocking! stream.filter(track=["#airpodsmax"]) except KeyboardInterrupt: print('KEYBOARD INTERRUPT', file=sys.stderr) return except (socket.error): global tcpip_delay print( 'TCP/IP Error: Restarting after {delay} seconds.'.format( delay=tcpip_delay, ), file=sys.stderr, ) time.sleep(min(tcpip_delay, MAX_TCPIP_TIMEOUT)) tcpip_delay += 0.25 finally: print('Disconnecting stream', file=sys.stderr) stream.disconnect() print('Waiting for last tweets to finish processing', file=sys.stderr) # Send poison pill to writer thread and wait for it to exit listener.queue.put(None) # listener.queue.join() print('Waiting for writer thread to finish', file=sys.stderr) writer_thread.join() print('Exit successful', file=sys.stderr)
def _test_ussd_validation(self, yaml_to_validate, expected_validation, expected_errors): if self.validate_ussd: namespace = self.namespace + str(expected_validation) staticconf.YamlConfiguration(path + '/' + yaml_to_validate, namespace=namespace, flatten=False) ussd_screens = staticconf.config. \ get_namespace(namespace). \ get_config_values() is_valid, error_message = UssdView.validate_ussd_journey( ussd_screens) self.assertEqual(is_valid, expected_validation, error_message) self.assertDictEqual(error_message, expected_errors)
def train_contextual_bandit(iterate_data, train_model, score_actions): """ :param iterate_data: :type iterate_data: function :param train_model: :type train_model: function :param score_actions: :type score_actions: function :returns: The contextual bandit, which is a list of the models and probablities you choose them with. :rtype: list """ staticconf.YamlConfiguration(CONFIG) contextual_bandit = [] for epoch, training_data in enumerate(iterate_data()): _validate_training_data(training_data=training_data, epoch=epoch) propensity_info = inverse_propensity_weighting.get_propensity_info( possible_actions=training_data['possible_actions'], chosen_actions=training_data['chosen_actions'], contextual_bandit=contextual_bandit, epoch=epoch, score_actions=score_actions, rewards=training_data['rewards'], ) model = train_model( X=numpy.array(training_data['chosen_actions']), Y=numpy.array(training_data['rewards']), weighted_rewards=numpy.array( propensity_info['weighted_rewards'], ), ) contextual_bandit = contextual_bandit_utils.add_model( model=model, contextual_bandit=contextual_bandit, possible_actions=training_data['possible_actions'], chosen_actions=training_data['chosen_actions'], weighted_rewards=propensity_info['weighted_rewards'], min_probs=propensity_info['min_probs'], score_actions=score_actions, ) return contextual_bandit
def main(args: argparse.Namespace) -> None: staticconf.YamlConfiguration(args.config, flatten=False) backup_set_config = staticconf.read('backups')[args.name] staticconf.DictConfiguration(backup_set_config, namespace=args.name) backup_store = get_backup_store(args.name) if args.manifest: manifest = Manifest(args.filename) private_key_filename = backup_store.config.read('private_key_filename', default='') lock_manifest( manifest, private_key_filename, backup_store._save, backup_store._load, backup_store.options, ) else: with backup_store.unlock(): backup_store.save_if_new(args.filename)
def ingest_multiple_dates_main(args_namespace): """ Main entry point for this script. Will parse args passed in and run ET/L as appropriate. Does not throw any exception. """ results = [] start_time = time.time() try: load_io_yaml_from_args(args_namespace.io_yaml) sc.YamlConfiguration(args_namespace.config_override, optional=True) if args_namespace.serial_stepper: etl_stepper = ETLStepper(args_namespace) else: etl_stepper = ParallelEtStepper(args_namespace) if not args_namespace.load_only: try: results += etl_stepper.execute_et_steps() except ProcessError as pe: results += pe.args[0]['results'] pass if not args_namespace.et_only: try: results += etl_stepper.execute_load_steps() except ProcessError as pe: results += pe.args[0]['results'] pass except (KeyboardInterrupt, KeyboardInterruptError, BaseException) as e: step_result = _init_step_result( ETLStep(args_namespace, args_namespace.start_date.strftime("%Y-%m-%d"), None)) if type(e) in [KeyboardInterrupt, KeyboardInterruptError]: step_result['status'] = 'cancelled' else: step_result['error_info'] = _capture_error_info() step_result['start_time'] = start_time step_result['end_time'] = time.time() results.append(step_result) return results
def handle(self, *args, **options): error_message = {} for ussd_config in options["ussd_configs"]: if not os.path.isfile(ussd_config): raise CommandError( "This file path {} does not exist".format(ussd_config)) staticconf.YamlConfiguration(ussd_config, namespace="validation", flatten=False) ussd_screens = staticconf.config. \ get_namespace("validation"). \ get_config_values() is_valid, error_ussd_config_message = UssdView.validate_ussd_journey( ussd_screens) error_message[ussd_config] = dict( valid=is_valid, error_message=error_ussd_config_message) if not is_valid: raise CommandError(json.dumps(error_message)) self.stdout.write(json.dumps(error_message))
def load_ussd_screen(file_path, namespace): staticconf.YamlConfiguration( os.path.abspath(file_path), namespace=namespace, flatten=False)
def configure_packages(configs, flatten=True, log_keys_only=True): """Load configuration into a :class:`staticconf.config.ConfigNamespace` and optionally call an initialize function after loading configuration for a package. `configs` should be a `meta-config`, which is a list of configuration sections. Each section **must** have a `namespace` key and *may* have one or more other keys: namespace the name of a :class:`staticconf.config.ConfigNamespace` which will hold the configs loaded from a `file` or `config` in this section file the path to a yaml file. The contents of this file are loaded using :func:`staticconf.loader.YamlConfiguration` config a yaml mapping structure. The contents of this mapping are loaded using :func:`staticconf.loader.DictConfiguration` For each section in the `meta-config` the following operations are performed: 1. Load the contents of `file`, if the key is present 2. Load the contents of `config`, if the key is present. If values were loaded from `file`, these new `config` values will override previous values. Example configuration: .. code-block:: yaml module_config: - namespace: prod_ns config: access_log_name: tmp_service_<service_name> error_log_name: tmp_service_error_<service_name> - namespace: memcache config: clients: encoders: service_prefix: '<service_name>_service' Usage: .. code-block:: python configs = {...} # The contents of the example configuration above configure_packages(configs['module_config']) :param configs: List of config dicts. :param flatten: boolean for whether staticconf should load each module config with a flattened dictionary. Defaults to True. :param log_keys_only: boolean for whether staticonf should only log unknown keys """ for config in configs or []: # 1st load a yaml if 'file' in config: staticconf.YamlConfiguration( config['file'], namespace=config['namespace'], flatten=flatten, log_keys_only=log_keys_only, ) # 2nd update with a config dict if 'config' in config: staticconf.DictConfiguration( config['config'], namespace=config['namespace'], flatten=flatten, log_keys_only=log_keys_only, )
def load_config(path): staticconf.YamlConfiguration(path, namespace=CONFIG_NS)
def register_app_config(config_path): staticconf.YamlConfiguration(config_path, namespace='adjure') return staticconf.NamespaceReaders('adjure')
def load_config(path): staticconf.YamlConfiguration(path)
def setup_config(config_file: str) -> None: staticconf.YamlConfiguration(config_file, flatten=False) for backup_name, backup_config in staticconf.read('backups').items(): staticconf.DictConfiguration(backup_config, namespace=backup_name)
def load_config(): staticconf.YamlConfiguration('config.yaml') staticconf.YamlConfiguration('config-env.yaml', optional=True)
def configure_packages(configs, ignore_initialize=False, flatten=True): """Load configuration into a :class:`staticconf.config.ConfigNamespace` and optionally call an initialize function after loading configuration for a package. `configs` should be a `meta-config`, which is a list of configuration sections. Each section **must** have a `namespace` key and *may* have one or more other keys: namespace the name of a :class:`staticconf.config.ConfigNamespace` which will hold the configs loaded from a `file` or `config` in this section file the path to a yaml file. The contents of this file are loaded using :func:`staticconf.loader.YamlConfiguration` config a yaml mapping structure. The contents of this mapping are loaded using :func:`staticconf.loader.DictConfiguration` initialize a callable which accepts no arguments. It can be used to initialize state after configuration is loaded. For each section in the `meta-config` the following operations are performed: 1. Load the contents of `file`, if the key is present 2. Load the contents of `config`, if the key is present. If values were loaded from `file`, these new `config` values will override previous values. 3. Call `initialize`, if the key is present Example configuration: .. code-block:: yaml module_config: - namespace: yelp_pyramid config: access_log_name: tmp_service_<service_name> error_log_name: tmp_service_error_<service_name> - namespace: clog initialize: yelp_servlib.clog_util.initialize config: log_stream_name: <service_name> file: /nail/srv/configs/clog.yaml - namespace: yelp_memcache config: clients: encoders: service_prefix: '<service_name>_service' Usage: .. code-block:: python configs = {...} # The contents of the example configuration above configure_packages(configs['module_config']) :param configs: List of config dicts. :param ignore_initialize: Whether to ignore the `initialize` key in the config and not call it. This may be used to reload a config whose initialize is only intended to be called once. Defaults to `False`. Available from version >=4.5.4. :param flatten: boolean for whether staticconf should load each module config with a flattened dictionary. Defaults to True. """ for config in configs or []: # 1st load a yaml if 'file' in config: staticconf.YamlConfiguration(config['file'], namespace=config['namespace'], flatten=flatten) # 2nd update with a config dict if 'config' in config: staticconf.DictConfiguration(config['config'], namespace=config['namespace'], flatten=flatten) # 3rd call the initialize method if not ignore_initialize and 'initialize' in config: path = config['initialize'].split('.') function = path.pop() module_name = '.'.join(path) module = __import__(str(module_name), globals(), locals(), [str(path[-1])]) getattr(module, function)()
from pyfiglet import Figlet def _get_memos_dir(): dir_memos = staticconf.read_string('dir_memos', default='') # try absolute dir path_memos = Path(dir_memos) if path_memos and path_memos.exists() and path_memos.is_dir(): return path_memos # try relative dir path_memos = PATH_CWD / dir_memos if path_memos and path_memos.exists() and path_memos.is_dir(): return path_memos return None # global utilities FIGLET = Figlet(font='big') PATH_CWD = Path.cwd() PATH_CONFIG = PATH_CWD / 'config.yaml' PATH_RECENT_OPEN = PATH_CWD / 'recent_open.yaml' staticconf.YamlConfiguration(PATH_CONFIG.as_posix()) PATH_MEMOS = _get_memos_dir() EDITOR = staticconf.read_string('default_editor', default=os.getenv('EDITOR', 'vi')) SEARCH_FUZZY = staticconf.read_string('default_search_fuzzy', default='True').lower() == 'true' RECENT_OPEN_LIST_SIZE = staticconf.read_int('recent_open_list_size', default=10)