def parse_config(self, config_file_path):
"""Parses the configuration file
Args:
config_file_path (string): path to the configuration file
"""
# Read main logfeeder configuration file
staticconf.YamlConfiguration(config_file_path)
self.aws_config_filepath = staticconf.read(
'logfeeder.aws_config_filepath')
self.domain = staticconf.read('logfeeder.domain')
app_file = staticconf.read('{0}.file'.format(self.APP_NAME))
# Read app specific configuration file
contents = staticconf.YamlConfiguration(app_file)
self.api_creds_filepath = staticconf.read('api_creds_filepath')
if 'rate_limiter_num_calls_per_timeunit' in contents:
self.rate_limiter = RateLimiter(
calls_per_timeunit=staticconf.read_int(
'rate_limiter_num_calls_per_timeunit'),
seconds_per_timeunit=staticconf.read_int(
'rate_limiter_num_seconds_per_timeunit'),
)
self.sub_apis = {}
for key in contents:
if key.startswith('enable_'):
name_of_subapi = key.split('enable_', 1)[1]
self.sub_apis[name_of_subapi] = staticconf.read_bool(key)
# If an API doesn't have any sub_apis, then set set its APP_NAME to self.sub_apis for code compatibility
if not self.sub_apis:
self.sub_apis = {self.APP_NAME: True}
def _load_module_configs(env_config_path: str):
staticconf.YamlConfiguration(env_config_path)
for config in staticconf.read_list('module_config', default=[]):
if 'file' in config:
staticconf.YamlConfiguration(config['file'],
namespace=config['namespace'])
staticconf.DictConfiguration(config.get('config', {}),
namespace=config['namespace'])
if 'initialize' in config:
path = config['initialize'].split('.')
function = path.pop()
module_name = '.'.join(path)
module = __import__(module_name, globals(), locals(), [path[-1]])
getattr(module, function)()
def load_package_config(filename, field='module_config', flatten=True):
"""Load the contents of a yaml configuration file using
:func:`staticconf.loader.YamlConfiguration`, and use
:func:`configure_packages` to load configuration at ``field``.
Usage:
.. code-block:: python
from yelp_servlib import config_util
config_util.load_package_config('config.yaml')
:param filename: file path to a yaml config file with a `meta-config`
section
:param field: field in the file `filename`, which contains a `meta-config`
section, which can be read by :func:`configure_packages`.
Defaults to `module_config`.
:param flatten: boolean for whether staticconf should load each module
config with a flattened dictionary. Defaults to True.
"""
config = staticconf.YamlConfiguration(filename)
configs = config.get(field)
if configs is None:
log.warning('Field: {field} was not found in {filename}'.format(
field=field, filename=filename))
else:
configure_packages(configs, flatten=flatten)
return config
def main():
staticconf.YamlConfiguration(CONFIG_FILE)
auth = OAuthHandler(
staticconf.read_string('twitter.consumer_key'),
staticconf.read_string('twitter.consumer_secret'),
)
auth.set_access_token(
staticconf.read_string('twitter.access_token'),
staticconf.read_string('twitter.access_token_secret'),
)
api = API(auth)
big_ben_ids = [
723360691457945600, 723375789467553793, 723390890664824834,
723405988221489154, 723421087703261186, 723436186644025344,
723451541563138052, 723466386304057344, 723481486737985536,
723497089410457600, 723511939465392128, 723528048931430400,
723541884208091137, 723556981991202816, 723572081485615104,
723587184276721665, 723602282374414338, 723617381017374720,
723632480964759553, 723647581516124160, 723662932664524800,
723678284538589184, 723693384272121857, 723709493939453952,
723723076614164480
]
for startid, endid in zip(big_ben_ids, big_ben_ids[1:]):
for tweet in limit_handled(
Cursor(
api.search,
q=' OR '.join(
'qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM1234567890'
),
since_id=str(startid),
max_id=str(endid),
lang="en").items(2500)):
print(json.dumps(tweet._json))
def main():
staticconf.YamlConfiguration(CONFIG_FILE)
auth = OAuthHandler(
staticconf.read_string('twitter.consumer_key'),
staticconf.read_string('twitter.consumer_secret'),
)
auth.set_access_token(
staticconf.read_string('twitter.access_token'),
staticconf.read_string('twitter.access_token_secret'),
)
api = API(auth)
big_ben_ids = [
727256357607464960, 727271714187522048, 727287317912817664,
727302414039158785, 727317768509480960, 727332108876705794,
727347714380419072, 727362055750176768, 727377660742123520,
727393264060534784, 727407354162122753, 727422705876762624,
727437555210293248, 727452651210809344, 727468761842876416,
727483610119413760, 727498961741856768, 727513051440762881,
727528910452305921, 727543248458149888, 727559107612422144,
727574712830857221, 727588550133288961, 727603646221914113,
727619000348348416
]
for startid, endid in zip(big_ben_ids[13:], big_ben_ids[14:]):
for tweet in limit_handled(
Cursor(
api.search,
q=' OR '.join(
'qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM1234567890'
),
since_id=str(startid),
max_id=str(endid),
lang="en").items(2500)):
print(json.dumps(tweet._json))
def init(self):
for file in self.files:
try:
staticconf.YamlConfiguration(file)
except Exception as e:
logging.warning(
{'message': f'Can\'t open config file - {file} - {e}'})
def load_package_config(filename):
"""Load the contents of a yaml configuration file using
:func:`staticconf.loader.YamlConfiguration`, and use
:param filename: filename and path to a yaml config file
:returns: the contents of ``filename`` as a dict
"""
return staticconf.YamlConfiguration(filename)
def get_clusterman_metrics():
try:
import clusterman_metrics
import clusterman_metrics.util.costs
staticconf.YamlConfiguration(
CLUSTERMAN_YAML_FILE_PATH,
namespace="clusterman",
)
staticconf.YamlConfiguration(
CLUSTERMAN_METRICS_YAML_FILE_PATH,
namespace="clusterman_metrics",
)
except (ImportError, FileNotFoundError):
clusterman_metrics = None
return clusterman_metrics
def test_status_record_get_insert_string():
staticconf.YamlConfiguration("config.yaml")
sr = StatusRecord(data_date="2014/10/13",
et_status="wip",
table_versions="search: 1")
output_under_test = get_insert_string(sr.minimized_record())
expected_value = "INSERT INTO sdw_pipeline_status (data_date, et_status, "\
"table_versions) VALUES (%(data_date)s, %(et_status)s, "\
"%(table_versions)s)"
assert expected_value == output_under_test
def test_status_record_get_update_string():
staticconf.YamlConfiguration("config.yaml")
sr = StatusRecord(et_status="complete", et_runtime=1000)
expected_value = "UPDATE sdw_pipeline_status SET et_runtime = "\
"%(et_runtime)s, et_status = %(et_status)s WHERE "\
"data_date = %(data_date)s AND "\
"table_versions = %(table_versions)s"
output_under_test = get_update_string(sr.get_set_clause())
assert expected_value == output_under_test
def setup_class(cls):
"""SmartStack needs to be configured to use service discovery, so
doing that for the entire class.
"""
if IS_OPEN_SOURCE_MODE:
return
staticconf.YamlConfiguration(
'/nail/etc/services/services.yaml',
namespace='smartstack_services'
)
def load_config(config_path):
"""Reads the configuration from a YAML file located at
`config_path`.
"""
config = {}
staticconf.YamlConfiguration(config_path)
# Get all brands for the ELG talk
config['brands'] = staticconf.read('brands')
return config
def main():
"""TODO DESCRIBE THIS
"""
staticconf.YamlConfiguration(CONFIG)
generate_biased_data(
policy=policy.get_policy(
complexity=staticconf.read('generate_data.policy.complexity'),
num_features=staticconf.read('generate_data.num_features'),
mean=staticconf.read('generate_data.policy.mean'),
std=staticconf.read('generate_data.policy.std'),
),
)
def get_clusterman_metrics():
try:
import clusterman_metrics
clusterman_yaml = CLUSTERMAN_YAML_FILE_PATH
staticconf.YamlConfiguration(CLUSTERMAN_METRICS_YAML_FILE_PATH,
namespace='clusterman_metrics')
except (ImportError, FileNotFoundError):
# our cluster autoscaler is not currently open source, sorry!
clusterman_metrics = None
clusterman_yaml = None
return clusterman_metrics, clusterman_yaml
def load_cluster_pool_config(cluster: str, pool: str, scheduler: str,
signals_branch_or_tag: Optional[str]) -> None:
pool_namespace = POOL_NAMESPACE.format(pool=pool, scheduler=scheduler)
pool_config_file = get_pool_config_path(cluster, pool, scheduler)
staticconf.YamlConfiguration(pool_config_file, namespace=pool_namespace)
if signals_branch_or_tag:
staticconf.DictConfiguration(
{'autoscale_signal': {
'branch_or_tag': signals_branch_or_tag
}},
namespace=pool_namespace,
)
def main():
"""Connects to the stream and starts threads to write them to a file."""
staticconf.YamlConfiguration(CONFIG_FILE)
listener = QueueListener()
auth = OAuthHandler(
staticconf.read_string('twitter.consumer_key'),
staticconf.read_string('twitter.consumer_secret'),
)
auth.set_access_token(
staticconf.read_string('twitter.access_token'),
staticconf.read_string('twitter.access_token_secret'),
)
writer_thread = threading.Thread(target=worker, args=(listener,))
writer_thread.start()
stream = Stream(auth, listener)
print_status(listener)
try:
while True:
try:
# stream.sample() # blocking!
stream.filter(track=["#airpodsmax"])
except KeyboardInterrupt:
print('KEYBOARD INTERRUPT', file=sys.stderr)
return
except (socket.error):
global tcpip_delay
print(
'TCP/IP Error: Restarting after {delay} seconds.'.format(
delay=tcpip_delay,
),
file=sys.stderr,
)
time.sleep(min(tcpip_delay, MAX_TCPIP_TIMEOUT))
tcpip_delay += 0.25
finally:
print('Disconnecting stream', file=sys.stderr)
stream.disconnect()
print('Waiting for last tweets to finish processing', file=sys.stderr)
# Send poison pill to writer thread and wait for it to exit
listener.queue.put(None)
# listener.queue.join()
print('Waiting for writer thread to finish', file=sys.stderr)
writer_thread.join()
print('Exit successful', file=sys.stderr)
def _test_ussd_validation(self, yaml_to_validate, expected_validation,
expected_errors):
if self.validate_ussd:
namespace = self.namespace + str(expected_validation)
staticconf.YamlConfiguration(path + '/' + yaml_to_validate,
namespace=namespace,
flatten=False)
ussd_screens = staticconf.config. \
get_namespace(namespace). \
get_config_values()
is_valid, error_message = UssdView.validate_ussd_journey(
ussd_screens)
self.assertEqual(is_valid, expected_validation, error_message)
self.assertDictEqual(error_message, expected_errors)
def train_contextual_bandit(iterate_data, train_model, score_actions):
"""
:param iterate_data:
:type iterate_data: function
:param train_model:
:type train_model: function
:param score_actions:
:type score_actions: function
:returns: The contextual bandit, which is a list of the models and
probablities you choose them with.
:rtype: list
"""
staticconf.YamlConfiguration(CONFIG)
contextual_bandit = []
for epoch, training_data in enumerate(iterate_data()):
_validate_training_data(training_data=training_data, epoch=epoch)
propensity_info = inverse_propensity_weighting.get_propensity_info(
possible_actions=training_data['possible_actions'],
chosen_actions=training_data['chosen_actions'],
contextual_bandit=contextual_bandit,
epoch=epoch,
score_actions=score_actions,
rewards=training_data['rewards'],
)
model = train_model(
X=numpy.array(training_data['chosen_actions']),
Y=numpy.array(training_data['rewards']),
weighted_rewards=numpy.array(
propensity_info['weighted_rewards'],
),
)
contextual_bandit = contextual_bandit_utils.add_model(
model=model,
contextual_bandit=contextual_bandit,
possible_actions=training_data['possible_actions'],
chosen_actions=training_data['chosen_actions'],
weighted_rewards=propensity_info['weighted_rewards'],
min_probs=propensity_info['min_probs'],
score_actions=score_actions,
)
return contextual_bandit
def main(args: argparse.Namespace) -> None:
staticconf.YamlConfiguration(args.config, flatten=False)
backup_set_config = staticconf.read('backups')[args.name]
staticconf.DictConfiguration(backup_set_config, namespace=args.name)
backup_store = get_backup_store(args.name)
if args.manifest:
manifest = Manifest(args.filename)
private_key_filename = backup_store.config.read('private_key_filename',
default='')
lock_manifest(
manifest,
private_key_filename,
backup_store._save,
backup_store._load,
backup_store.options,
)
else:
with backup_store.unlock():
backup_store.save_if_new(args.filename)
def ingest_multiple_dates_main(args_namespace):
""" Main entry point for this script. Will parse args passed in and run
ET/L as appropriate.
Does not throw any exception.
"""
results = []
start_time = time.time()
try:
load_io_yaml_from_args(args_namespace.io_yaml)
sc.YamlConfiguration(args_namespace.config_override, optional=True)
if args_namespace.serial_stepper:
etl_stepper = ETLStepper(args_namespace)
else:
etl_stepper = ParallelEtStepper(args_namespace)
if not args_namespace.load_only:
try:
results += etl_stepper.execute_et_steps()
except ProcessError as pe:
results += pe.args[0]['results']
pass
if not args_namespace.et_only:
try:
results += etl_stepper.execute_load_steps()
except ProcessError as pe:
results += pe.args[0]['results']
pass
except (KeyboardInterrupt, KeyboardInterruptError, BaseException) as e:
step_result = _init_step_result(
ETLStep(args_namespace,
args_namespace.start_date.strftime("%Y-%m-%d"), None))
if type(e) in [KeyboardInterrupt, KeyboardInterruptError]:
step_result['status'] = 'cancelled'
else:
step_result['error_info'] = _capture_error_info()
step_result['start_time'] = start_time
step_result['end_time'] = time.time()
results.append(step_result)
return results
def handle(self, *args, **options):
error_message = {}
for ussd_config in options["ussd_configs"]:
if not os.path.isfile(ussd_config):
raise CommandError(
"This file path {} does not exist".format(ussd_config))
staticconf.YamlConfiguration(ussd_config,
namespace="validation",
flatten=False)
ussd_screens = staticconf.config. \
get_namespace("validation"). \
get_config_values()
is_valid, error_ussd_config_message = UssdView.validate_ussd_journey(
ussd_screens)
error_message[ussd_config] = dict(
valid=is_valid, error_message=error_ussd_config_message)
if not is_valid:
raise CommandError(json.dumps(error_message))
self.stdout.write(json.dumps(error_message))
def load_ussd_screen(file_path, namespace):
staticconf.YamlConfiguration(
os.path.abspath(file_path),
namespace=namespace,
flatten=False)
def configure_packages(configs, flatten=True, log_keys_only=True):
"""Load configuration into a :class:`staticconf.config.ConfigNamespace`
and optionally call an initialize function after loading configuration
for a package.
`configs` should be a `meta-config`, which is a list of configuration
sections. Each section **must** have a `namespace` key and *may* have
one or more other keys:
namespace
the name of a :class:`staticconf.config.ConfigNamespace` which will hold
the configs loaded from a `file` or `config` in this section
file
the path to a yaml file. The contents of this file are loaded
using :func:`staticconf.loader.YamlConfiguration`
config
a yaml mapping structure. The contents of this mapping are loaded
using :func:`staticconf.loader.DictConfiguration`
For each section in the `meta-config` the following operations are
performed:
1. Load the contents of `file`, if the key is present
2. Load the contents of `config`, if the key is present. If values
were loaded from `file`, these new `config` values will override
previous values.
Example configuration:
.. code-block:: yaml
module_config:
- namespace: prod_ns
config:
access_log_name: tmp_service_<service_name>
error_log_name: tmp_service_error_<service_name>
- namespace: memcache
config:
clients:
encoders:
service_prefix: '<service_name>_service'
Usage:
.. code-block:: python
configs = {...} # The contents of the example configuration above
configure_packages(configs['module_config'])
:param configs: List of config dicts.
:param flatten: boolean for whether staticconf should load each module
config with a flattened dictionary. Defaults to True.
:param log_keys_only: boolean for whether staticonf should only log unknown keys
"""
for config in configs or []:
# 1st load a yaml
if 'file' in config:
staticconf.YamlConfiguration(
config['file'],
namespace=config['namespace'],
flatten=flatten,
log_keys_only=log_keys_only,
)
# 2nd update with a config dict
if 'config' in config:
staticconf.DictConfiguration(
config['config'],
namespace=config['namespace'],
flatten=flatten,
log_keys_only=log_keys_only,
)
def load_config(path):
staticconf.YamlConfiguration(path, namespace=CONFIG_NS)
def register_app_config(config_path):
staticconf.YamlConfiguration(config_path, namespace='adjure')
return staticconf.NamespaceReaders('adjure')
def load_config(path):
staticconf.YamlConfiguration(path)
def setup_config(config_file: str) -> None:
staticconf.YamlConfiguration(config_file, flatten=False)
for backup_name, backup_config in staticconf.read('backups').items():
staticconf.DictConfiguration(backup_config, namespace=backup_name)
def load_config():
staticconf.YamlConfiguration('config.yaml')
staticconf.YamlConfiguration('config-env.yaml', optional=True)
def configure_packages(configs, ignore_initialize=False, flatten=True):
"""Load configuration into a :class:`staticconf.config.ConfigNamespace`
and optionally call an initialize function after loading configuration
for a package.
`configs` should be a `meta-config`, which is a list of configuration
sections. Each section **must** have a `namespace` key and *may* have
one or more other keys:
namespace
the name of a :class:`staticconf.config.ConfigNamespace` which will hold
the configs loaded from a `file` or `config` in this section
file
the path to a yaml file. The contents of this file are loaded
using :func:`staticconf.loader.YamlConfiguration`
config
a yaml mapping structure. The contents of this mapping are loaded
using :func:`staticconf.loader.DictConfiguration`
initialize
a callable which accepts no arguments. It can be used to initialize
state after configuration is loaded.
For each section in the `meta-config` the following operations are
performed:
1. Load the contents of `file`, if the key is present
2. Load the contents of `config`, if the key is present. If values
were loaded from `file`, these new `config` values will override
previous values.
3. Call `initialize`, if the key is present
Example configuration:
.. code-block:: yaml
module_config:
- namespace: yelp_pyramid
config:
access_log_name: tmp_service_<service_name>
error_log_name: tmp_service_error_<service_name>
- namespace: clog
initialize: yelp_servlib.clog_util.initialize
config:
log_stream_name: <service_name>
file:
/nail/srv/configs/clog.yaml
- namespace: yelp_memcache
config:
clients:
encoders:
service_prefix: '<service_name>_service'
Usage:
.. code-block:: python
configs = {...} # The contents of the example configuration above
configure_packages(configs['module_config'])
:param configs: List of config dicts.
:param ignore_initialize: Whether to ignore the `initialize` key in the
config and not call it. This may be used to
reload a config whose initialize is only intended
to be called once. Defaults to `False`.
Available from version >=4.5.4.
:param flatten: boolean for whether staticconf should load each module
config with a flattened dictionary. Defaults to True.
"""
for config in configs or []:
# 1st load a yaml
if 'file' in config:
staticconf.YamlConfiguration(config['file'],
namespace=config['namespace'],
flatten=flatten)
# 2nd update with a config dict
if 'config' in config:
staticconf.DictConfiguration(config['config'],
namespace=config['namespace'],
flatten=flatten)
# 3rd call the initialize method
if not ignore_initialize and 'initialize' in config:
path = config['initialize'].split('.')
function = path.pop()
module_name = '.'.join(path)
module = __import__(str(module_name), globals(), locals(),
[str(path[-1])])
getattr(module, function)()
from pyfiglet import Figlet
def _get_memos_dir():
dir_memos = staticconf.read_string('dir_memos', default='')
# try absolute dir
path_memos = Path(dir_memos)
if path_memos and path_memos.exists() and path_memos.is_dir():
return path_memos
# try relative dir
path_memos = PATH_CWD / dir_memos
if path_memos and path_memos.exists() and path_memos.is_dir():
return path_memos
return None
# global utilities
FIGLET = Figlet(font='big')
PATH_CWD = Path.cwd()
PATH_CONFIG = PATH_CWD / 'config.yaml'
PATH_RECENT_OPEN = PATH_CWD / 'recent_open.yaml'
staticconf.YamlConfiguration(PATH_CONFIG.as_posix())
PATH_MEMOS = _get_memos_dir()
EDITOR = staticconf.read_string('default_editor',
default=os.getenv('EDITOR', 'vi'))
SEARCH_FUZZY = staticconf.read_string('default_search_fuzzy',
default='True').lower() == 'true'
RECENT_OPEN_LIST_SIZE = staticconf.read_int('recent_open_list_size',
default=10)
