def override_installer(platforms, installer, spec):
from glob import glob
from yaml import load as yaml_load
from os.path import basename, isfile
from copy import deepcopy
default_yaml = path_join(spec, '_default.yaml')
if isfile(default_yaml):
yaml = yaml_load(file(default_yaml).read())
default = yaml.get('default', {})
map(lambda x: x in yaml and default.update(yaml[x]), platforms)
else:
default = {}
installer['_default'].update(default)
for filename in glob(path_join(spec, '[a-z0-9A-Z]*.yaml')):
name = basename(filename)[0:-len('.yaml')]
yaml = yaml_load(file(filename).read())
result = deepcopy(installer['_default'])
result.update(yaml.get('default', {}))
map(lambda x: x in yaml and result.update(yaml[x]), platforms)
if name in installer:
installer[name].update(result)
else:
installer[name] = result
return installer
def create_form(draft, rmap, opts):
if opts.label not in rmap['forms']:
raise FormNotFoundError(label=opts.label)
with open(rmap['forms'][opts.label]) as y:
spec = yaml_load(y)
with open_datasource(rmap, spec['source']) as db:
initial = []
if 'comments' in spec:
wrapped = map(comment_yaml, wrap_text(spec['comments'], 78))
initial.extend(wrapped)
initial.append("# new values")
initial.append("# ----------")
avails = dict(map(lambda x: (x['name'], parse_column_default(x)),
spec['columns']))
initial.extend(yaml_dump(avails).split('\n'))
try:
if opts.values:
reviewed = avails
reviewed.update(parse_opt_values(opts.values))
else:
_, yaml_str = edit(draft, opts, initial="\n".join(initial))
reviewed = yaml_load(yaml_str)
db.execute(text(spec['insert']), **reviewed)
return 0, "Record created successfully!"
except (YAMLParserError, YAMLScannerError) as e:
err = EditError('YAML Parsing Error: %s' % str(e))
err.original_exception = e
raise err
except sqlexc.StatementError as e:
err = EditError('SQL Statement Error: %s' % str(e))
err.original_exception = e
raise err
def main(args=None):
"""
Execute the main body of the script.
Parameters
----------
args : list, optional
Command-line arguments. If unspecified, `sys.argv[1:]` is used.
"""
parser = argparse.ArgumentParser(description='Load a YAML file without '
'performing any training.')
parser.add_argument('yaml_file', type=argparse.FileType('r'),
help='The YAML file to load.')
parser.add_argument('-N', '--no-instantiate',
action='store_const', default=False, const=True,
help='Only verify that the YAML parses correctly '
'but do not attempt to instantiate the objects. '
'This might be used as a quick sanity check if '
'checking a file that requires a GPU in an '
'environment that lacks one (e.g. a cluster '
'head node)')
args = parser.parse_args(args=args)
name = args.yaml_file.name
initialize()
if args.no_instantiate:
yaml_load(args.yaml_file)
print("Successfully parsed %s (but objects not instantiated)." % name)
else:
load(args.yaml_file)
print("Successfully parsed and loaded %s." % name)
def __init__(self, config_file, sleep_file, hosts):
self.name, _ = splitext(basename(config_file))
self.sleep_file = sleep_file
with open(config_file) as f:
watcher_data = yaml_load(f)
monitor_name = watcher_data['monitor']['use']
monitor_data = watcher_data['monitor']
monitor_config = path_join('conf/monitors', monitor_name) + '.yaml'
self.rules = watcher_data['notification']
"""
`confirm' examples (3 out of 5 errors means a real error):
> monitor:
> - confirm: [3, 5]
"""
span = monitor_data['where']['confirm'][1]
self.moving_states = defaultdict(lambda: deque(maxlen=span))
self.events = dict()
self.freq = monitor_data['where']['frequency']
self.confirm_round = monitor_data['where']['confirm'][0]
with open(monitor_config) as f:
command_data = yaml_load(f)['command']
self.command_data = command_data
self.monitor_data = monitor_data
self.monitor_config = monitor_config
self.hosts = hosts
self.workers = int(monitor_data['where']['workers'])
def config_from_yaml(filepath=None, fileobj=None, silent=False, BANNED_SETTINGS=[], upper_only=False, parse_env=False, debug=False, **kwargs):
"""
Usage:
>>> from StringIO import StringIO
>>> fileobj = StringIO('DEBUG: true')
>>> config = config_from_yaml(fileobj=fileobj)
>>> config.get('DEBUG')
True
#>>> os.getenv('TEST_HOSTNAME') == 'MYTEST'
#True
#>>> fileobj = StringIO('HOSTNAME: ENV_TEST_HOSTNAME')
#>>> config = config_from_yaml(fileobj=fileobj, parse_env=True)
#>>> config.get('HOSTNAME')
#'MYTEST'
"""
if not filepath and not fileobj:
if silent is False:
raise ValueError(u"Not filepath or fileobj in parameters")
else:
return {}
if filepath and not os.path.exists(filepath):
if silent is False:
raise ValueError("file not found : %s" % filepath)
else:
return {}
config = {}
dict_update = {}
try:
if not fileobj:
with open(filepath, 'r') as fp:
config = yaml_load(fp, Loader=YAMLLoader)
else:
config = yaml_load(fileobj, Loader=YAMLLoader)
except:
if silent is False:
raise
try:
for k, v in six.iteritems(config):
if upper_only and not k.isupper():
continue
if not k in BANNED_SETTINGS:
dict_update[k] = v
except:
if silent is False:
raise
if parse_env:
dict_update = replace_with(dict_update.copy())
return dict_update
def is_yaml_string_valid(yaml_string):
try:
yaml_load(yaml_string)
except ConstructorError:
return True
except:
return False
return True
def _init(_yaml):
_setting = dict()
with codecs.open(yaml, "r", encoding="utf-8") as f:
_setting = yaml_load(f.read())
_setting["servers"] = dict(map(lambda x: (x["name"], x),
_setting["servers"]))
_setting["channels"] = dict(
map(lambda x: ((x["server"], x["name"]), x), _setting["channels"]))
if "users" in _setting:
_setting["users"] = map(compile_regex, _setting["users"])
else:
_setting["users"] = list()
# rearrange handlers' data structure
try:
h = dict(privmsg=list(), user_joined=list(), joined=list())
for item in _setting["handlers"]:
if "rewrites" in item:
item["rewrites"] = map(compile_regex, item["rewrites"])
if item["type"] in h:
data = dict(map(_compile_regex, item.items()))
h[item["type"]].append(data)
else:
log.msg("invalid message type: %s" % item["type"])
_setting["handlers"] = h
except Exception as e:
raise ConfigError("malformed handler configuration: %s" % str(e))
return _setting
def parse_all_kinds(yaml_path, sort_rank_path, include_kind_detail):
from glob import glob
from os.path import join, split, exists
from yaml import load as yaml_load
from vectordatasource.transform import CSVMatcher
all_kinds = {}
for yaml_file in glob(join(yaml_path, '*.yaml')):
layer_name = split(yaml_file)[1][:-5]
with open(yaml_file, 'r') as fh:
yaml_data = yaml_load(fh)
# by default, we don't have any sort_rank information
sort_rank = no_matcher
# look for a spreadsheet for sort_rank
csv_file = join(yaml_path, '..', 'spreadsheets', 'sort_rank',
'%s.csv' % layer_name)
if exists(csv_file):
with open(csv_file, 'r') as fh:
sort_rank = CSVMatcher(fh)
for item in yaml_data['filters']:
kinds = parse_item(layer_name, item, sort_rank,
include_kind_detail)
for k, v in kinds.iteritems():
prev_v = all_kinds.get(k)
if prev_v is not None:
v = merge_info(prev_v, v)
all_kinds[k] = v
return all_kinds
def get_yaml_from_mercurial(vcs_address, vcs_subdir):
from mercurial import ui, commands
from urllib2 import HTTPError
import hglib
vtemp = mkdtemp(prefix='multipkg-vcs-')
try:
commands.clone(ui.ui(), str(vcs_address), dest=vtemp)
client = hglib.open(vtemp)
# get index.yaml
path_to_yaml = path_join(vtemp, vcs_subdir, 'index.yaml')
yaml = yaml_load(file(path_to_yaml).read())
recent_changes = []
for entry in client.log('tip:tip^^'):
num, rev, none, branch, author, msg, date = entry
date = date.strftime('%Y-%m-%d %H:%M:%S')
recent_changes.append("commit %s | Author: %s | Date: %s \n%s\n" %
(rev, author, date, msg))
yaml['.'] = dict(recent_changes="\n".join(recent_changes))
return yaml
except HTTPError:
raise RemotePackageNotFoundError(vcs_address)
except IOError as e:
if e.errno == errno.ENOENT and e.filename.find('.yaml') > -1:
raise IndexNotFoundError('index.yaml not found in your repository')
raise
except:
raise
finally:
if isdir(vtemp):
rmtree(vtemp)
def __init__(self,
config = '/etc/chasm/zone.conf', auto_start = True):
self.cfg = yaml_load(open(config))
address = (self.cfg["server"]["address"], self.cfg["server"]["port"])
self._address = address
# this will contain all registered packet handlers
# the lock protects this from adds/gets during _receiving_loop
self._packet_handlers_lock = RLock()
self._packet_handlers = {}
# this is an arbitrary packet identifier which handlers can register
# if they want to be invoked when an address disconnects in the core.
# See the SessionManager._handle_disconnect() for an example.
self.DISCONNECT_PACKET_ID = "DISCONNECTED"
self._local_packet_handlers = {
c2s_packet.ArenaEnter._id : self._handle_arena_enter,
c2s_packet.ArenaLeave._id : self._handle_arena_leave,
self.DISCONNECT_PACKET_ID : self._handle_disconnect,
}
self.add_packet_handlers(**self._local_packet_handlers)
self.core = server.Server(self._address)
self.shutting_down = Event()
self._threads = {
"recv" : Thread(target=self._receiving_loop,name="Zone:recv")
}
# ping server runs on port + 1
ping_address = address[0], address[1] + 1
self.ping_server = ping.PingServer(ping_address, self)
self.sessions = session.SessionManager(self)
self.message = message.Messenger(self, self.cfg["messaging"])
self.arenas = [arena.Arena(self, arena_name, self.cfg["arenas"][arena_name])
for arena_name in self.cfg["arenas"].iterkeys()]
def _load_token(file):
with open(file) as tokenfile:
token = yaml_load(tokenfile)
return OAuth1(token['api_key'],
client_secret = token['api_secret'],
resource_owner_key = token['token'],
resource_owner_secret = token['token_secret'])
def get_yaml_from_subversion(vcs_address, vcs_subdir):
import pysvn
vtemp = mkdtemp(prefix='multipkg-vcs-')
client = pysvn.Client()
client.exception_style = 1
try:
client.checkout(vcs_address, vtemp, depth=pysvn.depth.empty)
# get index.yaml
path_to_yaml = path_join(vtemp, vcs_subdir, 'index.yaml')
client.update(path_to_yaml)
yaml = yaml_load(file(path_to_yaml).read())
recent_changes = []
for entry in client.log(vcs_address, limit=3):
date = datetime.datetime.fromtimestamp(int(entry.date))
date = date.strftime('%Y-%m-%d %H:%M:%S')
fmt = "revision #%-8s | Author: %-20s | Date: %-20s | Comment: %s"
recent_changes.append(fmt % (entry.revision.number,
entry.author, date, entry.message))
yaml['.'] = dict(recent_changes="\n".join(recent_changes))
return yaml
except pysvn.ClientError as e:
message, code = e.args[1][0]
if code == 170000:
raise RemotePackageNotFoundError(vcs_address)
else:
raise
except IOError as e:
if e.errno == errno.ENOENT and e.filename.find('.yaml') > -1:
raise IndexNotFoundError('index.yaml not found in your repository')
raise
finally:
if isdir(vtemp):
rmtree(vtemp)
def __init__(self, conf_file=None, defs_file=None):
self._base_defs = {}
self._plugins = {}
self._parsers = {}
self._config_file = conf_file
self.data = ConfigData()
if defs_file is None:
# Create configuration definitions from source
b_defs_file = to_bytes('%s/base.yml' % os.path.dirname(__file__))
else:
b_defs_file = to_bytes(defs_file)
# consume definitions
if os.path.exists(b_defs_file):
with open(b_defs_file, 'rb') as config_def:
self._base_defs = yaml_load(config_def, Loader=SafeLoader)
else:
raise AnsibleError("Missing base configuration definition file (bad install?): %s" % to_native(b_defs_file))
if self._config_file is None:
# set config using ini
self._config_file = find_ini_config_file()
# consume configuration
if self._config_file:
if os.path.exists(self._config_file):
# initialize parser and read config
self._parse_config_file()
# update constants
self.update_config_data()
def run(self):
"""
Implements the directive
"""
# Get content and options
file_path = self.options.get('file', None)
type = self.options.get('type', 'all')
if not file_path:
return [self._report('file_path -option missing')]
projects = yaml_load(open(self._get_directive_path(file_path)).read())
ret = []
for project in projects:
project = projects[project]
if type == 'top' and project['is_primary'] == False:
continue
node = doctrineprojects()
node['project'] = project
node['type'] = type
node['sort'] = project['sort']
ret.append(node)
ret.sort(key=operator.itemgetter('sort'))
return ret
def load_yaml_config(settings=None, default_config=None):
if not HAVE_YAML:
raise Exception("PyYAML is not installed\n")
default_config = default_config or {}
if isinstance(settings, list):
found = False
for s in settings:
if not s:
continue
filepath = os.path.abspath(os.path.expanduser(s))
logger.debug("search in %s" % filepath)
if filepath and os.path.exists(filepath):
logger.info("load from %s" % filepath)
settings = filepath
found = True
break
if not found:
raise Exception("file not found in all paths")
stream = settings
if isinstance(settings, string_types):
with open(settings) as fp:
stream = StringIO(fp.read())
yaml_config = yaml_load(stream)
default_config.update(yaml_config)
return default_config
def load_from_path(self, path):
self.path = path
if not os.path.exists(path):
console_ui.emit_error("Error", "Path does not exist")
return False
if not os.path.isfile and not os.path.islink(path):
console_ui.emit_error("Error", "File specified is not a file")
return False
filename = os.path.basename(path)
# We'll get rid of this at some point :P
if filename != "package.yml":
console_ui.emit_warning("Unnecessarily Anal Warning",
"File is not named package.yml")
# Attempt to parse the input file
with open(path, "r") as inpfile:
try:
yaml_data = yaml_load(inpfile, Loader=Loader)
except Exception as e:
console_ui.emit_error("YAML", "Failed to parse YAML file")
print(e)
return False
b = self.load_from_data(yaml_data)
if not b:
return b
return self.load_component()
def load_params_layer(self, params_file):
"""
Load an individual layer of params from file on to params
"""
yaml_string = self.env.get_template(params_file).render(**self.params)
yaml_dict = yaml_load(yaml_string)
self.params.update(yaml_dict)
self.params.setdefault('_files', {})[params_file] = yaml_dict
def _load_rules(self, file_path):
config = {}
with open(file_path, 'r') as config_stream:
config = yaml_load(config_stream, Loader=YAMLLoader)
self._settings = config.get('settings', self._settings)
self._rules = config.get('rules', self._rules)
def config_load():
'''load the commandr config files from the users home dir and the local
meta/ folder and merge them together into one'''
global_config = dict()
local_config = dict()
global_config_file = path("$HOME").expand() / ".commandr.yml"
if global_config_file.exists():
global_stream = open(global_config_file, 'r')
global_config = yaml_load(global_stream)
local_config_file = path_meta() / "commandr.yml"
if local_config_file.exists():
local_stream = open(local_config_file, 'r')
local_config = yaml_load(local_stream)
return dict_merge(global_config, local_config)
def get_atomic_config():
# Returns the atomic configuration file (/etc/atomic.conf)
# in a dict
# :return: dict based structure of the atomic config file
if not os.path.exists(ATOMIC_CONF):
raise ValueError("{} does not exist".format(ATOMIC_CONF))
with open(ATOMIC_CONF, 'r') as conf_file:
return yaml_load(conf_file)
def __init__(self, config_file_name='config.yaml'):
"""
Instantiate a configuration session
:param config_file_name: The name of the file with configuration
:type config_file_name: ``str``
"""
with open(config_file_name, "r") as config_stream:
self.config = yaml_load(config_stream)
def config(self):
"""
Load the config from config_path
"""
if self._config is None:
with open(self.config_path) as handle:
self._config = yaml_load(handle.read().decode())
return self._config
def from_yaml(cls, yaml):
"""
Conver a YAML formatted string into a dictionnary.
Argument:
yaml: A string, A YAML formatted string.
"""
return yaml_load(yaml)
def set_env(self, env):
assert env in ['dev', 'prod']
ifs = open(os.path.join(self.base_path, 'ansible/container.yml'))
config = yaml_load(ifs)
ifs.close()
for service, service_config in config['services'].items():
dev_overrides = service_config.pop('dev_overrides', {})
if env == 'dev':
service_config.update(dev_overrides)
self._config = config
def __init__(self, filename):
filename = path(filename)
if not filename.exists():
error("Unable to load config from %s" % filename)
stream = open(filename, 'r')
self.config = yaml_load(stream)
if self.config is None:
self.config = dict()
def list_all_views(rmap):
t = PrettyTable(['Label', 'Title', 'YAML', 'Description'])
for label, path in rmap['views'].iteritems():
with open(path) as y:
spec = yaml_load(y.read())
t.add_row([label,
spec.get('title', ''),
path,
spec.get('description', '')])
out(t)
def parse_yaml(file_path):
"""
Parse a YAML file
:param file_path:
:return: dict which includes other dicts and lists made up for the parsed YAML elements
"""
with open(file_path, "r") as stream:
stream = yaml_load(stream)
return stream
def configure(self, c):
field, manifest, cache = dict(), dict(), dict()
yaml = c.get("extra", "field")
with codecs.open(yaml, "r", encoding="utf-8") as f:
tree = yaml_load(f.read())
for catname in tree:
field.update(tree[catname])
yaml = c.get("extra", "manifest")
with codecs.open(yaml, "r", encoding="utf-8") as f:
tree = yaml_load(f.read())
for catname in tree:
manifest.update(tree[catname])
yaml = c.get("extra", "cache")
with codecs.open(yaml, "r", encoding="utf-8") as f:
cache = yaml_load(f.read())
return YAMLConfiguration(field=field, manifest=manifest, cache=cache)
def configure(filename):
"""
Arguments:
- `filename`:
"""
global __config__
with codecs.open(filename, "r", encoding="utf-8") as f:
__config__ = yaml_load(f.read())
def _load_yaml_mapping(filename):
try:
f = open(filename)
try:
yaml_versions = yaml_load(f)
finally:
f.close()
except IOError:
yaml_versions = { }
return yaml_versions
from datetime import (
datetime,
timedelta)
from os import (
path,
makedirs)
from timeit import Timer
from time import clock, mktime
from hdfs import InsecureClient
from hdfs.ext.avro import AvroReader, AvroWriter
from pymongo import MongoClient
from yaml import safe_load as yaml_load
with open('mongo_creds.yml') as credsfile:
creds = yaml_load(credsfile)
hdfs_client = InsecureClient('http://localhost:14000', user='******')
mongo_client = MongoClient(host='uphsvlndc058.uphs.upenn.edu',port=27017)
is_authed = mongo_client.admin.authenticate(creds['user'],creds['pass'])
psPreds = mongo_client.psPreds.preds
def generate_avro_schema():
q = {'modelName':'sepsismodel_noEpoch'}
predictions = list(psPreds.find(q).limit(1))
fields = []
fields.append({
'name': 'event_id',
'type': 'string'})
def __main__():
allnodes = []
for node in dynConfig.keys():
if node[0] == '_' or 'noServer' in dynConfig[node]:
continue
allnodes.append(node)
nginxConfig = [
nginxMainTemplate.render(config=config, dynConfig=dynConfig, tags=tags)
]
certifierConfig = {
'sites': [],
'zones': [],
'siteips4': dynConfigFindClosest('siteips4'),
'siteips6': dynConfigFindClosest('siteips6'),
'sitecname': dynConfigFindClosest('sitecname'),
'allnodes': allnodes,
'gitrev': dynConfig['_self']['_gitrev'],
}
loadedSites = {}
reloadDNS = False
sites = []
for fn in listdir(SITECONFIGDIR):
if fn[0] == '.' or fn[-4:] != '.yml' or fn == '__main__.yml':
continue
sites.append(fn)
print('Found sites: %s' % ', '.join(sites))
for site_name_raw in sorted(sites):
oldName = path.join(OLDDIR, site_name_raw)
site_name = site_name_raw[:-4]
print('[%s] Processing...' % site_name)
site, lastModified = loadSite(site_name_raw)
if zoneTplLastModified > lastModified:
lastModified = zoneTplLastModified
site['zoneSerial'] = lastModified.replace(
tzinfo=timezone.utc).timestamp()
site['name'] = site_name
loadedSites[site_name] = site
oldSite = None
try:
fh = open(oldName, 'r')
oldSite = yaml_load(fh)
fh.close()
print('[%s] Loaded old site data' % site_name)
except:
pass
if not oldSite:
print('[%s] No old site data' % site_name)
oldSite = {
'domains': [],
'type': 'none',
}
oldSite['name'] = site_name
typeChanged = site['type'] != oldSite['type']
if typeChanged:
print('[%s] Type changed from %s to %s' %
(site_name, oldSite['type'], site['type']))
try:
loader = SITE_LOADERS[site['type']]
loader(site, oldSite, typeChanged)
print('[%s] Loaded site' % site_name)
except Exception as e:
print('[%s] Error loading site:' % site_name, e)
pass
domains = site['domains']
allDomains = domains
if site['type'] != 'none':
domainsChanged = oldSite['domains'] != domains
if domainsChanged:
print('[%s] Domains changed from %s to %s' %
(site_name, ','.join(
oldSite['domains']), ','.join(domains)))
redirectDomains = []
if 'redirectDomains' in site:
redirectDomains = site['redirectDomains']
else:
site['redirectDomains'] = redirectDomains
if 'redirectWWW' in site and site['redirectWWW']:
newDomains = []
for domain in sorted(domains):
if domain[0:4] != 'www.':
newDomains.append(domain)
continue
redirectDomains.append({
'from': domain,
'to': domain[4:],
})
domains = newDomains
if len(redirectDomains) > 0:
domSet = set(domains)
for rdom in redirectDomains:
dom = rdom['from']
if dom in domSet:
domSet.remove(dom)
domains = list(domSet)
site['domains'] = domains
allDomains = domains + [val['from'] for val in redirectDomains]
certSite = {'name': site_name, 'domains': allDomains}
certifierConfig['sites'].append(certSite)
nginxConfig.append(
nginxSiteTemplate.render(site=site,
config=config,
dynConfig=dynConfig,
tags=tags))
else:
print(
'[%s] Site is type none. Not rendering nginx or certifier config'
% site_name)
for domain in allDomains:
addZoneFor(domain, site)
for zone_name in sorted(zones):
zone = zones[zone_name]
certifierConfig['zones'].append({
'name': zone_name,
'domains': zone['domains'],
})
zoneFile = '/etc/powerdns/sites/db.%s' % zone_name
zoneConfig = bindZoneTemplate.render(zone=zone,
config=config,
dynConfig=dynConfig,
tags=tags)
if swapFile(zoneFile, zoneConfig):
reloadDNS = True
signedFile = '%s.signed' % zoneFile
try:
stat(signedFile)
except FileNotFoundError:
fh = open(signedFile, 'w')
fh.close()
reloadDNS = True
certifierConfStr = yaml_dump(certifierConfig)
nginxConfStr = '\n'.join(nginxConfig)
swapFile(path.join(CERTIFIER_DIR, 'config.yml'), certifierConfStr)
if writeGlobalTpl('ips.sh.j2', path.join(OUTDIR, 'ips.sh')):
system('bash \'%s\'' % path.join(OUTDIR, 'ips.sh'))
if writeGlobalTpl('bird/main4.conf.j2', '/etc/bird/bird.conf'):
system('service bird reload')
if writeGlobalTpl('bird/main6.conf.j2', '/etc/bird/bird6.conf'):
system('service bird6 reload')
if writeGlobalTpl('bind/named.conf.j2',
'/etc/powerdns/named.conf') or reloadDNS:
system('pdns_control rediscover && pdns_control reload')
if writeGlobalTpl('chrony/chrony.conf.j2', '/etc/chrony/chrony.conf'):
system('service chrony restart')
if writeNginxInclude('hsts') | \
writeNginxInclude('proxy') | \
writeNginxInclude('varnish') | \
writeNginxInclude('wellknown') | \
writeNginxInclude('securitytxt') | \
writeNginxInclude('headers') | \
writeGlobalTpl('nginx/s3auth.js.j2', '/etc/nginx/s3auth.js') | \
writeGlobalTpl('nginx/security.txt.j2', path.join(SITEDIR, 'security.txt')) | \
swapFile('/etc/nginx/conf.d/cdn.conf', nginxConfStr) | \
writeGlobalTpl('nginx/nginx.conf.j2', '/etc/nginx/nginx.conf'):
system('docker exec nginx killall -HUP nginx')
for name in loadedSites:
oldName = path.join(OLDDIR, '%s.yml' % name)
site = loadedSites[name]
fh = open(oldName, 'w')
fh.write(yaml_dump(site))
fh.close()
def loadSite(name):
fn = path.join(SITECONFIGDIR, name)
fh = open(fn, 'r')
data = fh.read()
fh.close()
return yaml_load(data), getGitTime(fn)
def __init__(self, filename):
with open(filename) as f:
self.rules = yaml_load(f)
######################################
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
## assign the video filename from arg
if (len(sys.argv) == 2):
videoFilename = sys.argv[1]
else:
videoFilename = "BlinkVideo.mp4"
logging.debug(f"provided filename: {videoFilename}")
# Load secrets
with open("{0}/secrets.yaml".format(secretsFileLocation), "r") as secretsFile:
try:
secret = yaml_load(secretsFile)
except YAMLError as exc:
print(exc)
# Get an authentication token from Blink
endpoint = "https://rest-prod.immedia-semi.com/login"
headers = {
"Host": "prod.immedia-semi.com",
"Content-Type": "application/json"
}
data = {
"email": secret["blinkUsername"],
"password": secret["blinkPassword"],
"client_specifier": "iPhone 9.2 | 2.2 | 222"
if __name__ == '__main__':
argParser = argparse.ArgumentParser(
prog="deselect_tests.py",
description=
"Produce pytest CLI options to deselect tests specified in yaml file",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
argParser.add_argument('conf_file', nargs=1, type=str)
argParser.add_argument('--absolute', action='store_true')
args = argParser.parse_args()
fn = args.conf_file[0]
if os.path.exists(fn):
with open(fn, 'r') as fh:
dt = yaml_load(fh, Loader=FullLoader)
if args.absolute:
base_dir = os.path.relpath(os.path.dirname(sklearn.__file__),
os.path.expanduser('~')) + '/'
else:
base_dir = ""
filtered_deselection = [
filter_by_version(test_name, sklearn_version)
for test_name in dt.get('deselected_tests', [])
]
pytest_switches = [
"--deselect " + base_dir + test_name
for test_name in filtered_deselection if test_name
]
def _load(self, s: str, d: dict):
config_type = self._get_config_type()
if config_type == 'yaml' or config_type == 'yml':
d.update(yaml_load(s, Loader=Loader))
elif config_type == 'json':
d.update(json.loads(s, encoding='utf-8'))
def nested_render(cfg, fully_rendered_cfgs, replacements):
"""
Template render the provided cfg by recurisevly replacing {{var}}'s which values
from the current "namespace".
The nested config is treated like nested namespaces where the inner variables
are only available in current block and further nested blocks.
Said the opposite way: the namespace with available vars that can be used
includes the current block's vars and parent block vars.
This means that you can do replacements for top-level
(global namespaced) config vars anywhere, but you can only use inner configs within
that block or further nested blocks.
An example is worth a thousand words:
---------------------------------------------------------------------------------
fence-config.yaml
--------------------------------------------------------------------------------
BASE_URL: 'http://localhost/user'
OPENID_CONNECT:
fence:
api_base_url: 'http://other_fence/user'
client_kwargs:
redirect_uri: '{{BASE_URL}}/login/fence/login'
authorize_url: '{{api_base_url}}/oauth2/authorize'
THIS_WONT_WORK: '{{api_base_url}}/test'
--------------------------------------------------------------------------------
"redirect_uri" will become "http://localhost/user/login/fence/login"
- BASE_URL is in the global namespace so it can be used in this nested cfg
"authorize_url" will become "http://other_fence/user/oauth2/authorize"
- api_base_url is in the current namespace, so it is available
"THIS_WONT_WORK" will become "/test"
- Why? api_base_url is not in the current namespace and so we cannot use that
as a replacement. the configuration (instead of failing) will replace with
an empty string
Args:
cfg (TYPE): Description
fully_rendered_cfgs (TYPE): Description
replacements (TYPE): Description
Returns:
dict: Configurations with template vars replaced
"""
try:
for key, value in cfg.iteritems():
replacements.update(cfg)
fully_rendered_cfgs[key] = {}
fully_rendered_cfgs[key] = nested_render(
value,
fully_rendered_cfgs=fully_rendered_cfgs[key],
replacements=replacements,
)
# new namespace, remove current vars (no longer available as replacements)
for old_cfg, value in cfg.iteritems():
replacements.pop(old_cfg, None)
return fully_rendered_cfgs
except AttributeError:
# it's not a dict, so lets try to render it. But only if it's
# truthy (which means there's actually something to replace)
if cfg:
try:
t = Template(str(cfg))
rendered_value = t.render(**replacements)
except TemplateSyntaxError:
rendered_value = cfg
try:
cfg = yaml_load(rendered_value)
except ScannerError:
# it's not loading into yaml, so let's assume it's a string with special
# chars such as: {}[],&*#?|:-<>=!%@\)
#
# in YAML, we have to "quote" a string with special chars.
#
# since yaml_load isn't loading from a file, we need to wrap the Python
# str in actual quotes.
cfg = yaml_load('"{}"'.format(rendered_value))
return cfg