def parse_systemdir(systemdir):
"""
"""
files = set(iter(filter(lambda f: f.endswith('.yaml'), os.listdir(systemdir))))
systemfiles = list()
# parse default system file
if 'system.yaml' in files:
files.remove('system.yaml')
path = os.path.join(systemdir, 'system.yaml')
with open(path, 'r') as f:
streams = list(yaml.load_all(f))
if len(streams) > 0:
for i in range(len(streams)):
systemfiles.append(SystemFile(path, i, streams[i]))
else:
logger.warning("no systems found in system.yaml")
else:
logger.info("no system.yaml system file was found")
# parse all remaining system files
for systemfile in files:
path = os.path.join(systemdir, systemfile)
with open(path, 'r') as f:
streams = list(yaml.load_all(f))
if len(streams) > 0:
for i in range(len(streams)):
systemfiles.append(SystemFile(path, i, streams[i]))
else:
logger.warning("no systems found in %s", systemfile)
return systemfiles
def __init__(self, args):
# Load ballot records from yaml file
self.input = args[0]
self.precs = 0
try:
stream = open(self.input + '.yaml', 'r')
except:
print('Unable to open ' + self.input + '\n')
exit(0)
else:
a = audit_header.AuditHeader()
a.load_from_file(stream)
self.b = list(yaml.load_all(stream))
# If a jurisdiction_slate filename was given, then load its
# contents
if len(args) == 2:
try:
stream = open(args[1] + '.yaml', 'r')
except:
print('Unable to open ' + self.input + '\n')
exit(0)
else:
a = audit_header.AuditHeader()
a.load_from_file(stream)
self.j = yaml.load_all(stream)
else:
self.j = False
# Add the vote counts of candidates with the same ID# using
# sumation(). Write the vote totals for each candidate to the
# report stream.
self.serialize_csv(self.sumation())
def __init__(self, fn=None, content=None):
IDocumentFormatter.__init__(self)
if fn is not None:
with file(fn, "r") as f:
self.__content = load_all(f, Loader=Loader)
else:
self.__content = load_all(content, Loader=Loader)
def validate(self, **kwargs):
"""
Validates a submission file
:param file_path: path to file to be loaded.
:param data: pre loaded YAML object (optional).
:return: Bool to indicate the validity of the file.
"""
try:
submission_file_schema = json.load(open(self.default_schema_file, 'r'))
additional_file_section_schema = json.load(open(self.additional_info_schema, 'r'))
# even though we are using the yaml package to load,
# it supports JSON and YAML
data = kwargs.pop("data", None)
file_path = kwargs.pop("file_path", None)
if file_path is None:
raise LookupError("file_path argument must be supplied")
if data is None:
try:
# We try to load using the CLoader for speed improvements.
data = yaml.load_all(open(file_path, 'r'), Loader=yaml.CLoader)
except Exception as e: #pragma: no cover
data = yaml.load_all(open(file_path, 'r')) #pragma: no cover
for data_item in data:
if data_item is None:
continue
try:
if 'comment' in data_item:
validate(data_item, additional_file_section_schema)
else:
validate(data_item, submission_file_schema)
except ValidationError as ve:
self.add_validation_message(
ValidationMessage(file=file_path,
message=ve.message + ' in ' + str(ve.instance)))
if self.has_errors(file_path):
return False
else:
return True
except ScannerError as se: # pragma: no cover
self.add_validation_message( # pragma: no cover
ValidationMessage(file=file_path, message=
'There was a problem parsing the file. '
'This can be because you forgot spaces '
'after colons in your YAML file for instance. '
'Diagnostic information follows.\n' + str(se)))
return False
def __init__(self, election, record1, record2,
merge_output):
self.rstream = open(''.join([merge_output,'.log']), 'w')
# Load ballot records from yaml file
try:
stream = open(''.join([record1,'.yml']), 'r')
except IOError:
self.rstream.write(''.join(['Unable to open ',record1,'\n']))
print(''.join(['Unable to open ',record1,'\n']))
raise
else:
a = audit_header.AuditHeader()
a.load_from_file(stream)
guid1 = a.file_id
prov1 = a.provenance
self.b1 = list(yaml.load_all(stream))
stream.close()
try:
stream = open(''.join([record2,'.yml']), 'r')
except IOError:
self.rstream.write(''.join(['Unable to open ',record2,'\n']))
print(''.join(['Unable to open ',record2,'\n']))
raise
else:
a = audit_header.AuditHeader()
a.load_from_file(stream)
guid2 = a.file_id
prov2 = a.provenance
self.b2 = list(yaml.load_all(stream))
stream.close()
# Get the election specs from file
try:
stream = open(''.join([election,'.yml']), 'r')
except IOError:
self.rstream.write(''.join(['Unable to open ',record2,'\n']))
print(''.join(['Unable to open ',record2,'\n']))
raise
else:
a = audit_header.AuditHeader()
a.load_from_file(stream)
self.templ_type = a.type
self.e = yaml.load(stream)
stream.close()
if self.e.has_key('precinct_list'):
del self.e['precinct_list']
# Combine provenances and guids from input files
self.new_prov = []
self.new_prov.extend(prov1)
self.new_prov.extend(prov2)
self.new_prov.append(guid1)
self.new_prov.append(guid2)
def test_generating_simple_init(self, mock_conf):
mock_conf.yml_dir_path = RC_PATH
yml_path = _render_yml(self.init_variables, "generic-init.yml.j2")
with open(yml_path, 'r') as stream:
generated_files = yaml.load_all(stream.read())
with open("./fake_templates/simple-init.yml", 'r') as stream:
expected_files = yaml.load_all(stream.read())
try:
# compare pod
self.assertEqual(expected_files.next(), generated_files.next())
except StopIteration:
self.fail("Incorrect type of generated yml")
def test_generating_without_ports(self, mock_conf):
self.control_variables["ports"] = ['']
mock_conf.yml_dir_path = RC_PATH
yml_path = _render_yml(self.control_variables)
with open(yml_path, 'r') as stream:
generated_files = yaml.load_all(stream.read())
with open("./fake_templates/no_ports.yml", 'r') as stream:
expected_files = yaml.load_all(stream.read())
try:
# compare ReplicationController
self.assertEqual(expected_files.next(), generated_files.next())
except StopIteration:
self.fail("Incorrect type of generated yml")
def __init__(self,repo):
DataBase.__init__(self,"freecad",repo)
self.bases = []
self.base_classes = Links()
self.collection_bases = Links()
if not exists(self.backend_root):
e = MalformedRepositoryError("freecad directory does not exist")
e.set_repo_path(repo.path)
raise e
for coll in listdir(self.backend_root):
basefilename = join(self.backend_root,coll,"%s.base" % coll)
if not exists(basefilename):
#skip directory that is no collection
continue
try:
base_info = list(yaml.load_all(open(basefilename,"r","utf8"), Loader=yaml.FullLoader))
# FullLoader is not implemented in pyyaml < 5.1
except AttributeError:
# this is deprecated for newer pyyaml versions
base_info = list(yaml.load_all(open(basefilename,"r","utf8")))
if len(base_info) != 1:
raise MalformedCollectionError(
"Not exactly one YAML document found in file %s" % basefilename)
base_info = base_info[0]
for basefile in base_info:
if basefile["type"] == "function":
basepath = join(self.backend_root,coll,basefile["filename"])
if not exists(basepath):
raise MalformedBaseError("Python module %s does not exist" % basepath)
for func in basefile["functions"]:
try:
function = BaseFunction(func,basefile,coll,self.backend_root)
self.bases.append(function)
self.collection_bases.add_link(repo.collections[coll],function)
for id in func["classids"]:
if not id in repo.classes:
raise MalformedBaseError(
"Unknown class %s" % id)
if self.base_classes.contains_dst(repo.classes[id]):
raise NonUniqueBaseError(id)
self.base_classes.add_link(function,repo.classes[id])
except ParsingError as e:
e.set_base(basefile["filename"])
e.set_collection(coll)
raise e
else:
raise MalformedBaseError("Unknown base type %s" % basefile["type"])
def test_constructor(data_filename, canonical_filename, verbose=False):
_make_loader()
_make_canonical_loader()
native1 = None
native2 = None
try:
native1 = list(yaml.load_all(open(data_filename, 'rb'), Loader=MyLoader))
native2 = list(yaml.load_all(open(canonical_filename, 'rb'), Loader=MyCanonicalLoader))
assert native1 == native2, (native1, native2)
finally:
if verbose:
print("NATIVE1:")
pprint.pprint(native1)
print("NATIVE2:")
pprint.pprint(native2)
def test_generating_with_emprtDir_path(self, mock_conf):
self.control_variables["host_path"] = None
self.control_variables["container_path"] = "/foo/bar/container"
mock_conf.yml_dir_path = RC_PATH
yml_path = _render_yml(self.control_variables)
with open(yml_path, 'r') as stream:
generated_files = yaml.load_all(stream.read())
with open("./fake_templates/emptyDir.yml", 'r') as stream:
expected_files = yaml.load_all(stream.read())
try:
# compare ReplicationController
self.assertEqual(expected_files.next(), generated_files.next())
except StopIteration:
self.fail("Incorrect type of generated yml")
def ordered_load_all (stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
class OrderedLoader(Loader):
pass
def construct_mapping(loader, node):
loader.flatten_mapping(node)
#print 'NODE #############', node.start_mark.line, node.start_mark.column, node.value [0][0]
#if isinstance (node.value [0][0].value, (str, unicode)):
# lines [node.value[0][0].value] = node.start_mark.line
def add_line_map (node):
if hasattr (node, 'value'):
if isinstance (node.value, (str, unicode)):
lines [node.value] = node.start_mark.line
else:
#print type (node.value) # all 'list'
for subnode in node.value:
#print subnode [0]
add_line_map (subnode [0])
add_line_map (node)
# nope, these don't work, as the nodes are primitive types by the time they're loaded:
#node._line = node.start_mark.line
#node._column = node.start_mark.column
return object_pairs_hook(loader.construct_pairs(node))
OrderedLoader.add_constructor(
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
construct_mapping)
return yaml.load_all(stream, OrderedLoader)
def read_data(self, datafile):
with open(datafile, "rt") as f:
parsed = yaml.load_all(f)
layout = dict2obj(parsed.next())
meta = dict2obj(parsed.next())
content = dict2obj(parsed.next())
if layout.section != 'layout':
exit('layout document in "' + datafile + '" is malformed.')
elif meta.section != 'meta':
exit('meta document in "' + datafile + '" is malformed.')
elif content.section != 'content':
exit('content document in "' + datafile + '" is malformed.')
rows = { 'rows': [] }
if layout.header:
header = []
for cell in layout.header:
header.append([eval(cell)])
else:
header = None
for rownum in layout.rows:
parsed_cell = []
for cell in rownum.items()[0][1]:
parsed_cell.append(eval(cell))
rows['rows'].append( dict(zip(rownum.keys(), [parsed_cell])) )
# return header, rows
self.header = header
self.rows = rows
def load_djs(path, db):
with open(os.path.join(path, "djs.yaml")) as f:
for dj in yaml.load_all(f):
_populate_slug_and_name(dj)
_store_replacement(db["dj"], dj)
db["dj"][dj["slug"]] = dj
if "city" in dj:
city = dj["city"]
cityslug = slugify(city)
if cityslug not in db["city"]:
db["city"][cityslug] = dict(name=city, slug=cityslug, lint_created_by_dj=dj["slug"])
dj["city"] = cityslug
city_djs = db["city"][cityslug].setdefault("djs", [])
city_djs.append(dj["slug"])
# Normalize 'genre' to 'genres'.
genres = dj.setdefault("genres", [])
if "genre" in dj:
genres.append(dj.pop("genre"))
genreslugs = []
for genre in genres:
genreslug = slugify(genre)
if genreslug not in db["genre"]:
db["genre"][genreslug] = dict(name=genre, slug=genreslug, lint_created_by_dj=dj["slug"])
genreslugs.append(genreslug)
genredjs = db["genre"][genreslug].setdefault("djs", [])
genredjs.append(dj["slug"])
dj["genres"] = genreslugs
def read_yaml(yaml_filename):
docs=[]
with open(yaml_filename) as f:
docs=yaml.load_all(f)
docs=list(docs)
# docs=yaml.dump_all(ya, encoding='utf8', allow_unicode=True)
return docs
def update_all(self, form, force_rollback=False):
form = self._quote_element_names(form)
form_dict_list = yaml.load_all(form)
self.session.begin()
is_modified = False
for form_dict in form_dict_list:
element_name = form_dict['element_name']
ename = ObjectSpec.parse(element_name, expected=ElementName)
# update_all() does not look in the instance_cache, must exist in DB
instance = self.session.resolve_element_spec(ename)
self._update_instance(instance, form_dict)
is_modified = is_modified or self.session.is_modified(instance)
desc = self.session.create_change_description()
if force_rollback:
self.log.info("Forced Rollback")
self.session.rollback()
elif is_modified:
self.log.fine("Instance Modified: Commit")
self.session.commit()
else:
self.log.fine("Instance Unchanged: Rollback")
self.session.rollback()
return desc
def construct_case(filename, name):
"""
Parse a definition of a test case from a yaml file and construct the
TestCase subclass dynamically.
"""
def make_test(test_name, definition, i):
def m(self):
if name in SKIP_TESTS.get(self.es_version, ()):
raise SkipTest()
self.run_code(definition)
m.__doc__ = '%s:%s.test_from_yaml_%d (%s): %s' % (
__name__, name, i, '/'.join(filename.split('/')[-2:]), test_name)
m.__name__ = 'test_from_yaml_%d' % i
return m
with open(filename) as f:
tests = list(yaml.load_all(f))
attrs = {
'_yaml_file': filename
}
i = 0
for test in tests:
for test_name, definition in test.items():
if test_name == 'setup':
attrs['_setup_code'] = definition
continue
attrs['test_from_yaml_%d' % i] = make_test(test_name, definition, i)
i += 1
return type(name, (YamlTestCase, ), attrs)
def load_packages_multi(resources):
root = os.path.dirname(resources)
pkgs = list(yaml.load_all(open(resources)))[0]
use = []
for p in pkgs:
p['id_package'] = basename(p['dir'])
if not exists(join(root, p['dir'], 'setup.py')):
continue
src = join(root, p['dir'], 'src')
if not exists(src):
#print('unknonw %r' % src)
continue
roots = os.listdir(src)
for r in roots:
if 'egg-info' in r:
continue
if not isdir(join(src,r)):
continue
a = dict()
a['src'] = join(src,r)
a['repo'] = basename(p['dir'])
a['id_package'] = r
use.append(a)
return use
def parse_post(f):
parsed = yaml.load_all(f)
post_header = next(parsed)
if 'postready' in post_header and (not post_header['postready']):
print("Post not ready")
return None
f.seek(0)
l = f.readline()
while (l != '...\n') and (l != ''):
l = f.readline()
if l == '':
return None
comment = f.read()
post_header['comment'] = comment
imgs = []
if 'image1' in post_header and post_header['image1']:
with open(post_header['image1'], 'rb') as f:
imgs += [f.read()]
if 'image2' in post_header and post_header['image2']:
with open(post_header['image2'], 'rb') as f:
imgs += [f.read()]
if 'image3' in post_header and post_header['image3']:
with open(post_header['image3'], 'rb') as f:
imgs += [f.read()]
if 'image4' in post_header and post_header['image4']:
with open(post_header['image4'], 'rb') as f:
imgs += [f.read()]
post_header['imgs'] = imgs
return post_header
def kick(self, params):
"""
Dump a list of datasets which need to be kicked
"""
if not params.dataset:
print "Please specify --dataset"
raise SystemExit
match = re.compile("^.*?-P(.+)-R(\d+).root$").match
matches = ((f, match(f)) for f in listdir("."))
files = [(name, f.groups()) for name, f in matches if f]
files = sorted([(name, int(run), period.split("_")[-1]) for name, (period, run) in files])
_, data_info = load_all(open(params.dataset))
by_period, by_run = {}, {}
for d in data_info["datasets"]:
by_period.setdefault(d.period, []).append(d)
by_period.setdefault(d.period[0], []).append(d)
assert not d.run in by_run
by_run[d.run] = d
#by_run.setdefault(d.run, []).append(d)
if not exists("kicked"):
makedirs("kicked")
for name, run, period in files:
if period != by_run[run].period or period == "UNK":
print name, period, by_run[run].period
rename(name, pjoin("kicked", name))
def load_packages(resources):
root = os.path.dirname(resources)
pkgs = list(yaml.load_all(open(resources)))
use = []
for p in pkgs:
p['id_package'] = basename(p['dir'])
if not exists(join(root, p['dir'], 'setup.py')):
continue
src = join(root, p['dir'], 'src')
if not exists(src):
#print('unknonw %r' % src)
continue
roots = os.listdir(src)
this_pkgs = []
for r in roots:
if 'egg-info' in r:
continue
if not isdir(join(src,r)):
continue
this_pkgs.append(r)
p['src'] = src
p['packages'] = this_pkgs
use.append(p)
return use
def take_action(self, parsed_args):
self.log.debug('take_action(%s)' % parsed_args)
# set default max-width
if parsed_args.max_width == 0:
parsed_args.max_width = 80
client = self.app.client_manager.congressclient
with open(parsed_args.policy_file_path, "r") as stream:
policies = yaml.load_all(stream)
try:
body = next(policies)
except StopIteration:
raise Exception('No policy found in file.')
try:
body = next(policies)
raise Exception(
'More than one policy found in file. None imported.')
except StopIteration:
pass
data = client.create_policy(body)
def rule_dict_to_string(rules):
rule_str_list = [rule['rule'] for rule in rules]
return "\n".join(rule_str_list)
data['rules'] = rule_dict_to_string(data['rules'])
return zip(*sorted(six.iteritems(data)))
def test_constructor_types(data_filename, code_filename, verbose=False):
_make_objects()
native1 = None
native2 = None
try:
native1 = list(yaml.load_all(open(data_filename, 'rb'), Loader=MyLoader))
if len(native1) == 1:
native1 = native1[0]
native2 = _load_code(open(code_filename, 'rb').read())
try:
if native1 == native2:
return
except TypeError:
pass
if verbose:
print("SERIALIZED NATIVE1:")
print(_serialize_value(native1))
print("SERIALIZED NATIVE2:")
print(_serialize_value(native2))
assert _serialize_value(native1) == _serialize_value(native2), (native1, native2)
finally:
if verbose:
print("NATIVE1:")
pprint.pprint(native1)
print("NATIVE2:")
pprint.pprint(native2)
def _exec_module(self, module):
fullname = module.__name__
if yaml is None:
raise ImportError('PyYaml is not installed')
class MyYamlLoader(YamlLoader):
pass
for tag, func in iteritems(self._defaults):
@functools.wraps(func)
def constructor(loader, node):
return func(fullname, loader.construct_mapping(node))
yaml.add_constructor(tag, constructor, Loader=MyYamlLoader)
try:
stream = file(self.get_filename(fullname), 'r')
docs = yaml.load_all(stream, Loader=MyYamlLoader)
except IOError:
raise ImportError("IO error while reading a stream")
except yaml.YAMLError as e:
raise e # XXX convert into SyntaxError
else:
for doc in docs:
if not hasattr(doc, '__name__'):
continue
setattr(module, doc.__name__, doc)
def decomposeRedux(osTemplateFilename):
output = []
with open(osTemplateFilename, 'r') as f:
for data in yaml.load_all(f, Loader=yaml.Loader):
objVals = data['objects']
for obj in objVals:
filename = 'kube-' + data['metadata']['name']
fileType = None
outValue = {}
for objkey, objval in obj.items():
if objkey == 'kind' and objval == 'Service':
fileType = 'svc'
outValue[objkey] = objval
elif objkey == 'kind' and objval == 'ReplicationController':
fileType = 'rc'
outValue[objkey] = objval
elif objkey == 'kind' and objval == 'Pod':
fileType = 'po'
outValue[objkey] = objval
elif objkey == 'id':
filename = filename + '-' + objval
else:
outValue[objkey] = objval
filename = filename + '-' + fileType + '.yaml'
with open(filename, 'w') as outfile:
outfile.write(
yaml.dump(outValue, default_flow_style=False))
output.append(filename)
return output
def get(self, source, name='theme', div_id=''):
"""
Get theme from source stream.
Parameters
----------
source: str
name: str
div_id: str
"""
self.div_id = div_id
if len(source) > 0:
try:
for data in load_all(source):
if name in data:
for element in data[name]:
self.__get_copy_from(data=element)
self.__get_canvas(data=element)
self.__get_ordered_list(data=element)
self.__get_unordered_list(data=element)
self.__get_ordered_list_items(data=element)
self.__get_unordered_list_items(data=element)
self.__get_toc(data=element)
self.__get_slide(data=element)
self.__get_box_like(env='box', data=element)
self.__get_box_like(env='note', data=element)
self.__get_box_like(env='table', data=element)
self.__get_box_like(env='figure', data=element)
self.__get_box_like(env='video', data=element)
self.custom = True
except YAMLError:
print('No valid definition of theme has been found')
self.__get_css()
def _show_log(self, request):
log_data = "Log not found"
MIN_LEVEL = LEVEL_IMPORTANT
try:
MIN_LEVEL = int(request.args.get(b"level", [MIN_LEVEL])[0])
except ValueError as e:
logger.warn("Got invalid log 'level' in request arguments: "
"{level}", level=request.args[b"level"])
filename = None
date = bytes_to_str(request.args.get(b"date", [b"current"])[0])
if date == datetime.today().strftime("%Y-%m-%d"):
filename = "{}.yaml".format(self.channel)
elif date_regex.match(date):
filename = "{}.{}.yaml".format(self.channel, date)
elif date == "current":
filename = "{}.yaml".format(self.channel)
date = datetime.today().strftime("%Y-%m-%d")
if filename and os.path.isfile(os.path.join(self.log_dir, filename)):
with open(os.path.join(self.log_dir, filename)) as logfile:
log_data = '<table>'
for i, data in enumerate(yaml.load_all(logfile)):
if data["levelno"] > MIN_LEVEL:
_prepare_yaml_element(data)
log_data += line_templates[data["levelname"]].format(
index=i, **data)
log_data += '</table>'
request.write(str_to_bytes(log_page_template.format(
log_data=log_data, title=self.title, header=header,
footer=footer, channel=self.channel_link(), date=date,
Level=MIN_LEVEL)))
request.finish()
def main(argv=None):
parser = OptionParser(description='fabtests yaml parsing utility')
parser.add_option('-d', action='store_true', default=False, help=difference.__doc__)
parser.add_option('-v', action='store_true', default=False, help=pretty.__doc__)
(options, args) = parser.parse_args()
if len(args) == 0:
fd = sys.stdin
elif len(args) > 1:
class fd:
@staticmethod
def read():
r1 = list(map(open, args))
r2 = [x.read() for x in r1]
return reduce(operator.add, r2)
else:
fd = open(args[0], 'r')
yi = yaml.load_all(fd.read())
if options.d:
perfprint(difference(yi))
if options.v:
pretty(yi)
return 0
def prepare_reporting(mongo_addr, dbname, config_path):
from citizendesk.common.utils import get_logger
from citizendesk.common.dbc import mongo_dbs
from citizendesk.feeds.config import set_config
import citizendesk.outgest.dispatch as dispatch
logger = get_logger()
mongo_dbs.set_dbname(dbname)
DbHolder = namedtuple('DbHolder', 'db')
mongo_dbs.set_db(DbHolder(db=MongoClient(mongo_addr[0], mongo_addr[1])[mongo_dbs.get_dbname()]))
config_data = None
if config_path:
try:
cfghf = open(config_path)
config_data = cfghf.read()
cfghf.close()
config_yaml = yaml.load_all(config_data)
config_data = config_yaml.next()
config_yaml.close()
except:
config_data = None
logger.error('can not read config file: ' + str(config_path))
return False
dispatch.setup_blueprints(app, config_data)
return True
def _ImportBackupResources(self, file_name):
f = file(os.path.dirname(__file__) + file_name, 'r')
for res in yaml.load_all(f):
try:
author_key = models.Author.get_by_key_name(res['author_id'])
author_key2 = None
if 'author_id2' in res:
author_key2 = models.Author.get_by_key_name(res['author_id2'])
resource = models.Resource(
title=res['title'],
subtitle=res.get('subtitle') or None,
description=res.get('description') or None,
author=author_key,
second_author=author_key2,
url=unicode(res['url']),
social_url=unicode(res.get('social_url') or ''),
browser_support=res.get('browser_support') or [],
update_date=res.get('update_date'),
publication_date=res['publication_date'],
tags=res['tags'],
draft=False # These are previous resources. Mark them as published.
)
resource.put()
except TypeError:
pass # TODO(ericbidelman): Not sure why this is throwing an error, but
# ignore it, whatever it is.
f.close()
def construct_panes(self, fields_file):
"""Read the fields file and use the data to construct the
collapsible panes"""
fields_data = yaml.load_all(open(fields_file))
for pane_data in fields_data:
self.panes.append(Pane(self, pane_data))
self.Layout()
def validate(self, **kwargs):
"""
Validates a submission file.
:param file_path: path to file to be loaded.
:param data: pre loaded YAML object (optional).
:return: Bool to indicate the validity of the file.
"""
data_file_handle = None
return_value = False
try:
submission_file_schema = None
additional_file_section_schema = None
with file_opener(self.default_schema_file,
'r') as submission_schema:
submission_file_schema = json.load(submission_schema)
with file_opener(self.additional_info_schema,
'r') as additional_schema:
additional_file_section_schema = json.load(additional_schema)
# even though we are using the yaml package to load,
# it supports JSON and YAML
data = kwargs.pop("data", None)
file_path = kwargs.pop("file_path", None)
if file_path is None:
raise LookupError("file_path argument must be supplied")
if data is None:
data_file_handle = file_opener(file_path, 'r')
data = yaml.load_all(data_file_handle, Loader=Loader)
for data_item_index, data_item in enumerate(data):
if data_item is None:
continue
try:
if not data_item_index and 'data_file' not in data_item:
validate(data_item, additional_file_section_schema)
else:
validate(data_item, submission_file_schema)
except ValidationError as ve:
self.add_validation_message(
ValidationMessage(file=file_path,
message=ve.message + ' in ' +
str(ve.instance)))
if not self.has_errors(file_path):
return_value = True
except LookupError as le:
raise le
except ScannerError as se: # pragma: no cover
self.add_validation_message( # pragma: no cover
ValidationMessage(
file=file_path,
message='There was a problem parsing the file. '
'This can be because you forgot spaces '
'after colons in your YAML file for instance. '
'Diagnostic information follows.\n' + str(se)))
return_value = False
except Exception as e:
self.add_validation_message(
ValidationMessage(file=file_path, message=e.__str__()))
return_value = False
finally:
if data_file_handle:
data_file_handle.close()
return return_value
def generate_configuration(cls,
cluster_spec,
cvmfs=False,
cephfs=False,
cephfs_volume_size=None,
debug=False):
"""Generate Kubernetes manifest files used to init REANA cluster.
:param cluster_spec: Dictionary representing complete REANA
cluster spec file.
:param cephfs: Boolean which represents whether REANA is
deployed with CEPH or not.
:param cephfs_volume_size: Int to set CEPH volume size in GB.
:param cvmfs: Boolean which represents whether REANA is
deployed with CVMFS or not.
:param debug: Boolean which represents whether REANA is
deployed in debug mode or not.
:return: A generator/iterable of generated Kubernetes YAML manifests
as Python objects.
"""
# Setup an Environment for Jinja
env = Environment(loader=FileSystemLoader(
cls._conf['templates_folder']),
keep_trailing_newline=False)
# Define where are backend conf params needed when rendering templates.
be_conf_params_fp = cls._conf['templates_folder'] + '/config.yaml'
try:
with open(be_conf_params_fp) as f:
# Load backend conf params
backend_conf_parameters = yaml.load(f.read())
# change type of deployment (cephfs|cvmfs|hostpath)
if cephfs or cluster_spec['cluster'].get('cephfs'):
backend_conf_parameters['CEPHFS'] = True
if not cephfs_volume_size:
cephfs_volume_size = \
cluster_spec['cluster'].get(
'cephfs_volume_size',
200)
if debug or cluster_spec['cluster'].get('debug'):
backend_conf_parameters['DEBUG'] = True
if cluster_spec['cluster'].get('cephfs_monitors'):
backend_conf_parameters['CEPHFS_MONITORS'] = \
cluster_spec['cluster'].get('cephfs_monitors')
if cluster_spec['cluster'].get('root_path'):
backend_conf_parameters['ROOT_PATH'] = \
cluster_spec['cluster'].get('root_path')
if cluster_spec['cluster'].get('shared_volume_path'):
backend_conf_parameters['SHARED_VOLUME_PATH'] = \
cluster_spec['cluster'].get('shared_volume_path')
if cluster_spec['cluster'].get('db_persistence_path'):
backend_conf_parameters['DB_PERSISTENCE_PATH'] = \
cluster_spec['cluster'].get('db_persistence_path')
# Would it be better to combine templates or populated
# templates in Python code for improved extensibility?
# Just drop a .yaml template and add necessary to config.yaml
# without changing anything?
# Load template combining all other templates from
# templates folder
template = env.get_template('backend_conf.yaml')
components = cluster_spec['components']
rs_img = components['reana-server']['image']
rjc_img = components['reana-job-controller']['image']
rwfc_img = components['reana-workflow-controller']['image']
rwm_img = components['reana-workflow-monitor']['image']
rmb_img = components['reana-message-broker']['image']
rs_environment = components['reana-server']\
.get('environment', [])
rjc_environment = components['reana-job-controller'] \
.get('environment', [])
rwfc_environment = components['reana-workflow-controller'] \
.get('environment', [])
rwm_environment = components['reana-workflow-monitor'] \
.get('environment', [])
rmb_environment = components['reana-message-broker'] \
.get('environment', [])
rs_environment = components['reana-server']\
.get('environment', [])
rjc_environment = components['reana-job-controller'] \
.get('environment', [])
rwfc_environment = components['reana-workflow-controller'] \
.get('environment', [])
rwm_environment = components['reana-workflow-monitor'] \
.get('environment', [])
rmb_environment = components['reana-message-broker'] \
.get('environment', [])
rs_mountpoints = components['reana-server']\
.get('mountpoints', [])
rjc_mountpoints = components['reana-job-controller']\
.get('mountpoints', [])
rwfc_mountpoints = components['reana-workflow-controller']\
.get('mountpoints', [])
rwm_mountpoints = components['reana-workflow-monitor'] \
.get('mountpoints', [])
rmb_mountpoints = components['reana-message-broker'] \
.get('mountpoints', [])
# Render the template using given backend config parameters
cluster_conf = template.\
render(backend_conf_parameters,
REANA_URL=cluster_spec['cluster'].get(
'reana_url',
'reana.cern.ch'),
CEPHFS_VOLUME_SIZE=cephfs_volume_size or 1,
SERVER_IMAGE=rs_img,
JOB_CONTROLLER_IMAGE=rjc_img,
WORKFLOW_CONTROLLER_IMAGE=rwfc_img,
WORKFLOW_MONITOR_IMAGE=rwm_img,
MESSAGE_BROKER_IMAGE=rmb_img,
RS_MOUNTPOINTS=rs_mountpoints,
RJC_MOUNTPOINTS=rjc_mountpoints,
RWFC_MOUNTPOINTS=rwfc_mountpoints,
RWM_MOUNTPOINTS=rwm_mountpoints,
RMB_MOUNTPOINTS=rmb_mountpoints,
RS_ENVIRONMENT=rs_environment,
RJC_ENVIRONMENT=rjc_environment,
RWFC_ENVIRONMENT=rwfc_environment,
RWM_ENVIRONMENT=rwm_environment,
RMB_ENVIRONMENT=rmb_environment,
)
# Strip empty lines for improved readability
cluster_conf = '\n'.join([
line for line in cluster_conf.splitlines() if line.strip()
])
# Should print the whole configuration in a loop
# Now prints just memory address of generator object
logging.debug('Loaded K8S config successfully: \n {}'.format(
yaml.load_all(cluster_conf)))
except TemplateNotFound as e:
logging.info(
'Something wrong when fetching K8S config file templates from '
'{filepath} : \n'
'{error}'.format(filepath=cls._conf['templates_folder'],
error=e.strerror))
raise e
except TemplateSyntaxError as e:
logging.info('Something went wrong when parsing K8S template from '
'{filepath} : \n'
'{error}'.format(filepath=e.filename,
error=e.strerror))
raise e
except IOError as e:
logging.info(
'Something wrong when reading K8S config parameters-file from '
'{filepath} : \n'
'{error}'.format(filepath=be_conf_params_fp, error=e.strerror))
raise e
# As Jinja rendered string is basically multiple YAML documents in one
# string parse it with YAML-library and return a generator containing
# independent YAML documents (split from `---`) as Python objects.
return yaml.load_all(cluster_conf)
def load_all(stream, additional_vars={}):
for data in yaml.load_all(stream, loader.Loader):
yield env.interpolate(data, additional_vars)
def parse(cls, *args):
filename = args[0]
if filename is not None and os.path.exists(filename):
with open(filename) as fp:
return yaml.load_all(fp)
def load_dataset(dataset, queried_year, queried_value):
pointIndicatorGroup = pygame.sprite.Group()
if dataset == "aged population":
stream = urllib.request.urlopen(
"http://52.187.242.219:5000/data").read()
# stream = open("../data/output/data.yml")
xs = []
ys = []
indices = []
total_pops = []
aged_pops = []
proportional_pops = []
for data in yaml.load_all(stream):
for k in data.keys():
# import pdb; pdb.set_trace()
sa2_name = data[k]["sa2_name"]
index = 0
elder_population = 0
total_population = 0
proportional_pop = 0
for year in data[k]["by_year"]:
if (year['year'] != queried_year):
continue
index = year["index"]
elder_population = year["elder_population"]
total_population = year["total_population"]
if not total_population == 0:
proportional_pop = year["elder_population"] / year[
"total_population"]
# skip if we can't find a lat long
try:
lat, long = name_to_lat_long[sa2_name.rstrip(
" (ACT)").upper()]
except KeyError:
# print ("can't find long lat for '" + sa2_name.rstrip(" (ACT)").upper() +"'")
continue
x, y = lat_long_to_x_y(lat, long)
xs.append(x)
ys.append(y)
total_pops.append(total_population)
aged_pops.append(elder_population)
proportional_pops.append(proportional_pop)
indices.append(index)
# print(len(xs))
xs, ys = normalise_utm_coords(xs, ys)
total_pops = normalise(total_pops)
aged_pops = normalise(aged_pops)
proportional_pops = normalise(proportional_pops)
indices = normalise(indices)
values = {
"total_pops": total_pops,
"aged_pops": aged_pops,
"proportional_pops": proportional_pops,
"indices": indices
}
for i in range(len(xs)):
x = xs[i]
y = ys[i]
value = values[queried_value][i]
PointIndicator((x, y), value, pointIndicatorGroup)
if dataset == "test":
text_to_render = [] # list of (string, (x,y))
xs = []
ys = []
names = []
for _, name, lat, long in postcode_lat_long_list:
x, y = lat_long_to_x_y(lat, long)
xs.append(x)
ys.append(y)
names.append(name)
max_x = max(xs)
min_x = min(xs)
max_y = max(ys)
min_y = min(ys)
# normalise
for i in range(len(xs)):
x = (xs[i] - max_x) / (max_x - min_x)
y = -(ys[i] - max_y) / (max_y - min_y)
# print(x, y)
x *= map_width
y *= map_height
x += width
if (not names[i] in ("TOPLEFT", "BOTTOMRIGHT")):
pi = PointIndicator((x, y), 1, pointIndicatorGroup)
text_to_render.append((names[i], (x, y)))
return pointIndicatorGroup
if os.environ["SNAPPY_USE_STAGING_STORE"] == "1":
snap_ids = {
"test-snapd-tools": "02AHdOomTzby7gTaiLX3M3SGMmXDfLJp",
"test-snapd-devmode": "FcHyKyMiQh71liP8P82SsyMXtZI5mvVj",
"test-snapd-python-webserver": "uHjTANBWSXSiYzNOUXZNDnOSH3POSqWS",
}
else:
snap_ids = {
"test-snapd-tools": "eFe8BTR5L5V9F7yHeMAPxkEr2NdUXMtw",
"test-snapd-devmode": "821MII7GAzoRnPvTEb8R51Z1s9e0XmK5",
"test-snapd-python-webserver": "Wcs8QL2iRQMjsPYQ4qz4V1uOlElZ1ZOb",
}
res = list(yaml.load_all(sys.stdin))
equals("number of entries", len(res), 7)
check(
"basic",
res[0],
("name", equals, "basic"),
("summary", equals, "Basic snap"),
("path", matches, r"^basic_[0-9.]+_all\.snap$"),
("version", matches, verNotesRx),
("license", equals, "unset"),
("description", equals, "A basic buildable snap\n"),
("build-date", exists),
)
json_dump = get_response(api_link.format(curr1, curr2))
hasher = {}
return self.named_tuple(type="BTC",
price=json_dump["lprice"],
curr=json_dump["curr2"])
def send_messages(key, val, conf_obj, producer):
crypto_tuple = config.get_config(val, key)
value = json.dumps({
"price": str(crypto_tuple.price),
"curr": str(crypto_tuple.curr)
})
producer.send(topic=key, key=crypto_tuple.type, value=value)
print "Sending messages for", key
return producer
if __name__ == "__main__":
config = Config()
while True:
with open("ap_config.yaml", "r") as stream:
yaml_obj = yaml.load_all(stream)
[[
send_messages(key, val, config, producer)
for (key, val) in config_obj.items()
] for config_obj in yaml_obj]
sleep(15)
def load_yaml(self, yaml_str):
""" Load the blue print, if variables are defined substitute them
Note that the variables must be defined as an array """
def get_variables_dict(yaml_str):
""" get the defined variables """
# Can't read enitre yaml as jinja2 may cause invalid yaml
variables_str = ""
for line in yaml_str.splitlines():
if line.startswith('variables'):
variables_str += line + "\n"
elif len(variables_str) > 0:
variables_str += line + "\n"
if len(line.strip()) == 0:
break
if len(variables_str) == 0:
# no variables defined
return {}
blueprint = yaml.load(variables_str, Loader=yaml.Loader)
if 'variables' in blueprint:
# resubstitute vairables
var_dict = {}
for var_list in blueprint['variables']:
for key, val in var_list.items():
if type(val) == type(""):
val = val.format(**var_dict)
#if type(val)==type({}):
# if len(val)==1:
# val=var_dict[list(val)[0]]
var_dict[key] = val
return var_dict
def update_yaml(yaml_str, var_dict={}):
template = jinja2.Template(yaml_str)
yaml_str = template.render(**var_dict)
#open('temp.yaml','w').write(yaml_str)
blueprints = yaml.load_all(yaml_str, Loader=yaml.Loader)
return list(blueprints)
var_dict = get_variables_dict(yaml_str)
blueprints = update_yaml(yaml_str, var_dict)
# call import yamls
imported = {}
setup_bp = blueprints[0]
if 'import' in setup_bp:
for filename in setup_bp['import']:
with open(filename + '.yaml') as f:
imported = list(yaml.load_all(f.read(),
Loader=yaml.Loader))[0]
setup_bp = deep_update(imported, setup_bp)
#raise_error(list(setup_bp['analysis']))
self.doc_meta = self.get_doc_meta(setup_bp,
defaults={
'output_type': 'md',
'test': 'test'
})
self.doc_format = self.doc_meta['output_type']
blueprints[0] = setup_bp
return blueprints
def update_yaml(yaml_str, var_dict={}):
template = jinja2.Template(yaml_str)
yaml_str = template.render(**var_dict)
#open('temp.yaml','w').write(yaml_str)
blueprints = yaml.load_all(yaml_str, Loader=yaml.Loader)
return list(blueprints)
def load(self, *args):
with open(self.filename) as fp:
return [JobBase.unserialize(job) for job in yaml.load_all(fp) if job is not None]
import yaml
import io
filename = "dynatrace_conf.yaml"
# open the file
with open(filename, 'r') as f:
try:
docs = yaml.load_all(f, Loader=yaml.FullLoader)
for doc in docs:
for k,v in doc.items():
if "name" in k:
print k, "->", v
if "url" in k:
print k, "->", v
print "\n"
except yaml.YAMLError as exc:
print(exc)
import os
import sys
import subprocess as sp
import yaml
cfile = open('probe/config.yaml', 'r')
cconfig = yaml.load_all(cfile)
for datac in cconfig:
c = datac
def build_wav(source_video):
wav_out = c['wav_sources'] + os.path.split(source_video)[1] + '.wav'
try:
wav_cmd = [
'ffmpeg', '-y', '-i', source_video, '-vn', '-sn', '-map', '0:a',
wav_out
]
sp.check_output(wav_cmd, stderr=sp.STDOUT)
except sp.SubprocessError:
sys.exit("Error: WAV audio extraction")
finally:
return wav_out
def build_ogg(source_wav):
ogg_out = c['ogg_sources'] + os.path.split(source_wav)[1] + '.ogg'
try:
ogg_cmd = ['oggenc', source_wav, '-o', ogg_out]
sp.check_output(ogg_cmd, stderr=sp.STDOUT)
except sp.SubprocessError:
M_PER_TICK = SPEED_OF_LIGHT / LOCODECK_TS_FREQ
if len(sys.argv) > 1:
for arg in sys.argv[1:]:
if arg == 'm':
unit = 'meters'
elif arg == 't':
unit = 'ticks'
else:
print("usage: {} [unit]".format(sys.argv[0]))
print(
" unit: t = ticks (default), m = meters (excluding antenna delay)"
)
sys.exit(1)
for packet in yaml.load_all(sys.stdin, Loader=yaml.CLoader):
if not packet:
continue
data = {'id': packet['from'], 'tof': {}}
for remote in packet['remoteAnchorData']:
if 'distance' in remote:
tof = remote['distance']
remote_id = remote['id']
if unit == 'ticks':
data['tof'][remote_id] = tof
if unit == 'meters':
data['tof'][remote_id] = tof * M_PER_TICK - ANTENNA_OFFSET
print("---")
def yq(input_streams=None,
output_stream=None,
input_format="yaml",
output_format="json",
program_name="yq",
width=None,
indentless_lists=False,
xml_root=None,
xml_dtd=False,
xml_force_list=frozenset(),
explicit_start=False,
explicit_end=False,
jq_args=frozenset(),
exit_func=None):
if not input_streams:
input_streams = [sys.stdin]
if not output_stream:
output_stream = sys.stdout
if not exit_func:
exit_func = sys.exit
converting_output = True if output_format != "json" else False
try:
# Note: universal_newlines is just a way to induce subprocess to make stdin a text buffer and encode it for us
jq = subprocess.Popen(
["jq"] + list(jq_args),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE if converting_output else None,
universal_newlines=True)
except OSError as e:
msg = "{}: Error starting jq: {}: {}. Is jq installed and available on PATH?"
exit_func(msg.format(program_name, type(e).__name__, e))
try:
if converting_output:
# TODO: enable true streaming in this branch (with asyncio, asyncproc, a multi-shot variant of
# subprocess.Popen._communicate, etc.)
# See https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
use_annotations = True if output_format == "annotated_yaml" else False
input_docs = []
for input_stream in input_streams:
if input_format == "yaml":
loader = get_loader(use_annotations=use_annotations)
input_docs.extend(
yaml.load_all(input_stream, Loader=loader))
elif input_format == "xml":
import xmltodict
input_docs.append(
xmltodict.parse(input_stream.read(),
disable_entities=True,
force_list=xml_force_list))
elif input_format == "toml":
import toml
input_docs.append(toml.load(input_stream))
else:
raise Exception("Unknown input format")
input_payload = "\n".join(
json.dumps(doc, cls=JSONDateTimeEncoder) for doc in input_docs)
jq_out, jq_err = jq.communicate(input_payload)
json_decoder = json.JSONDecoder(object_pairs_hook=OrderedDict)
if output_format == "yaml" or output_format == "annotated_yaml":
yaml.dump_all(decode_docs(jq_out, json_decoder),
stream=output_stream,
Dumper=get_dumper(
use_annotations=use_annotations,
indentless=indentless_lists),
width=width,
allow_unicode=True,
default_flow_style=False,
explicit_start=explicit_start,
explicit_end=explicit_end)
elif output_format == "xml":
import xmltodict
for doc in decode_docs(jq_out, json_decoder):
if xml_root:
doc = {xml_root: doc}
elif not isinstance(doc, OrderedDict):
msg = (
"{}: Error converting JSON to XML: cannot represent non-object types at top level. "
"Use --xml-root=name to envelope your output with a root element."
)
exit_func(msg.format(program_name))
full_document = True if xml_dtd else False
try:
xmltodict.unparse(doc,
output=output_stream,
full_document=full_document,
pretty=True,
indent=" ")
except ValueError as e:
if "Document must have exactly one root" in str(e):
raise Exception(
str(e) +
" Use --xml-root=name to envelope your output with a root element"
)
else:
raise
output_stream.write(b"\n" if sys.version_info < (
3, 0) else "\n")
elif output_format == "toml":
import toml
for doc in decode_docs(jq_out, json_decoder):
if not isinstance(doc, OrderedDict):
msg = "{}: Error converting JSON to TOML: cannot represent non-object types at top level."
exit_func(msg.format(program_name))
if USING_PYTHON2:
# For Python 2, dump the string and encode it into bytes.
output = toml.dumps(doc)
output_stream.write(output.encode("utf-8"))
else:
# For Python 3, write the unicode to the buffer directly.
toml.dump(doc, output_stream)
else:
if input_format == "yaml":
loader = get_loader(use_annotations=False)
for input_stream in input_streams:
for doc in yaml.load_all(input_stream, Loader=loader):
json.dump(doc, jq.stdin, cls=JSONDateTimeEncoder)
jq.stdin.write("\n")
elif input_format == "xml":
import xmltodict
for input_stream in input_streams:
json.dump(
xmltodict.parse(input_stream.read(),
disable_entities=True,
force_list=xml_force_list), jq.stdin)
jq.stdin.write("\n")
elif input_format == "toml":
import toml
for input_stream in input_streams:
json.dump(toml.load(input_stream), jq.stdin)
jq.stdin.write("\n")
else:
raise Exception("Unknown input format")
try:
jq.stdin.close()
except Exception:
pass
jq.wait()
for input_stream in input_streams:
input_stream.close()
exit_func(jq.returncode)
except Exception as e:
exit_func("{}: Error running jq: {}: {}.".format(
program_name,
type(e).__name__, e))
def build_from_dir(dirname, output_file):
"""
Read a GlobalMind database exported in YAML files, translate
it into ConceptNet 5 edges, and write those edges to disk using
a JSONStreamWriter.
"""
out = JSONStreamWriter(output_file)
userdata = yaml.load_all(open(dirname + '/GMUser.yaml'))
users = {}
for userinfo in userdata:
users[userinfo['pk']] = userinfo
frame_data = yaml.load_all(open(dirname + '/GMFrame.yaml'))
frames = {}
for frame in frame_data:
frames[frame['pk']] = frame['fields']
assertiondata = yaml.load_all(open(dirname + '/GMAssertion.yaml'))
assertions = {}
for assertion in assertiondata:
obj = assertion['fields']
frame = frames[obj['frame']]
frametext = frame['text']
userinfo = users[obj['author']]
username = userinfo['fields']['username']
# GlobalMind provides information about what country the user is from, which
# we can preserve in the contributor URI.
#
# If I got to re-choose these URIs, I would distinguish usernames with
# a country code from those without a country code by something more
# than the number of slashes, and I would write the country code in
# capital letters.
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale,
username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [user_source, "/s/activity/globalmind/assert"]
lang = LANG_CODES[obj['lcode']]
start = normalized_concept_uri(lang, obj['node1'])
end = normalized_concept_uri(lang, obj['node2'])
rel = '/r/' + RELATION_MAP.get(frame['relation'], frame['relation'])
# fix messy english "around in"
if ' around ' in frametext:
if obj['node2'].startswith('in '):
frametext = frametext.replace(' around ', ' in ')
obj['node2'] = obj['node2'][3:]
else:
frametext = frametext.replace(' around ', ' near ')
rel = '/r/LocatedNear'
# fix more awkward English. I wonder how bad the other languages are.
frametext = frametext.replace('hits your head', 'comes to mind')
frametext = frametext.replace(': [node1], [node2]',
' [node1] and [node2]')
node1 = u'[[' + obj['node1'] + u']]'
node2 = u'[[' + obj['node2'] + u']]'
surfaceText = frametext.replace('//', '').replace('[node1]',
node1).replace(
'[node2]', node2)
edge = make_edge(rel,
start,
end,
dataset='/d/globalmind',
license='/l/CC/By',
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
assertions[assertion['pk']] = edge
translationdata = yaml.load_all(open(dirname + '/GMTranslation.yaml'))
for translation in translationdata:
obj = translation['fields']
assertion1 = assertions[obj['assertion1']]
assertion2 = assertions[obj['assertion2']]
start = assertion1['uri']
end = assertion2['uri']
rel = '/r/TranslationOf'
text1 = assertion1['surfaceText'].replace('[[', '').replace(']]', '')
text2 = assertion2['surfaceText'].replace('[[', '').replace(']]', '')
lang1 = LANG_NAMES[get_lang(assertion1)]
lang2 = LANG_NAMES[get_lang(assertion2)]
surfaceText = u"[[%s]] in %s means [[%s]] in %s." % (text1, lang1,
text2, lang2)
userinfo = users[obj['author']]
username = userinfo['fields']['username']
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale,
username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [user_source, "/s/activity/globalmind/translate"]
edge = make_edge(rel,
start,
end,
dataset='/d/globalmind',
license=Licenses.cc_attribution,
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
def _load_from_file_path(self, path):
with open(path) as source_file:
for document in yaml.load_all(source_file):
self._add_indicator_from_dict(document)
test.columns[0]: test[test.columns[0]],
test.columns[1]: test[test.columns[1]],
ss.columns[2]: preds
})
preds.to_csv('nnSolSubmit3.csv', index=False)
import json
with open("kerasModelJson.txt") as json_file:
json_data = json.load(json_file)
import yaml
stream = open("kerasModel.yml", "r")
docs = yaml.load_all(stream)
import pandas as pd
import numpy as np
import xgboost as xgb
import copy
import array
from sknn.mlp import Classifier, Layer
import cPickle as pickle
import numpy as np
import scipy.sparse
from scipy.sparse import hstack
import cPickle as pickle
import numpy as np
import scipy.sparse
import pandas as pd
def main(argv):
global verbose
opts = docopt.docopt(__doc__, argv)
verbose = opts['-v']
genomeDir = '.'
if opts['-g']:
genomeDir = opts['-g']
sf = SequenceFactory(genomeDir)
if opts['-P']:
if opts['-t']:
prepareBedFileGeneTx(opts['<gene-list>'], opts['<refgene>'],
opts['<bedfile>'])
else:
prepareBedFileGene(opts['<gene-list>'], opts['<refgene>'],
opts['<bedfile>'])
return
if opts['-X']:
with openFile(opts['<index>'], 'w') as out:
yaml.safe_dump_all(indexBedFiles(opts['<must-have>'], sf),
out,
default_flow_style=False)
return
K = int(opts['-k'])
minGeneReads = int(opts['-M'])
minExonReads = int(opts['-m'])
minGeneRate = float(opts['-R'])
minExonRate = float(opts['-r'])
(minGeneCount, maxGeneCount) = map(int, opts['-Z'].split(':'))
(minExonCount, maxExonCount) = map(int, opts['-z'].split(':'))
with openFile(opts['<index>']) as f:
ref = list(yaml.load_all(f, Loader=yaml.BaseLoader))
if True:
# Test the double-layer index
idx = ExonIndex(K, ref)
acc = {}
toc = {}
rn = 0
for itm in reads(opts['<input>'],
K=K,
paired=True,
reads=True,
kmers=False,
both=True,
verbose=verbose):
rn += 1
(lhsFwd, lhsRev) = kmersLists(K, itm.reads[0][1])
(rhsFwd, rhsRev) = kmersLists(K, itm.reads[1][1])
xs0 = lhsFwd + rhsRev
rh0 = idx.readHash(xs0)
if rh0 is not None:
(h0, ys0) = rh0
if h0 not in acc:
acc[h0] = []
toc[h0] = ys0
acc[h0].append((compressRead(itm.reads[0][1]),
compressRead(itm.reads[1][1])))
xs1 = lhsRev + rhsFwd
rh1 = idx.readHash(xs1)
if rh1 is not None:
(h1, ys1) = rh1
if h1 not in acc:
acc[h1] = []
toc[h1] = ys1
acc[h1].append((compressRead(itm.reads[0][1]),
compressRead(itm.reads[1][1])))
nx = 0
for h in sorted(acc.keys()):
for (x, c) in sorted(acc[h].items()):
nx += 1
if c <= 1:
continue
print '%016x\t%s\t%d' % (h, render(K, x), c)
print >> sys.stderr, 'nx =', nx
return
if False:
# Position index
idx = {}
for i in range(len(ref)):
itm = ref[i]
for (x, p) in kmersWithPosList(K, itm['seq'], False):
p -= 1
if x not in idx:
idx[x] = []
idx[x].append((i, p))
if True:
# Exon tuple index
idx = {}
lens = [0 for i in range(len(ref))]
for i in range(len(ref)):
itm = ref[i]
for (x, p) in kmersWithPosList(K, itm['seq'], False):
if x not in idx:
idx[x] = set([])
idx[x].add(i)
lens[i] += 1
for x in idx.iterkeys():
idx[x] = tuple(sorted(idx[x]))
if opts['-T']:
ak = {}
for x in sorted(idx.iterkeys()):
if len(idx[x]) == 1:
continue
xStr = render(K, x)
ak[xStr] = []
for i in idx[x]:
itm = ref[i]
k = '%s/%s' % (itm['gene'], itm['exon'])
ak[xStr].append(k)
ak[xStr].sort()
rep = {}
rep['aliasing-within'] = ak
chrs = set([])
for i in range(len(ref)):
itm = ref[i]
chrs.add(itm['chr'])
counts = [0 for i in range(len(ref))]
for ch in sorted(chrs):
if verbose:
print >> sys.stderr, 'processing %s' % (ch, )
seq = sf[ch]
for (x, p) in kmersWithPos(K, seq, True):
if x not in idx:
continue
for i in idx[x]:
counts[i] += 1
gk = {}
for i in range(len(ref)):
if lens[i] == counts[i]:
continue
itm = ref[i]
k = '%s/%s' % (itm['gene'], itm['exon'])
gk[k] = {'indexed': lens[i], 'genomic': counts[i]}
rep['aliasing-genomic'] = gk
yaml.safe_dump(rep, sys.stdout, default_flow_style=False)
return
acc = {}
rn = 0
hitStats = Summary()
hitHist = [0 for i in range(1000)]
for itm in reads(opts['<input>'],
K=K,
paired=True,
reads=True,
kmers=False,
both=True,
verbose=verbose):
rn += 1
(lhsFwd, lhsRev) = kmersWithPosLists(K, itm.reads[0][1])
(rhsFwd, rhsRev) = kmersWithPosLists(K, itm.reads[1][1])
(hits0, hitCount0) = recHits(idx, lhsFwd + rhsRev)
(hits1, hitCount1) = recHits(idx, lhsRev + rhsFwd)
if len(hits0) > 0:
k = tuple(sorted(hits0.keys()))
v = sum(hits0.values())
if k not in acc:
acc[k] = [0, 0]
acc[k][0] += 1
acc[k][1] += v
hitStats.add(hitCount0)
hitHist[hitCount0] += 1
if len(hits1) > 0:
k = tuple(sorted(hits1.keys()))
v = sum(hits1.values())
if k not in acc:
acc[k] = [0, 0]
acc[k][0] += 1
acc[k][1] += v
hitStats.add(hitCount1)
hitHist[hitCount1] += 1
if verbose:
print >> sys.stderr, 'total read hits: %d' % (len(hitStats), )
print >> sys.stderr, 'total hits per read: %g (%g)' % (hitStats.mean(),
hitStats.sd())
print >> sys.stderr, 'total reads: %d' % (rn, )
for i in range(len(hitHist)):
if hitHist[i] > 0:
print >> sys.stderr, '\t%d\t%d' % (i, hitHist[i])
def gex(s):
r = []
for n in s:
itm = ref[n]
r.append('%s/%s' % (itm['gene'], itm['exon']))
return '|'.join(r)
def fmtKey(k):
nex = len(k)
gx = set([])
kStrParts = []
for s in k:
kStrParts.append(gex(s))
gx |= set([ref[i]['gene'] for i in s])
kStr = '--'.join(sorted(kStrParts))
return (nex, gx, kStr)
gxCounts = {}
for k in acc.keys():
gx = set([])
ex = set([])
for s in k:
gx |= set([ref[i]['gene'] for i in s])
ex |= set(s)
gx = tuple(sorted(gx))
if gx not in gxCounts:
gxCounts[gx] = [0, 0]
gxCounts[gx][0] += acc[k][0]
gxCounts[gx][1] += acc[k][1]
hdr = ['numReads', 'numKmers', 'kmersPerRead']
hdr += ['ggNumReads', 'ggNumKmers', 'ggKmersPerRead']
hdr += ['numExons', 'numGenes', 'geneGroup', 'exonGroup']
print '\t'.join(hdr)
for k in acc.keys():
(nex, gx, kStr) = fmtKey(k)
gx = tuple(sorted(gx))
if len(gx) < minGeneCount or len(gx) > maxGeneCount:
continue
if len(ex) < minExonCount or len(ex) > maxExonCount:
continue
if gxCounts[gx][0] < minGeneReads:
continue
if acc[k][0] < minExonReads:
continue
gxRate = float(gxCounts[gx][1]) / float(gxCounts[gx][0])
if gxRate < minGeneRate:
continue
exRate = float(acc[k][1]) / float(acc[k][0])
if exRate < minExonRate:
continue
gxStr = ':'.join(gx)
print '%d\t%d\t%g\t%d\t%d\t%g\t%d\t%d\t%s\t%s' % (
acc[k][0], acc[k][1], exRate, gxCounts[gx][0], gxCounts[gx][1],
gxRate, nex, len(gx), gxStr, kStr)
csv['spec']['install']['spec']['clusterPermissions'] = []
# Add operator role to the CSV:
with open('deploy/role.yaml', 'r') as stream:
operator_role = yaml.load(stream)
csv['spec']['install']['spec']['clusterPermissions'].append(
{
'rules': operator_role['rules'],
'serviceAccountName': operator_name,
})
# Add our deployment spec for the operator:
with open('deploy/operator.yaml', 'r') as stream:
operator_components = []
operator = yaml.load_all(stream)
for doc in operator:
operator_components.append(doc)
# There is only one yaml document in the operator deployment
operator_deployment = operator_components[0]
csv['spec']['install']['spec']['deployments'][0]['spec'] = operator_deployment['spec']
# Update the deployment to use the defined image:
csv['spec']['install']['spec']['deployments'][0]['spec']['template']['spec']['containers'][0]['image'] = operator_image
# Update the versions to include git hash:
csv['metadata']['name'] = "{}.v{}".format(operator_name, full_version)
csv['spec']['version'] = full_version
csv['spec']['replaces'] = "{}.v{}".format(operator_name, prev_version)
# Set the CSV createdAt annotation:
def include_slides(filename):
""" loads efficient yaml slide definitions as slides to be put in your main presentation definition"""
p = presenter
slides = yaml.load_all(file(filename))
background = 'bg'
for i, slide in enumerate(slides):
if slide == None:
continue
if 'bookmark' in slide:
p.bookmark(slide['bookmark'])
elif 'title' in slide:
p.bookmark(slide['title'])
elif 'bmark' in slide:
p.bookmark(slide['bmark'])
else:
p.bookmark(filename + ' - ' + str(i))
if 'background' in slide:
background = slide['background']
if 'bg' in slide:
background = slide['bg']
if 'pdf' in slide and 'slides' in slide:
images = pdf2ppm_cache(slide['pdf'], slide['slides'])
p.play(
load_image_slides(images,
library='pdf',
background=background,
content=slide.get('content', None)))
p.pause()
elif 'svg' in slide:
images = svg2png_cache(slide['svg'])
p.play(
load_image_slides(images,
library='svg',
background=background,
content=slide.get('content', None)))
p.pause()
elif 'image_files' in slide:
images = imagefiles_to_images(slide['image_files'])
p.play(
load_image_slides(images,
library='image_files',
background=background,
content=slide.get('content', None)))
p.pause()
elif 'slideshow' in slide:
images = imagefiles_to_images(slide['slideshow'])
slideshow_anim = images_slideshow(images,
library='image_files',
background=background,
delay=slide.get('delay', 2.0),
repeat=slide.get('repeat', 1),
fade_time=slide.get(
'fade_time', 0.5))
p.play(slideshow_anim)
p.pause()
elif 'images' in slide:
if 'library' in slide:
lib = slide['library']
else:
lib = 'default'
p.play(
load_image_slides(slide['images'],
library=lib,
background=background,
content=slide.get('content', None)))
p.pause()
elif 'rst' in slide:
images = rst2ppm_cache(i, slide.get('title', ''), slide['rst'],
slide.get('rst_style'))
p.play(
load_image_slides(images,
library='pdf',
background=background,
content=slide.get('content', None)))
p.pause()
else:
p.play(load_slide_content(slide['content']))
p.pause()
def run_test(command):
wrapped_command = "cd %s && %s" % (root_dir, command)
pipe = subprocess.Popen(
wrapped_command, shell=True,
)
pipe.wait()
if pipe.returncode == 0:
print(green("TEST PASSED"))
else:
print(red("TEST FAILED"))
return pipe.returncode
# load the script tests from the .travis.yml file
with open(os.path.join(root_dir, '.travis.yml')) as stream:
travis_yml = yaml.load_all(stream.read())
config = travis_yml.next()
tests = config['script']
# run the tests
if isinstance(tests, (str, unicode)):
returncode = run_test(tests)
elif isinstance(tests, (list, tuple)):
returncode = 0
for test in tests:
returncode += run_test(test)
if returncode == 0:
print(green("ALL TESTS PASSED"))
else:
print(red("SOME TESTS FAILED, SEE ABOVE"))
def load_yaml(fn):
with session_scope() as s:
for data in yaml.load_all(open(fn), Loader=yaml.FullLoader):
bookyml = Book(**data)
s.add(bookyml)
def gather_yaml(path):
ships = []
extra_info = get_parts(path)
for e in os.listdir(path):
fullpath = os.path.join(path, e)
if os.path.isfile(fullpath):
if e.endswith('.yaml'):
for ship in yaml.load_all(open(fullpath, 'r')):
try:
set_default(ship, 'info', extra_info)
ship['path'] = path
# Get image width/height
im = Image.open(os.path.join(path, ship['filename']))
ship['image_size'] = im.size
# Pull out size keys into standard location
size_axis = 0
for (dim, axis) in dimension_axes.items():
if dim in ship['info']:
ship['info']['Size'] = ship['info'][dim]
ship['info']['Dimension'] = dim
size_axis = axis
break
# Initialize scale information
m = px = unit = m_per_px = None
# First, check for explicit x_per_px
for pre in unit_conversions.keys():
key = pre + '_per_px'
if key in ship:
m_per_px = ship[key]
unit = pre
if isinstance(m_per_px, str):
(m, px) = m_per_px.split('/')
break
# Otherwise, load from the size keys
if not m_per_px and 'Size' in ship['info']:
m = ship['info']['Size']
# Load px from correct image axis
if 'size_px' in ship:
px = ship['size_px']
else:
px = im.size[size_axis]
# Look for a unit as well
if ('Unit' in ship['info']) and ship['info']['Unit']:
unit = ship['info']['Unit']
# At this point we should have something
if m and px:
m_per_px = float(m) / float(px)
if unit and unit != 'm':
print("Converting {} to m".format(unit))
m_per_px = m_per_px * unit_conversions[unit]
# Assign to ship
if m_per_px:
ship['m_per_px'] = m_per_px
ship['real_size'] = [
d * ship['m_per_px']
for d in ship['image_size']
]
if 'Size' not in ship['info']:
ship['info']['Size'] = round_to_n(
ship['real_size'][0], 3)
ship['info']['Dimension'] = 'Size'
ship['info']['Size Notes'] = 'approximately'
else:
# If we don't have things by now throw a fit
raise ArithmeticError("Couldn't determine m/px!")
ships.append(ship)
except ValueError as e:
print("Failed to load ship from {}:\n{}\n{}".format(
fullpath, str(e), str(ship)))
except IOError as e:
print("Failed to load image from {}:\n{}\n{}".format(
fullpath, str(e), str(ship)))
except ArithmeticError as e:
print("Failed to load ship from {}:\n{}\n{}".format(
fullpath, str(e), str(ship)))
elif os.path.isdir(fullpath):
ships += gather_yaml(fullpath)
return ships
def evaluate_models():
stream = open("evaluate_models.yml", 'r')
docs = yaml.load_all(stream)
for doc in docs:
true_pos_reward = doc['true_pos_reward']
true_neg_reward = doc['true_neg_reward']
false_pos_penalty = doc['false_pos_penalty']
false_neg_penalty = doc['false_neg_penalty']
pic_status = doc['pics']
k = doc['k']
if os.path.isfile('./model_scores.csv'):
model_scores_df = pd.read_csv('./model_scores.csv')
for col in model_scores_df.columns:
if 'Unnamed' in col:
model_scores_df = model_scores_df.drop(col, 1)
else:
model_scores_df = pd.DataFrame(columns=[
'model', 'score', 'recall_at_' + str(k), 'precision_at_' + str(k)
])
filenames = []
scores = []
p_at_k = []
r_at_k = []
cost_mtx = {
'[1, 1]': true_pos_reward,
'[0, 0]': true_neg_reward,
'[1, 0]': false_pos_penalty,
'[0, 1]': false_neg_penalty
}
for fn in os.listdir('./model_output/'):
if (os.path.isfile('./model_output/tracker.s') == False) or (
os.path.isfile('./model_output/tracker.s') == True
and os.path.getmtime('./model_output/' + fn) >
os.path.getmtime('./model_output/tracker.s')):
print fn
filenames.append(fn)
result_df_tup = pickle.load(open('./model_output/' + fn, "rb"))
result_df = result_df_tup[0]
top_3_feats = result_df_tup[1]
temp_df = pd.DataFrame(columns=[
'model', 'timestamp', 'score', 'recall_at_' +
str(k), 'precision_at_' + str(k), 'top_3_feats'
])
temp_df.model = [fn]
pr = pr_at_k(k, result_df)
s = score_at_k(k, result_df, cost_mtx)
temp_df.score = [s]
temp_df['precision_at_' + str(k)] = [pr[0]]
temp_df['recall_at_' + str(k)] = [pr[1]]
temp_df['timestamp'] = [datetime.datetime.now()]
temp_df['top_3_feats'] = [top_3_feats]
model_scores_df = model_scores_df.append(temp_df,
ignore_index=True)
model_scores_df.to_csv('model_scores.csv', index=False)
if (pic_status):
plot_pr_curve(result_df, fn[:-2])
if os.path.isfile('./model_output/tracker.s') == False:
open('./model_output/tracker.s', 'w+')
else:
os.utime('./model_output/tracker.s', None)
def load_names_aliases(self):
self.name_aliases = [doc for doc in load_all(self.geo_coding_file)]
def rook_operator(ctx, config):
cluster_name = config['cluster']
rook_branch = config.get('rook_branch', 'master')
rook_git_url = config.get('rook_git_url', 'https://github.com/rook/rook')
log.info(f'Cloning {rook_git_url} branch {rook_branch}')
ctx.rook[cluster_name].remote.run(args=[
'rm',
'-rf',
'rook',
run.Raw('&&'),
'git',
'clone',
'--single-branch',
'--branch',
rook_branch,
rook_git_url,
'rook',
])
# operator.yaml
operator_yaml = ctx.rook[cluster_name].remote.read_file(
'rook/cluster/examples/kubernetes/ceph/operator.yaml')
rook_image = config.get('rook_image')
if rook_image:
log.info(f'Patching operator to use image {rook_image}')
crs = list(yaml.load_all(operator_yaml, Loader=yaml.FullLoader))
assert len(crs) == 2
crs[1]['spec']['template']['spec']['containers'][0][
'image'] = rook_image
operator_yaml = yaml.dump_all(crs)
ctx.rook[cluster_name].remote.write_file('operator.yaml', operator_yaml)
op_job = None
try:
log.info('Deploying operator')
_kubectl(ctx, config, [
'create',
'-f',
'rook/cluster/examples/kubernetes/ceph/crds.yaml',
'-f',
'rook/cluster/examples/kubernetes/ceph/common.yaml',
'-f',
'operator.yaml',
])
# on centos:
if teuthology.get_distro(ctx) == 'centos':
_kubectl(ctx, config, [
'-n', 'rook-ceph', 'set', 'env', 'deploy/rook-ceph-operator',
'ROOK_HOSTPATH_REQUIRES_PRIVILEGED=true'
])
# wait for operator
op_name = None
with safe_while(sleep=10, tries=90,
action="wait for operator") as proceed:
while not op_name and proceed():
p = _kubectl(
ctx,
config,
[
'-n', 'rook-ceph', 'get', 'pods', '-l',
'app=rook-ceph-operator'
],
stdout=BytesIO(),
)
for line in p.stdout.getvalue().decode(
'utf-8').strip().splitlines():
name, ready, status, _ = line.split(None, 3)
if status == 'Running':
op_name = name
break
# log operator output
op_job = _kubectl(
ctx,
config,
['-n', 'rook-ceph', 'logs', '-f', op_name],
wait=False,
logger=log.getChild('operator'),
)
yield
except Exception as e:
log.exception(e)
raise
finally:
log.info('Cleaning up rook operator')
_kubectl(ctx, config, [
'delete',
'-f',
'operator.yaml',
])
if False:
# don't bother since we'll tear down k8s anyway (and this mysteriously
# fails sometimes when deleting some of the CRDs... not sure why!)
_kubectl(ctx, config, [
'delete',
'-f',
'rook/cluster/examples/kubernetes/ceph/common.yaml',
])
_kubectl(ctx, config, [
'delete',
'-f',
'rook/cluster/examples/kubernetes/ceph/crds.yaml',
])
ctx.rook[cluster_name].remote.run(
args=['rm', '-rf', 'rook', 'operator.yaml'])
if op_job:
op_job.wait()
run.wait(ctx.cluster.run(args=['sudo', 'rm', '-rf', '/var/lib/rook']))
'imageadd', 'imagedel', 'spellcheck', 'experiment', 'balance', 'code_imp',
'refactor', 'config', 'admin', 'server'
]
def dictToTuples(inp):
return [(k, v) for k, v in inp.items()]
changelog_cache = os.path.join(args.ymlDir, '.all_changelog.yml')
failed_cache_read = True
if os.path.isfile(changelog_cache):
try:
with open(changelog_cache, encoding='utf-8') as f:
(_, all_changelog_entries) = yaml.load_all(f,
Loader=yaml.SafeLoader)
failed_cache_read = False
# Convert old timestamps to newer format.
new_entries = {}
for _date in all_changelog_entries.keys():
ty = type(_date).__name__
# print(ty)
if ty in ['str', 'unicode']:
temp_data = all_changelog_entries[_date]
_date = datetime.strptime(_date, dateformat).date()
new_entries[_date] = temp_data
else:
new_entries[_date] = all_changelog_entries[_date]
all_changelog_entries = new_entries
except Exception as e:
def layers():
path = os.path.join(path_yaml_examples, 'layers_metadata.yml')
stream = open(path, 'r')
yield yaml.load_all(stream)
stream.close()
def index_coro(dirpath_lwc_root, indices=None): # pylint: disable=R0912
"""
Yield mappings corresponding to the sent sequence of multi-key tuples.
"""
if indices is not None:
(line_index, references_index, objects_index) = indices
else:
line_index = None
references_index = None
objects_index = None
line_idx_builder = da.util.index_builder_coro(line_index)
references_idx_builder = da.util.index_builder_coro(references_index)
objects_idx_builder = da.util.index_builder_coro(objects_index)
matcher = _id_matcher_coro(dirpath_lwc_root)
while True:
indices = (line_index, references_index, objects_index)
build_unit = (yield indices)
file = build_unit['file']
relpath = build_unit['relpath']
# The first pass over the file is for
# relatively unsophisticated indexing
# - not accounting for any formatting
# of the file other than the presence
# of newline delimiters.
#
file.seek(0)
for iline, binary_line in enumerate(file):
text_line = binary_line.decode('utf-8')
for (match_class, idstr, line_offset,
col_offset) in matcher.send(text_line):
line_num = 1 + iline + line_offset
col_num = 1 + col_offset
pos = (line_num, col_num)
line_index = line_idx_builder.send(
(match_class, idstr, relpath, pos))
# The second pass over the file takes
# account of the file format - We try
# to parse the file to identify the
# context within which each identifier
# is placed.
#
file.seek(0, os.SEEK_SET)
# YAML files are grist to the mill --
# any part of a YAML data structure
# is potentially of interest to us.
#
if relpath.endswith('yaml'):
for data in yaml.load_all(file, Loader=da.util.marked_yaml.Loader):
(maybe_ref_idx,
maybe_obj_idx) = _index_yaml(references_idx_builder,
objects_idx_builder, matcher,
relpath, data)
if maybe_ref_idx is not None:
references_index = maybe_ref_idx
if maybe_obj_idx is not None:
objects_index = maybe_obj_idx
# We are only really interested in YAML or
# JSON structures that are embedded in
# comments or docstrings within the Python
# source document.
#
# ---
# i00022_store_requirements_in_python_source_documents:
# - "The system SHALL extract and process requirement clauses stored
# as comments in Python source documents."
# - notes: "We want to encourage the use of requirements to document
# both top-down and bottom-up aspects of the design process.
# Allowing requirement clauses to be stored as comments in
# source files makes it much easier for developers to create
# bottom-up requirements. Additionally, by placing the
# requirements next to the site of the implementation makes
# it leess likely that requirements will be forgotten or
# ignored."
# - type: mandate
# - state: draft
# ...
if relpath.endswith('py'):
pass
def main():
argparser = set_argparser()
cmdargs = argparser.parse_args()
scm = SigmaConfigurationManager()
logger = logging.getLogger(__name__)
if cmdargs.debug: # pragma: no cover
logger.setLevel(logging.DEBUG)
if cmdargs.lists:
print("Backends (Targets):")
list_backends(cmdargs.debug)
print()
print("Configurations (Sources):")
list_configurations(backend=cmdargs.target, scm=scm)
print()
print("Modifiers:")
list_modifiers(modifiers=modifiers)
sys.exit(0)
elif len(cmdargs.inputs) == 0:
print("Nothing to do!")
argparser.print_usage()
sys.exit(0)
if cmdargs.target is None:
print("No target selected, select one with -t/--target")
argparser.print_usage()
sys.exit(ERR_NO_TARGET)
rulefilter = None
if cmdargs.filter:
try:
rulefilter = SigmaRuleFilter(cmdargs.filter)
except SigmaRuleFilterParseException as e:
print("Parse error in Sigma rule filter expression: %s" % str(e),
file=sys.stderr)
sys.exit(ERR_RULE_FILTER_PARSING)
sigmaconfigs = SigmaConfigurationChain()
backend_class = backends.getBackend(cmdargs.target)
if cmdargs.config is None:
if backend_class.config_required and not cmdargs.shoot_yourself_in_the_foot:
print(
"The backend you want to use usually requires a configuration to generate valid results. Please provide one with --config/-c.",
file=sys.stderr)
print(
"Available choices for this backend (get complete list with --lists/-l):"
)
list_configurations(backend=cmdargs.target, scm=scm)
sys.exit(ERR_CONFIG_REQUIRED)
if backend_class.default_config is not None:
cmdargs.config = backend_class.default_config
if cmdargs.config:
order = 0
for conf_name in cmdargs.config:
try:
sigmaconfig = scm.get(conf_name)
if sigmaconfig.order is not None:
if sigmaconfig.order <= order and not cmdargs.shoot_yourself_in_the_foot:
print(
"The configurations were provided in the wrong order (order key check in config file)",
file=sys.stderr)
sys.exit(ERR_CONFIG_ORDER)
order = sigmaconfig.order
try:
if cmdargs.target not in sigmaconfig.config["backends"]:
print(
"The configuration '{}' is not valid for backend '{}'. Valid choices are: {}"
.format(conf_name, cmdargs.target,
", ".join(sigmaconfig.config["backends"])),
file=sys.stderr)
sys.exit(ERR_CONFIG_ORDER)
except KeyError:
pass
sigmaconfigs.append(sigmaconfig)
except OSError as e:
print("Failed to open Sigma configuration file %s: %s" %
(conf_name, str(e)),
file=sys.stderr)
exit(ERR_OPEN_CONFIG_FILE)
except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e:
print("Sigma configuration file %s is no valid YAML: %s" %
(conf_name, str(e)),
file=sys.stderr)
exit(ERR_CONFIG_INVALID_YAML)
except SigmaConfigParseError as e:
print("Sigma configuration parse error in %s: %s" %
(conf_name, str(e)),
file=sys.stderr)
exit(ERR_CONFIG_PARSING)
if cmdargs.output_fields:
if cmdargs.output_format:
output_fields_rejected = [
field for field in cmdargs.output_fields.split(",")
if field not in allowed_fields
] # Not allowed fields
if output_fields_rejected:
print(
"These fields are not allowed (check help for allow field list) : %s"
% (", ".join(output_fields_rejected)),
file=sys.stderr)
exit(ERR_OUTPUT_FORMAT)
else:
output_fields_filtered = [
field for field in cmdargs.output_fields.split(",")
if field in allowed_fields
] # Keep only allowed fields
else:
print(
"The '--output-fields' or '-of' arguments must be used with '--output-format' or '-oF' equal to 'json' or 'yaml'",
file=sys.stderr)
exit(ERR_OUTPUT_FORMAT)
backend_options = BackendOptions(cmdargs.backend_option,
cmdargs.backend_config)
backend = backend_class(sigmaconfigs, backend_options)
filename_ext = cmdargs.output_extention
filename = cmdargs.output
fileprefix = None
if filename:
if filename_ext:
if filename_ext[0] == '.':
pass
else:
filename_ext = '.' + filename_ext
else:
filename_ext = '.rule'
if filename[-1:] in ['_', '/', '\\']:
fileprefix = filename
else:
try:
out = open(filename, "w", encoding='utf-8')
except (IOError, OSError) as e:
print("Failed to open output file '%s': %s" %
(filename, str(e)),
file=sys.stderr)
exit(ERR_OUTPUT)
else:
out = sys.stdout
error = 0
output_array = []
for sigmafile in get_inputs(cmdargs.inputs, cmdargs.recurse):
logger.debug("* Processing Sigma input %s" % (sigmafile))
try:
if cmdargs.inputs == ['-']:
f = sigmafile
else:
f = sigmafile.open(encoding='utf-8')
parser = SigmaCollectionParser(f, sigmaconfigs, rulefilter,
sigmafile)
results = parser.generate(backend)
nb_result = len(list(copy.deepcopy(results)))
inc_filenane = None if nb_result < 2 else 0
newline_separator = '\0' if cmdargs.print0 else '\n'
results = list(
results
) # Since results is an iterator and used twice we convert it a list
for result in results:
if not fileprefix == None and not inc_filenane == None: #yml action
try:
filename = fileprefix + str(sigmafile.name)
filename = filename.replace(
'.yml', '_' + str(inc_filenane) + filename_ext)
inc_filenane += 1
out = open(filename, "w", encoding='utf-8')
except (IOError, OSError) as e:
print("Failed to open output file '%s': %s" %
(filename, str(e)),
file=sys.stderr)
exit(ERR_OUTPUT)
elif not fileprefix == None and inc_filenane == None: # a simple yml
try:
filename = fileprefix + str(sigmafile.name)
filename = filename.replace('.yml', filename_ext)
out = open(filename, "w", encoding='utf-8')
except (IOError, OSError) as e:
print("Failed to open output file '%s': %s" %
(filename, str(e)),
file=sys.stderr)
exit(ERR_OUTPUT)
if not cmdargs.output_fields:
print(result, file=out, end=newline_separator)
if cmdargs.output_fields: # Handle output fields
output = {}
f.seek(0)
docs = yaml.load_all(f, Loader=yaml.FullLoader)
for doc in docs:
for k, v in doc.items():
if k in output_fields_filtered:
output[k] = v
output['rule'] = [result for result in results]
if "filename" in output_fields_filtered:
output['filename'] = str(sigmafile.name)
output_array.append(output)
if nb_result == 0: # backend get only 1 output
if not fileprefix == None: # want a prefix anyway
try:
filename = "%s%s_mono_output%s" % (
fileprefix, cmdargs.target, filename_ext)
out = open(filename, "w", encoding='utf-8')
fileprefix = None # no need to open the same file many time
except (IOError, OSError) as e:
print("Failed to open output file '%s': %s" %
(filename, str(e)),
file=sys.stderr)
exit(ERR_OUTPUT)
except OSError as e:
print("Failed to open Sigma file %s: %s" % (sigmafile, str(e)),
file=sys.stderr)
error = ERR_OPEN_SIGMA_RULE
except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e:
print("Error: Sigma file %s is no valid YAML: %s" %
(sigmafile, str(e)),
file=sys.stderr)
error = ERR_INVALID_YAML
if not cmdargs.defer_abort:
sys.exit(error)
except (SigmaParseError, SigmaCollectionParseError) as e:
print("Error: Sigma parse error in %s: %s" % (sigmafile, str(e)),
file=sys.stderr)
error = ERR_SIGMA_PARSING
if not cmdargs.defer_abort:
sys.exit(error)
except NotSupportedError as e:
print(
"Error: The Sigma rule requires a feature that is not supported by the target system: "
+ str(e),
file=sys.stderr)
if not cmdargs.ignore_backend_errors:
error = ERR_NOT_SUPPORTED
if not cmdargs.defer_abort:
sys.exit(error)
except BackendError as e:
print("Error: Backend error in %s: %s" % (sigmafile, str(e)),
file=sys.stderr)
if not cmdargs.ignore_backend_errors:
error = ERR_BACKEND
if not cmdargs.defer_abort:
sys.exit(error)
except (NotImplementedError, TypeError) as e:
print(
"An unsupported feature is required for this Sigma rule (%s): "
% (sigmafile) + str(e),
file=sys.stderr)
if not cmdargs.ignore_backend_errors:
error = ERR_NOT_IMPLEMENTED
if not cmdargs.defer_abort:
sys.exit(error)
except PartialMatchError as e:
print("Error: Partial field match error: %s" % str(e),
file=sys.stderr)
if not cmdargs.ignore_backend_errors:
error = ERR_PARTIAL_FIELD_MATCH
if not cmdargs.defer_abort:
sys.exit(error)
except FullMatchError as e:
print("Error: Full field match error", file=sys.stderr)
if not cmdargs.ignore_backend_errors:
error = ERR_FULL_FIELD_MATCH
if not cmdargs.defer_abort:
sys.exit(error)
finally:
try:
f.close()
except:
pass
result = backend.finalize()
if result:
print(result, file=out)
if cmdargs.output_fields:
if cmdargs.output_format == 'json':
print(json.dumps(output_array, indent=4, ensure_ascii=False),
file=out)
elif cmdargs.output_format == 'yaml':
print(ruamel.yaml.round_trip_dump(output_array), file=out)
out.close()
sys.exit(error)
