def export_ecosystem(filename):
api = os.environ.get("CYBERGRX_API",
"https://api.cybergrx.com").rstrip("/")
token = os.environ.get("CYBERGRX_API_TOKEN", None)
if not token:
raise Exception(
"The environment variable CYBERGRX_API_TOKEN must be set")
uri = api + "/bulk-v1/third-parties"
print("Fetching third parties from " + uri + " this can take some time.")
response = requests.get(uri, headers={"Authorization": token.strip()})
result = json.loads(response.content.decode("utf-8"))
print("Retrieved " + str(len(result)) +
" third parties from your ecosystem, building an excel.")
wb = Workbook()
wb["Sheet"].title = THIRD_PARTY_TABLE
wb.create_sheet(GAPS_TABLE)
wb.create_sheet(CONTROL_SCORES)
wb.create_sheet(COMPANY_TAGS)
third_party_writer = sheet_writer(wb,
THIRD_PARTY_TABLE,
TP_COLUMNS,
mapping=TP_MAPPING)
findings_writer = sheet_writer(wb, GAPS_TABLE, GAPS_COLUMNS)
scores_writer = sheet_writer(wb,
CONTROL_SCORES,
SCORE_COLUMNS,
mapping=SCORE_MAPPING)
tags_writer = sheet_writer(wb, COMPANY_TAGS, TAG_COLUMNS)
for tp in tqdm(result, total=len(result), desc="Third Party"):
third_party_writer(tp)
for tag in glom(tp, Coalesce("tags", default=[])):
tags_writer({"tag": tag, "company_name": tp["name"]})
for finding in glom(tp, Coalesce("residual_risk.findings",
default=[])):
finding["company_name"] = tp["name"]
findings_writer(finding)
for score in glom(tp, Coalesce("residual_risk.scores", default=[])):
score["company_name"] = tp["name"]
scores_writer(score)
# Finalize each writer (fix width, ETC)
third_party_writer.finalizer()
findings_writer.finalizer()
scores_writer.finalizer()
tags_writer.finalizer()
wb.save("ecosystem.xlsx")
def _spec_to_type(
key: str, value: Dict[str, Dict], bases: Tuple[Type, ...] = ()) -> Type:
"""Using the type specification, create the custom type objects
Parameters
----------
key : str
The key name corresponding to the specification. It is used as a
template for the custom type name.
value : Dict
The dictionary with the type specification. It looks like:
{
"key1": {"type": <type1>, "validator": <validator1>},
"key2": {"type": <type2>, "validator": <validator2>},
# ...
}
bases : Tuple[Type, ...]
Base classes
Returns
-------
Type
Custom type object with validators
"""
type_k = _type_spec[0]
dflt_k = _type_spec[6]
fields = glom(
# NOTE: original ordering is preserved, apart from moving the data
# members w/ default arguments later.
[(k, v) for k, v in value.items() if type_k in v and dflt_k not in v] +
[(k, v) for k, v in value.items() if type_k in v and dflt_k in v],
[(
{
"k": "0",
"v": f"1.{type_k}",
# TODO: non-trivial defaults like mutable types
"d": Coalesce(f"1.{dflt_k}", default=SKIP),
},
T.values(),
tuple,
)],
) # extract key, value and convert to list of tuples
ns = dict(
chain(*glom(
value.values(),
[(Coalesce("validator", default_factory=dict), T.items())],
))) # chain dict.items() and create namespace
return make_typedconfig(f"{key}_t", fields, namespace=ns, bases=bases)
def classes_sync(master_id, environments):
cert_instance = CertsClass(master_id)
cert_path = cert_instance.get_cert()
private_key_path = cert_instance.get_key()
master_address = cert_instance.get_master_address()
url = f"https://{master_address}:8140/puppet/v3/environment_classes/"
data = []
for environment in environments:
req = requests.get(
f"{url}?environment={environment}",
verify=False,
cert=(cert_path, private_key_path),
)
target = req.json()["files"]
spec = [
Coalesce(
(
"classes",
[{
"name":
"name",
"params": (
"params",
[{
"name":
"name",
"type":
Coalesce("type", Literal("String")),
"default_source":
Coalesce("default_source", Literal("")),
}],
),
"master":
Literal(master_id),
}],
),
Literal([]),
)
]
for class_data in list(chain(*glom(target, spec))):
class_data["environment"] = environment
data.append(class_data)
user = os.environ.get("WEBAPP_USER", "")
password = os.environ.get("WEBAPP_PASS", "")
requests.post("http://webapp:8000/api/classes/sync/",
json=data,
auth=(user, password))
def get_proc_info():
ret = {}
ret['pid'] = os.getpid()
_user_t, _sys_t = os.times()[:2]
ret['cpu_times'] = {'user_time': _user_t, 'sys_time': _sys_t}
ret['cwd'] = os.getcwdu()
ret['umask'] = os.umask(os.umask(2)) # have to set to get
ret['umask_str'] = '{0:03o}'.format(ret['umask'])
ret['owner'] = glom(globals(),
Coalesce(T['getpass'].getuser(), T['os'].getuid()),
skip_exc=Exception)
# use 0 to get current niceness, seems to return process group's nice level
unix_only_vals = glom(os, {
'ppid': T.getppid(),
'pgid': T.getpgrp(),
'niceness': T.nice(0)
},
skip_exc=AttributeError)
ret.update(unix_only_vals)
ret['rusage'] = get_rusage_dict()
ret['rlimit'] = get_rlimit_dict()
return ret
def test_skip():
assert OMIT is SKIP # backwards compat
target = {
'a': {
'b': 'c'
}, # basic dictionary nesting
'd': {
'e': ['f'], # list in dictionary
'g': 'h'
},
'i': [{
'j': 'k',
'l': 'm'
}], # list of dictionaries
'n': 'o'
}
res = glom(target, {'a': 'a.b', 'z': Coalesce('x', 'y', default=SKIP)})
assert res['a'] == 'c' # sanity check
assert 'x' not in target
assert 'y' not in target
assert 'z' not in res
# test that skip works on lists
target = range(7)
res = glom(target, [lambda t: t if t % 2 else SKIP])
assert res == [1, 3, 5]
# test that skip works on chains (enable conditional applications of transforms)
target = range(7)
# double each value if it's even, but convert all values to floats
res = glom(target, [(lambda x: x * 2 if x % 2 == 0 else SKIP, float)])
assert res == [0.0, 1.0, 4.0, 3.0, 8.0, 5.0, 12.0]
def test_omit():
target = {
'a': {
'b': 'c'
}, # basic dictionary nesting
'd': {
'e': ['f'], # list in dictionary
'g': 'h'
},
'i': [{
'j': 'k',
'l': 'm'
}], # list of dictionaries
'n': 'o'
}
res = glom(target, {'a': 'a.b', 'z': Coalesce('x', 'y', default=OMIT)})
assert res['a'] == 'c' # sanity check
assert 'x' not in target
assert 'y' not in target
assert 'z' not in res
# test that it works on lists
target = range(7)
res = glom(target, [lambda t: t if t % 2 else OMIT])
assert res == [1, 3, 5]
def process(self, task):
"""Return list of jobs"""
# NOTE: use dictionary unpacking to optionally overwrite global modules
# with task specific modules
opts = {
"module": self.options["module"],
**self.options[task],
self.backend: self.options[self.backend],
}
if opts.get("inputs", None) == "ignore":
jobs = [BatchJob(task, opts, self.tmpl_dir, self.backend)]
elif opts.get("inputs", None) == "all": # e.g. panel of normal
keys = reduce(
lambda i, j: i.union(set(j)), [i.keys() for i in self.inputs], set()
)
inputs = glom(
self.inputs, dict((key, [Coalesce(key, default="")]) for key in keys)
)
for key in keys: # filter out "no files" (shows as empty string above)
inputs[key] = [i for i in filter(None, inputs[key])]
jobs = [BatchJob(task, dict(**inputs, **opts), self.tmpl_dir, self.backend)]
else:
inputs = opts.get("inputs", self.inputs) # allow overriding inputs per job
jobs = [
BatchJob(task, dict(**infile, **opts), self.tmpl_dir, self.backend)
for infile in inputs
]
return jobs
def process_third_party(tp):
third_party_writer(tp)
for outcome in glom(
tp, Coalesce("residual_risk.residual_risk_outcomes",
default=[])):
outcome["company_name"] = tp["name"]
residual_risk_writer(outcome)
def _spec_to_type(
key: str, value: Dict[str, Dict], bases: Tuple[Type, ...] = ()) -> Type:
"""Using the type specification, create the custom type objects
Parameters
----------
key : str
The key name corresponding to the specification. It is used as a
template for the custom type name.
value : Dict
The dictionary with the type specification. It looks like:
{
"key1": {"type": <type1>, "validator": <validator1>},
"key2": {"type": <type2>, "validator": <validator2>},
# ...
}
bases : Tuple[Type]
Base classes
Returns
-------
Type
Custom type object with validators
"""
fields = glom(
value.items(),
[(
{
"k": "0",
"v": "1.type",
# TODO: non-trivial defaults like mutable types
"d": Coalesce("1.default", default=SKIP),
},
T.values(),
tuple,
)],
) # extract key, value and convert to list of tuples
ns = dict(
chain(*glom(
value.values(),
[(Coalesce("validator", default_factory=dict), T.items())],
))) # chain dict.items() and create namespace
return make_dataconfig(f"{key}_t", fields, namespace=ns, bases=bases)
def test_coalesce():
val = {
'a': {
'b': 'c'
}, # basic dictionary nesting
'd': {
'e': ['f'], # list in dictionary
'g': 'h'
},
'i': [{
'j': 'k',
'l': 'm'
}], # list of dictionaries
'n': 'o'
}
assert glom(val, 'a.b') == 'c'
assert glom(val, Coalesce('xxx', 'yyy', 'a.b')) == 'c'
# check that defaulting works
spec = Coalesce('xxx', 'yyy', default='zzz')
assert glom(val, spec) == 'zzz'
assert repr(spec) == "Coalesce('xxx', 'yyy', default='zzz')"
# check that default_factory works
sentinel_list = []
factory = lambda: sentinel_list
assert glom(val, Coalesce('xxx', 'yyy',
default_factory=factory)) is sentinel_list
with pytest.raises(ValueError):
Coalesce('x', 'y', default=1, default_factory=list)
# check that arbitrary values can be skipped
assert glom(val, Coalesce('xxx', 'yyy', 'a.b', default='zzz',
skip='c')) == 'zzz'
# check that arbitrary exceptions can be ignored
assert glom(val,
Coalesce(lambda x: 1 / 0, 'a.b',
skip_exc=ZeroDivisionError)) == 'c'
target = {'a': 1, 'b': 3, 'c': 4}
spec = Coalesce('a', 'b', 'c', skip=lambda x: x % 2)
assert glom(target, spec) == 4
spec = Coalesce('a', 'b', 'c', skip=(1, ))
assert glom(target, spec) == 3
with pytest.raises(TypeError):
Coalesce(bad_kwarg=True)
def test_check_basic():
assert glom([0, SKIP], [T]) == [0] # sanity check SKIP
target = [{'id': 0}, {'id': 1}, {'id': 2}]
# check that skipping non-passing values works
assert glom(target, ([Coalesce(Check('id', equal_to=0), default=SKIP)], T[0])) == {'id': 0}
assert glom(target, ([Check('id', equal_to=0, default=SKIP)], T[0])) == {'id': 0}
# check that stopping iteration on non-passing values works
assert glom(target, [Check('id', equal_to=0, default=STOP)]) == [{'id': 0}]
# check that stopping chain execution on non-passing values works
spec = (Check(validate=lambda x: len(x) > 0, default=STOP), T[0])
assert glom('hello', spec) == 'h'
assert glom('', spec) == '' # would fail with IndexError if STOP didn't work
assert repr(Check()) == 'Check()'
assert repr(Check(T.a)) == 'Check(T.a)'
assert repr(Check(equal_to=1)) == 'Check(equal_to=1)'
target = [1, 'a']
assert glom(target, [Check(type=str, default=SKIP)]) == ['a']
assert glom(target, [Check(type=(str, int))]) == [1, 'a']
assert glom(target, [Check(instance_of=str, default=SKIP)]) == ['a']
assert glom(target, [Check(instance_of=(str, int))]) == [1, 'a']
target = ['1']
assert glom(target, [Check(validate=(int, float))])
assert glom(target, [Check()]) # bare check does a truthy check
failing_checks = [({'a': {'b': 1}}, {'a': ('a', 'b', Check(type=str))},
'''target at path ['a', 'b'] failed check, got error: "expected type to be 'str', found type 'int'"'''),
({'a': {'b': 1}}, {'a': ('a', Check('b', type=str))},
'''target at path ['a'] failed check, subtarget at 'b' got error: "expected type to be 'str', found type 'int'"'''),
(1, Check(type=(str, bool))),
(1, Check(instance_of=str)),
(1, Check(instance_of=(str, bool))),
(1, Check(equal_to=0)),
(1, Check(one_of=(0,))),
(1, Check(one_of=(0, 2))),
('-3.14', Check(validate=int)),
('', Check(validate=lambda x: False)),]
for fc in failing_checks:
if len(fc) == 2:
target, check = fc
msg = None
else:
target, check, msg = fc
with raises(CheckError) as exc_info:
glom(target, check)
if msg is not None:
assert str(exc_info.value) == msg
assert repr(exc_info.value)
def handle(self, *args, person_email, **options):
try:
p = Person.objects.get_by_natural_key(person_email)
except Person.DoesNotExist:
raise CommandError("L'email donné est inconnu.")
spec_role = get_all_fields(Role)
spec_event = {
"Nom": "name",
"URL": ("id", lambda id: front_url("view_event", args=[id])),
}
spec_membership = {
"Nom":
"supportgroup.name",
"URL":
("supportgroup.id", lambda id: front_url("view_group", args=[id])),
"Animateur":
"is_referent",
"Gestionnaire":
"is_manager",
}
spec_payment = get_all_fields(Payment)
spec_subscription = get_all_fields(Subscription)
spec_event_images = get_all_fields(EventImage)
spec_form_submissions = get_all_fields(PersonFormSubmission)
spec_tags = get_all_fields(PersonTag)
spec_person = {
**get_all_fields(Person),
"pays": ("location_country", str),
"Rôle": ("role", spec_role),
"événements organisés":
("organized_events", T.all(), [spec_event]),
"participations aux événements": (
"rsvps",
T.all(),
[("event", spec_event)],
),
"participations à des groupes":
("memberships", T.all(), [spec_membership]),
"paiements": ("payments", T.all(), [spec_payment]),
"souscription au don mensuel":
Coalesce(("subscription", spec_subscription), default=None),
"images d'événements":
("event_images", T.all(), [spec_event_images]),
"réponses à des formulaires": (
"form_submissions",
T.all(),
[spec_form_submissions],
),
"libellés": ("tags", T.all(), [spec_tags]),
}
self.stdout.ending = ""
json.dump(glom(p, spec_person), self.stdout, cls=DjangoJSONEncoder)
def test_check_ported_tests():
"""
Tests ported from Check() to make sure all the functionality has an analogue.
"""
target = [{'id': 0}, {'id': 1}, {'id': 2}]
# check that skipping non-passing values works
assert glom(target, [Coalesce(M(T['id']) == 0, default=SKIP)]) == [{'id': 0}]
# TODO: should M(subspec, default='') work? I lean no.
# NB: this is not a very idiomatic use of Match, just brought over for Check reasons
assert glom(target, [Match({'id': And(int, M == 1)}, default=SKIP)]) == [{'id': 1}]
assert glom(target, [Match({'id': And(int, M <= 1)}, default=STOP)]) == [{'id': 0}, {'id': 1}]
# check that stopping chain execution on non-passing values works
spec = (Or(Match(len), Val(STOP)), T[0])
assert glom('hello', spec, glom_debug=True) == 'h'
assert glom('', spec) == '' # would fail with IndexError if STOP didn't work
target = [1, u'a']
assert glom(target, [Match(unicode, default=SKIP)]) == ['a']
assert glom(target, Match([Or(unicode, int)])) == [1, 'a']
target = ['1']
assert glom(target, [(M(T), int)]) == [1]
assert glom(target, M(T)) == ['1']
failing_checks = [({'a': {'b': 1}}, {'a': ('a', 'b', Match(str))},
'''expected type str, not int'''), # TODO: bbrepr at least, maybe include path like Check did
({'a': {'b': 1}}, {'a': ('a', Match({'b': str}))},
'''expected type str, not int'''), # TODO: include subspec path ('b')
(1, Match(Or(unicode, bool))),
(1, Match(unicode)),
(1, Match(0)),
(1, Match(Or(0, 2))),
('-3.14', Match(lambda x: int(x) > 0)),
# ('-3.14', M(lambda x: int(x) > 0)),
# TODO: M doesn't behave quite like Match because it's mode-free
]
for fc in failing_checks:
if len(fc) == 2:
target, check = fc
msg = None
else:
target, check, msg = fc
with pytest.raises(MatchError) as exc_info:
glom(target, check)
if msg is not None:
actual_msg = str(exc_info.value)
assert actual_msg.find(msg) != -1
assert repr(exc_info.value)
return
def _parse_api_response(resp_json):
"""Parse the json returned by the alerts API and extract relevant bits."""
service_subspec = {
'ServiceType': 'ServiceType',
'ServiceTypeDescription': 'ServiceTypeDescription',
'ServiceName': 'ServiceName',
'ServiceId': 'ServiceId'
}
spec = (
'CTAAlerts.Alert',
[
# for each entry in CTAAlerts.Alert pull out the following info
{
'AlertId': ('AlertId', int),
'ShortDescription':
'ShortDescription',
'FullDescription':
'FullDescription.#cdata-section',
'Impact':
'Impact',
'SeverityScore': ('SeverityScore', int),
'LastSeen':
Literal(pytz.UTC.localize(dt.datetime.utcnow()).isoformat()),
'EventStart':
Coalesce(('EventStart', _chicago_to_utc), default=None),
'EventEnd':
Coalesce(('EventEnd', _chicago_to_utc), default=None),
# pull out each entry in ImpactedService if there are multiple
'ImpactedService': (
'ImpactedService.Service',
# some chicanery to make sure this entry is always a
# list even though the API just returns a nested dict
# when there's only one service
lambda s: glom(s, [service_subspec])
if isinstance(s, list) else [glom(s, service_subspec)])
}
])
return glom(resp_json, spec)
def test_initial_integration():
class Example(object):
pass
example = Example()
subexample = Example()
subexample.name = 'good_name'
example.mapping = {'key': subexample}
val = {
'a': {
'b': 'c'
}, # basic dictionary nesting
'example': example, # basic object
'd': {
'e': ['f'], # list in dictionary
'g': 'h'
},
'i': [{
'j': 'k',
'l': 'm'
}], # list of dictionaries
'n': 'o'
}
spec = {
'a': (Inspect(recursive=True), 'a', 'b'), # inspect just prints here
'name': 'example.mapping.key.name', # test object access
'e': 'd.e', # d.e[0] or d.e: (callable to fetch 0)
'i': (
'i', [{
'j': 'j'
}]
), # TODO: support True for cases when the value should simply be mapped into the field name?
'n': ('n', lambda n: n.upper()),
'p': Coalesce('xxx', 'yyy', default='zzz')
}
ret = glom(val, spec)
print('in: ', val)
print('got:', ret)
expected = {
'a': 'c',
'name': 'good_name',
'e': ['f'],
'i': [{
'j': 'k'
}],
'n': 'O',
'p': 'zzz'
}
print('exp:', expected)
assert ret == expected
def serialize_notifications(notifications):
# All fields are either
spec = [{
"id":
"id",
"status":
"status",
"content":
Coalesce(T.html_content(),
T.announcement.html_content(),
skip="",
default=""),
"icon":
Coalesce("icon", "announcement.icon", skip="", default=""),
"link":
Coalesce("link", "announcement.link", skip="", default=""),
"created": (Coalesce("announcement.start_date",
"created"), T.isoformat()),
}]
return glom(notifications, spec)
def test_coalesce():
val = {
'a': {
'b': 'c'
}, # basic dictionary nesting
'd': {
'e': ['f'], # list in dictionary
'g': 'h'
},
'i': [{
'j': 'k',
'l': 'm'
}], # list of dictionaries
'n': 'o'
}
assert glom(val, 'a.b') == 'c'
assert glom(val, Coalesce('xxx', 'yyy', 'a.b')) == 'c'
try:
glom(val, Coalesce('xxx', 'yyy'))
except CoalesceError as ce:
msg = str(ce)
assert "'xxx'" in msg
assert "'yyy'" in msg
assert msg.count('PathAccessError') == 2
else:
assert False, 'expected a CoalesceError'
# check that defaulting works
assert glom(val, Coalesce('xxx', 'yyy', default='zzz')) == 'zzz'
# check that arbitrary values can be skipped
assert glom(val, Coalesce('xxx', 'yyy', 'a.b', default='zzz',
skip='c')) == 'zzz'
# check that arbitrary exceptions can be ignored
assert glom(val,
Coalesce(lambda x: 1 / 0, 'a.b',
skip_exc=ZeroDivisionError)) == 'c'
def test_coalesce():
val = {
'a': {
'b': 'c'
}, # basic dictionary nesting
'd': {
'e': ['f'], # list in dictionary
'g': 'h'
},
'i': [{
'j': 'k',
'l': 'm'
}], # list of dictionaries
'n': 'o'
}
assert glom(val, 'a.b') == 'c'
assert glom(val, Coalesce('xxx', 'yyy', 'a.b')) == 'c'
with pytest.raises(CoalesceError) as exc_info:
glom(val, Coalesce('xxx', 'yyy'))
msg = exc_info.exconly()
assert "'xxx'" in msg
assert "'yyy'" in msg
assert msg.count('PathAccessError') == 2
expected = "[PathAccessError(KeyError('xxx',), Path('xxx'), 0), PathAccessError(KeyError('yyy',), Path('yyy'), 0)], [])"
received = repr(exc_info.value)
assert expected.replace(',', '') in received.replace(
',', '') # normalize commas for py3.7+ repr change
# check that defaulting works
spec = Coalesce('xxx', 'yyy', default='zzz')
assert glom(val, spec) == 'zzz'
assert repr(spec) == "Coalesce('xxx', 'yyy', default='zzz')"
# check that default_factory works
sentinel_list = []
factory = lambda: sentinel_list
assert glom(val, Coalesce('xxx', 'yyy',
default_factory=factory)) is sentinel_list
with pytest.raises(ValueError):
Coalesce('x', 'y', default=1, default_factory=list)
# check that arbitrary values can be skipped
assert glom(val, Coalesce('xxx', 'yyy', 'a.b', default='zzz',
skip='c')) == 'zzz'
# check that arbitrary exceptions can be ignored
assert glom(val,
Coalesce(lambda x: 1 / 0, 'a.b',
skip_exc=ZeroDivisionError)) == 'c'
def field_expression(f):
if isinstance(f, PhoneNumberField):
return (f.name, Coalesce("as_e164", default=""))
elif isinstance(f, GeometryField):
return (f.name, str)
elif isinstance(f, CountryField):
return (f.name, str)
elif isinstance(f, StdImageField):
return (f.name,
lambda u: settings.FRONT_DOMAIN + settings.MEDIA_URL + u.url)
elif f.name == "password":
return Literal("(caché)")
return f.name
def render_annotations(annos):
for anno in annos:
textcontent = '\n'.join(
glom(
anno,
Coalesce(
(('passport.annotation', [
'lemma',
]), flatten),
default=[],
)))
textcontent = re.sub(r'\n', '<br/>', textcontent)
anno['body'] = mark_safe(tag_transcription(textcontent))
anno['name'] = mark_safe(tag_transcription(anno.get('name', '')))
return annos
def test_sum_integers():
target = list(range(5))
assert glom(target, Sum()) == 10
assert glom(target, Sum(init=lambda: 2)) == 12
target = []
assert glom(target, Sum()) == 0
target = [{"num": 3}, {"num": 2}, {"num": -1}]
assert glom(target, Sum(['num'])) == 4
target = target + [{}] # add a non-compliant dict
assert glom(target, Sum([Coalesce('num', default=0)])) == 4
repr(Sum())
def lemma_details_page(request, lemma_id):
lemma = store.get('lemma', lemma_id)
bibl = glom(
lemma,
Coalesce(
(
(
'passport.bibliography.0.bibliographical_text_field.0',
lambda x: re.sub(r';\s*([A-Z])', r'|\1', x).split(
'|'
) # Workaround: splitten nur wenn nach ";" kein Großbuchstabe folgt
#lambda x: x.split(';')
),
[str.strip]),
default=[]))
lemma['relations'] = {
predicate: [
relation for relation in relations
if relation.get('eclass') == 'BTSLemmaEntry'
]
for predicate, relations in lemma.get('relations', {}).items()
}
return render(
request, 'details/lemma.html', {
'lemma': lemma,
'bibl': bibl,
'ext': {
provider: format_ext_refs(provider, refs)
for provider, refs in (
lemma.get('externalReferences') or {}).items()
},
'coins': coins_openurl_kev(lemma),
'occurrences': {
'corpus': occurrence_count(lemma_id),
},
'annotations': render_annotations(lemma_annotations(lemma_id)),
'tlaVersion': tlaVersion,
'tlaTitle': tlaTitle,
'tlaVersion': tlaVersion,
'tlaIssue': tlaIssue,
'tlaReleaseDate': tlaReleaseDate,
'tlaEditor': tlaEditor,
'tlaBaseURL': tlaBaseURL,
'dateToday': datetime.now().strftime("%d.%m.%Y"),
})
def test_coalesce_stack():
val = {
'a': {
'b': 'c'
}, # basic dictionary nesting
'd': {
'e': ['f'], # list in dictionary
'g': 'h'
},
'i': [{
'j': 'k',
'l': 'm'
}], # list of dictionaries
'n': 'o'
}
actual = _make_stack(Coalesce('xxx', 'yyy'), target=val)
expected = """\
Traceback (most recent call last):
File "test_error.py", line ___, in _make_stack
glom(target, spec)
File "core.py", line ___, in glom
raise err
glom.core.CoalesceError: error raised while processing, details below.
Target-spec trace (most recent last):
- Target: {'a': {'b': 'c'}, 'd': {'e': ['f'], 'g': 'h'}, 'i': [{'j... (len=4)
+ Spec: Coalesce('xxx', 'yyy')
|\\ Spec: 'xxx'
|X glom.core.PathAccessError: could not access 'xxx', part 0 of Path('xxx'), got error: KeyError('xxx')
|\\ Spec: 'yyy'
|X glom.core.PathAccessError: could not access 'yyy', part 0 of Path('yyy'), got error: KeyError('yyy')
glom.core.CoalesceError: no valid values found. Tried ('xxx', 'yyy') and got (PathAccessError, PathAccessError) (at path [])
"""
if _PY2: # see https://github.com/pytest-dev/pytest/issues/1347
assert len(actual.split("\n")) == len(expected.split("\n"))
else:
assert actual == expected
data = {"student": {"info": [{"name": "张三"}, {"name": "李四"}]}}
info = glom(data, ("student.info", ["name"]))
print(info)
info = glom(data, {"info": ("student.info", ["name"])})
print(info)
data_1 = {"school": {"student": [{"name": "张三"}, {"name": "李四"}]}}
data_2 = {"school": {"teacher": [{"name": "王老师"}, {"name": "赵老师"}]}}
spec_1 = {"name": ("school.student", ["name"])}
spec_2 = {"name": ("school.teacher", ["name"])}
print(glom(data_1, spec_1))
print(glom(data_2, spec_2))
spec = {"name": (Coalesce("school.student", "school.teacher"), ["name"])}
print(glom(data_1, spec))
print(glom(data_2, spec))
data = {
"school": {
"student": [{
"name": "张三",
"age": 18
}, {
"name": "李四",
"age": 20
}]
}
}
["Regulation", "regulation", "red"],
["Likelihood", "likelihood_label", "orange"],
["Likelihood Value", "likelihood_score", "orange"],
["Impact", "impact_label", "orange"],
["Impact Value", "impact_score", "orange"],
["Assessment State", "assessment_status"],
["Assessment Progress", "assessment_progress"],
["Report order status", "subscription_status"],
["Report tier", "subscription_tier"],
["Report available", "subscription_available"],
["Industry", "industry"],
]
TP_MAPPING = {
"likelihood_label":
Coalesce("inherent_risk.likelihood_label", default=None),
"likelihood_score":
Coalesce("inherent_risk.likelihood_score", default=None),
"impact_label":
Coalesce("inherent_risk.impact_label", default=None),
"impact_score":
Coalesce("inherent_risk.impact_score", default=None),
"assessment_status":
Coalesce("assessment.status", default=None),
"assessment_progress":
Coalesce("assessment.progress", default=None),
"subscription_status":
Coalesce("subscription.status", default=None),
"subscription_tier":
Coalesce("subscription.tier", default=None),
"subscription_available":
'power{}'.format(i): ('channelPowers.{}.power'.format(i),
Check(type=int)),
'export_energy{}'.format(i):
('channelPowers.{}.exportEnergy'.format(i), Check(type=int)),
'import_energy{}'.format(i):
('channelPowers.{}.importEnergy'.format(i), Check(type=int)),
'phase_id{}'.format(i): ('channelPowers.{}.phaseId'.format(i),
Check(type=int)),
'current{}'.format(i): ('channelPowers.{}.current'.format(i),
Check(type=int))
})
# Voltages
for i in range(3):
spec.update({
'volt{}'.format(i): (Coalesce('voltages.{}.voltage'.format(i),
default=0), Check(type=int)),
'phase_id{}'.format(i): (Coalesce('voltages.{}.phaseId'.format(i),
default=0), Check(type=int))
})
def _external_to_internal(external_dict: dict) -> dict:
"""
Transform incoming message to a format that the db can understand.
:param external_dict: external message format
:return: internal message format
"""
return glom(external_dict, spec)
def transform_smappee(message: dict) -> Smappee:
import json
import requests
import dicttoxml
from tqdm import tqdm
from glom import glom, Coalesce, OMIT
from xml.dom.minidom import parseString
# yapf: disable
TP_MAPPING = {
"id": "id",
"name": "name",
"primary_url": "primary_url",
"industry": "industry",
"custom_id": "custom_id",
"custom_metadata": "custom_metadata",
"report_id": Coalesce("residual_risk.id", default=None),
"report_type": Coalesce("residual_risk.report_type", default=None),
"report_release_date": Coalesce("residual_risk.date", default=None),
"report_tier": Coalesce("residual_risk.tier", default=None),
"likelihood_label": Coalesce("inherent_risk.likelihood_label", default=None),
"likelihood_score": Coalesce("inherent_risk.likelihood_score", default=None),
"impact_label": Coalesce("inherent_risk.impact_label", default=None),
"impact_score": Coalesce("inherent_risk.impact_score", default=None),
"assessment_status": Coalesce("assessment.status", default=None),
"assessment_progress": Coalesce("assessment.progress", default=None),
"subscription_status": Coalesce("subscription.status", default=None),
"subscription_tier": Coalesce("subscription.tier", default=None),
"subscription_available": Coalesce("subscription.is_report_available", default=None),
"residual_risk_outcomes": Coalesce("residual_risk.residual_risk_outcomes", default=[]),
"scores": Coalesce(("residual_risk.scores", [{
"name": "name",
"""Module with data schemas."""
from app.models import enums
from app.utils import datetime
from glom import Coalesce, OMIT, Call, T
DEFAULT_PAGE = 1
DEFAULT_LIMIT = 10
CALL_RECORD_REQUEST = {
"body": {
"type": ("type", enums.CallRecordType),
"timestamp": ("timestamp", float),
"call_id": ("call_id", int),
"source": Coalesce("source", str, default=OMIT),
"destination": Coalesce("destination", str, default=OMIT)
}
}
CALL_RECORD_RESPONSE = {
"id": "id",
"type": "type",
"timestamp": "timestamp",
"call_id": ("call_id", int),
"source": Coalesce("source", default=OMIT),
"destination": Coalesce("destination", default=OMIT)
}
PHONE_BILL_REQUEST = {
"params": {
"phone_number": "phone_number",
"period": (Coalesce("period", default='')),
["Impact", "impact_label", "orange"],
["Impact Value", "impact_score", "orange"],
["Assessment State", "assessment_status"],
["Assessment Progress", "assessment_progress"],
["Requested Completion Date", "assessment_requested_completion_date"],
["Assessment Completion Date", "assessment_completion_date"],
["Report order status", "subscription_status"],
["Report tier", "subscription_tier"],
["Report validated", "subscription_validated"],
["Report available", "subscription_available"],
["Industry", "industry"],
["Tags", "tags"],
]
TP_MAPPING = {
"likelihood_label": Coalesce("inherent_risk.likelihood_label", default=None),
"likelihood_score": Coalesce("inherent_risk.likelihood_score", default=None),
"impact_label": Coalesce("inherent_risk.impact_label", default=None),
"impact_score": Coalesce("inherent_risk.impact_score", default=None),
"assessment_status": Coalesce("assessment.status", default=None),
"assessment_progress": Coalesce("assessment.progress", default=None),
"assessment_requested_completion_date": Coalesce("assessment.requested_completion_date", default=None),
"assessment_completion_date": Coalesce("assessment.completion_date", default=None),
"subscription_status": Coalesce("subscription.status", default=None),
"subscription_tier": Coalesce("subscription.tier", default=None),
"subscription_available": Coalesce("subscription.is_report_available", default=None),
"subscription_validated": Coalesce("subscription.is_validated", default=None),
"tags": (Coalesce("tags", default=[]), ",".join),
}
GAPS_COLUMNS = [
def generer_fichier_deputes(
deputes_path,
groupes_path,
partis_path,
deputes_groupes_path,
deputes_partis_path,
dest,
):
deputes = pd.read_csv(deputes_path)
groupes = pd.read_csv(groupes_path)
deputes_groupes = pd.read_csv(deputes_groupes_path).join(
groupes.set_index("code")[["nom", "sigle"]], on="code")
deputes_groupes = (
deputes_groupes[deputes_groupes.date_fin.isnull()].sort_values(
["code_depute", "relation"]).drop_duplicates(
"code_depute",
keep="last") # garder "P" (président) plutôt que "M" (membre)
.set_index("code_depute"))
deputes_groupes[
"groupe"] = deputes_groupes.nom + " (" + deputes_groupes.sigle + ")"
partis = pd.read_csv(partis_path)
deputes_partis = pd.read_csv(deputes_partis_path).join(
partis.set_index("code")[["nom", "sigle"]], on="code")
deputes_partis = deputes_partis[
deputes_partis.date_fin.isnull()].set_index("code_depute")
deputes_partis = deputes_partis.nom + " (" + deputes_partis.sigle + ")"
deputes_partis.name = "parti"
deputes = deputes.join(deputes_groupes[["groupe", "relation"]],
on=["code"]).join(deputes_partis, on=["code"])
with lzma.open(dest, "wt") as f, id_from_file(
"circonscriptions_legislatives.csv") as id_circos, id_from_file(
"deputes.csv") as id_deputes:
spec = {
"id":
Invoke(id_deputes).specs(code=T.code),
"circonscription_id":
Invoke(id_circos).specs(code=T.circonscription),
**{
c: getattr(T, c)
for c in [
"code",
"nom",
"prenom",
"sexe",
"date_naissance",
"legislature",
"date_debut_mandat",
]
},
"groupe":
Coalesce(T.groupe, skip=pd.isna, default=""),
"parti":
Coalesce(T.parti, skip=pd.isna, default=""),
"date_fin_mandat":
Coalesce(T.date_fin_mandat, skip=pd.isna, default=NULL),
"relation":
Coalesce(T.relation, skip=pd.isna, default=""),
"profession":
Val(NULL),
}
w = csv.DictWriter(f, fieldnames=spec)
w.writeheader()
w.writerows(glom(deputes.itertuples(), Iter(spec)))
