def export_ecosystem(filename): api = os.environ.get("CYBERGRX_API", "https://api.cybergrx.com").rstrip("/") token = os.environ.get("CYBERGRX_API_TOKEN", None) if not token: raise Exception( "The environment variable CYBERGRX_API_TOKEN must be set") uri = api + "/bulk-v1/third-parties" print("Fetching third parties from " + uri + " this can take some time.") response = requests.get(uri, headers={"Authorization": token.strip()}) result = json.loads(response.content.decode("utf-8")) print("Retrieved " + str(len(result)) + " third parties from your ecosystem, building an excel.") wb = Workbook() wb["Sheet"].title = THIRD_PARTY_TABLE wb.create_sheet(GAPS_TABLE) wb.create_sheet(CONTROL_SCORES) wb.create_sheet(COMPANY_TAGS) third_party_writer = sheet_writer(wb, THIRD_PARTY_TABLE, TP_COLUMNS, mapping=TP_MAPPING) findings_writer = sheet_writer(wb, GAPS_TABLE, GAPS_COLUMNS) scores_writer = sheet_writer(wb, CONTROL_SCORES, SCORE_COLUMNS, mapping=SCORE_MAPPING) tags_writer = sheet_writer(wb, COMPANY_TAGS, TAG_COLUMNS) for tp in tqdm(result, total=len(result), desc="Third Party"): third_party_writer(tp) for tag in glom(tp, Coalesce("tags", default=[])): tags_writer({"tag": tag, "company_name": tp["name"]}) for finding in glom(tp, Coalesce("residual_risk.findings", default=[])): finding["company_name"] = tp["name"] findings_writer(finding) for score in glom(tp, Coalesce("residual_risk.scores", default=[])): score["company_name"] = tp["name"] scores_writer(score) # Finalize each writer (fix width, ETC) third_party_writer.finalizer() findings_writer.finalizer() scores_writer.finalizer() tags_writer.finalizer() wb.save("ecosystem.xlsx")
def _spec_to_type( key: str, value: Dict[str, Dict], bases: Tuple[Type, ...] = ()) -> Type: """Using the type specification, create the custom type objects Parameters ---------- key : str The key name corresponding to the specification. It is used as a template for the custom type name. value : Dict The dictionary with the type specification. It looks like: { "key1": {"type": <type1>, "validator": <validator1>}, "key2": {"type": <type2>, "validator": <validator2>}, # ... } bases : Tuple[Type, ...] Base classes Returns ------- Type Custom type object with validators """ type_k = _type_spec[0] dflt_k = _type_spec[6] fields = glom( # NOTE: original ordering is preserved, apart from moving the data # members w/ default arguments later. [(k, v) for k, v in value.items() if type_k in v and dflt_k not in v] + [(k, v) for k, v in value.items() if type_k in v and dflt_k in v], [( { "k": "0", "v": f"1.{type_k}", # TODO: non-trivial defaults like mutable types "d": Coalesce(f"1.{dflt_k}", default=SKIP), }, T.values(), tuple, )], ) # extract key, value and convert to list of tuples ns = dict( chain(*glom( value.values(), [(Coalesce("validator", default_factory=dict), T.items())], ))) # chain dict.items() and create namespace return make_typedconfig(f"{key}_t", fields, namespace=ns, bases=bases)
def classes_sync(master_id, environments): cert_instance = CertsClass(master_id) cert_path = cert_instance.get_cert() private_key_path = cert_instance.get_key() master_address = cert_instance.get_master_address() url = f"https://{master_address}:8140/puppet/v3/environment_classes/" data = [] for environment in environments: req = requests.get( f"{url}?environment={environment}", verify=False, cert=(cert_path, private_key_path), ) target = req.json()["files"] spec = [ Coalesce( ( "classes", [{ "name": "name", "params": ( "params", [{ "name": "name", "type": Coalesce("type", Literal("String")), "default_source": Coalesce("default_source", Literal("")), }], ), "master": Literal(master_id), }], ), Literal([]), ) ] for class_data in list(chain(*glom(target, spec))): class_data["environment"] = environment data.append(class_data) user = os.environ.get("WEBAPP_USER", "") password = os.environ.get("WEBAPP_PASS", "") requests.post("http://webapp:8000/api/classes/sync/", json=data, auth=(user, password))
def get_proc_info(): ret = {} ret['pid'] = os.getpid() _user_t, _sys_t = os.times()[:2] ret['cpu_times'] = {'user_time': _user_t, 'sys_time': _sys_t} ret['cwd'] = os.getcwdu() ret['umask'] = os.umask(os.umask(2)) # have to set to get ret['umask_str'] = '{0:03o}'.format(ret['umask']) ret['owner'] = glom(globals(), Coalesce(T['getpass'].getuser(), T['os'].getuid()), skip_exc=Exception) # use 0 to get current niceness, seems to return process group's nice level unix_only_vals = glom(os, { 'ppid': T.getppid(), 'pgid': T.getpgrp(), 'niceness': T.nice(0) }, skip_exc=AttributeError) ret.update(unix_only_vals) ret['rusage'] = get_rusage_dict() ret['rlimit'] = get_rlimit_dict() return ret
def test_skip(): assert OMIT is SKIP # backwards compat target = { 'a': { 'b': 'c' }, # basic dictionary nesting 'd': { 'e': ['f'], # list in dictionary 'g': 'h' }, 'i': [{ 'j': 'k', 'l': 'm' }], # list of dictionaries 'n': 'o' } res = glom(target, {'a': 'a.b', 'z': Coalesce('x', 'y', default=SKIP)}) assert res['a'] == 'c' # sanity check assert 'x' not in target assert 'y' not in target assert 'z' not in res # test that skip works on lists target = range(7) res = glom(target, [lambda t: t if t % 2 else SKIP]) assert res == [1, 3, 5] # test that skip works on chains (enable conditional applications of transforms) target = range(7) # double each value if it's even, but convert all values to floats res = glom(target, [(lambda x: x * 2 if x % 2 == 0 else SKIP, float)]) assert res == [0.0, 1.0, 4.0, 3.0, 8.0, 5.0, 12.0]
def test_omit(): target = { 'a': { 'b': 'c' }, # basic dictionary nesting 'd': { 'e': ['f'], # list in dictionary 'g': 'h' }, 'i': [{ 'j': 'k', 'l': 'm' }], # list of dictionaries 'n': 'o' } res = glom(target, {'a': 'a.b', 'z': Coalesce('x', 'y', default=OMIT)}) assert res['a'] == 'c' # sanity check assert 'x' not in target assert 'y' not in target assert 'z' not in res # test that it works on lists target = range(7) res = glom(target, [lambda t: t if t % 2 else OMIT]) assert res == [1, 3, 5]
def process(self, task): """Return list of jobs""" # NOTE: use dictionary unpacking to optionally overwrite global modules # with task specific modules opts = { "module": self.options["module"], **self.options[task], self.backend: self.options[self.backend], } if opts.get("inputs", None) == "ignore": jobs = [BatchJob(task, opts, self.tmpl_dir, self.backend)] elif opts.get("inputs", None) == "all": # e.g. panel of normal keys = reduce( lambda i, j: i.union(set(j)), [i.keys() for i in self.inputs], set() ) inputs = glom( self.inputs, dict((key, [Coalesce(key, default="")]) for key in keys) ) for key in keys: # filter out "no files" (shows as empty string above) inputs[key] = [i for i in filter(None, inputs[key])] jobs = [BatchJob(task, dict(**inputs, **opts), self.tmpl_dir, self.backend)] else: inputs = opts.get("inputs", self.inputs) # allow overriding inputs per job jobs = [ BatchJob(task, dict(**infile, **opts), self.tmpl_dir, self.backend) for infile in inputs ] return jobs
def process_third_party(tp): third_party_writer(tp) for outcome in glom( tp, Coalesce("residual_risk.residual_risk_outcomes", default=[])): outcome["company_name"] = tp["name"] residual_risk_writer(outcome)
def _spec_to_type( key: str, value: Dict[str, Dict], bases: Tuple[Type, ...] = ()) -> Type: """Using the type specification, create the custom type objects Parameters ---------- key : str The key name corresponding to the specification. It is used as a template for the custom type name. value : Dict The dictionary with the type specification. It looks like: { "key1": {"type": <type1>, "validator": <validator1>}, "key2": {"type": <type2>, "validator": <validator2>}, # ... } bases : Tuple[Type] Base classes Returns ------- Type Custom type object with validators """ fields = glom( value.items(), [( { "k": "0", "v": "1.type", # TODO: non-trivial defaults like mutable types "d": Coalesce("1.default", default=SKIP), }, T.values(), tuple, )], ) # extract key, value and convert to list of tuples ns = dict( chain(*glom( value.values(), [(Coalesce("validator", default_factory=dict), T.items())], ))) # chain dict.items() and create namespace return make_dataconfig(f"{key}_t", fields, namespace=ns, bases=bases)
def test_coalesce(): val = { 'a': { 'b': 'c' }, # basic dictionary nesting 'd': { 'e': ['f'], # list in dictionary 'g': 'h' }, 'i': [{ 'j': 'k', 'l': 'm' }], # list of dictionaries 'n': 'o' } assert glom(val, 'a.b') == 'c' assert glom(val, Coalesce('xxx', 'yyy', 'a.b')) == 'c' # check that defaulting works spec = Coalesce('xxx', 'yyy', default='zzz') assert glom(val, spec) == 'zzz' assert repr(spec) == "Coalesce('xxx', 'yyy', default='zzz')" # check that default_factory works sentinel_list = [] factory = lambda: sentinel_list assert glom(val, Coalesce('xxx', 'yyy', default_factory=factory)) is sentinel_list with pytest.raises(ValueError): Coalesce('x', 'y', default=1, default_factory=list) # check that arbitrary values can be skipped assert glom(val, Coalesce('xxx', 'yyy', 'a.b', default='zzz', skip='c')) == 'zzz' # check that arbitrary exceptions can be ignored assert glom(val, Coalesce(lambda x: 1 / 0, 'a.b', skip_exc=ZeroDivisionError)) == 'c' target = {'a': 1, 'b': 3, 'c': 4} spec = Coalesce('a', 'b', 'c', skip=lambda x: x % 2) assert glom(target, spec) == 4 spec = Coalesce('a', 'b', 'c', skip=(1, )) assert glom(target, spec) == 3 with pytest.raises(TypeError): Coalesce(bad_kwarg=True)
def test_check_basic(): assert glom([0, SKIP], [T]) == [0] # sanity check SKIP target = [{'id': 0}, {'id': 1}, {'id': 2}] # check that skipping non-passing values works assert glom(target, ([Coalesce(Check('id', equal_to=0), default=SKIP)], T[0])) == {'id': 0} assert glom(target, ([Check('id', equal_to=0, default=SKIP)], T[0])) == {'id': 0} # check that stopping iteration on non-passing values works assert glom(target, [Check('id', equal_to=0, default=STOP)]) == [{'id': 0}] # check that stopping chain execution on non-passing values works spec = (Check(validate=lambda x: len(x) > 0, default=STOP), T[0]) assert glom('hello', spec) == 'h' assert glom('', spec) == '' # would fail with IndexError if STOP didn't work assert repr(Check()) == 'Check()' assert repr(Check(T.a)) == 'Check(T.a)' assert repr(Check(equal_to=1)) == 'Check(equal_to=1)' target = [1, 'a'] assert glom(target, [Check(type=str, default=SKIP)]) == ['a'] assert glom(target, [Check(type=(str, int))]) == [1, 'a'] assert glom(target, [Check(instance_of=str, default=SKIP)]) == ['a'] assert glom(target, [Check(instance_of=(str, int))]) == [1, 'a'] target = ['1'] assert glom(target, [Check(validate=(int, float))]) assert glom(target, [Check()]) # bare check does a truthy check failing_checks = [({'a': {'b': 1}}, {'a': ('a', 'b', Check(type=str))}, '''target at path ['a', 'b'] failed check, got error: "expected type to be 'str', found type 'int'"'''), ({'a': {'b': 1}}, {'a': ('a', Check('b', type=str))}, '''target at path ['a'] failed check, subtarget at 'b' got error: "expected type to be 'str', found type 'int'"'''), (1, Check(type=(str, bool))), (1, Check(instance_of=str)), (1, Check(instance_of=(str, bool))), (1, Check(equal_to=0)), (1, Check(one_of=(0,))), (1, Check(one_of=(0, 2))), ('-3.14', Check(validate=int)), ('', Check(validate=lambda x: False)),] for fc in failing_checks: if len(fc) == 2: target, check = fc msg = None else: target, check, msg = fc with raises(CheckError) as exc_info: glom(target, check) if msg is not None: assert str(exc_info.value) == msg assert repr(exc_info.value)
def handle(self, *args, person_email, **options): try: p = Person.objects.get_by_natural_key(person_email) except Person.DoesNotExist: raise CommandError("L'email donné est inconnu.") spec_role = get_all_fields(Role) spec_event = { "Nom": "name", "URL": ("id", lambda id: front_url("view_event", args=[id])), } spec_membership = { "Nom": "supportgroup.name", "URL": ("supportgroup.id", lambda id: front_url("view_group", args=[id])), "Animateur": "is_referent", "Gestionnaire": "is_manager", } spec_payment = get_all_fields(Payment) spec_subscription = get_all_fields(Subscription) spec_event_images = get_all_fields(EventImage) spec_form_submissions = get_all_fields(PersonFormSubmission) spec_tags = get_all_fields(PersonTag) spec_person = { **get_all_fields(Person), "pays": ("location_country", str), "Rôle": ("role", spec_role), "événements organisés": ("organized_events", T.all(), [spec_event]), "participations aux événements": ( "rsvps", T.all(), [("event", spec_event)], ), "participations à des groupes": ("memberships", T.all(), [spec_membership]), "paiements": ("payments", T.all(), [spec_payment]), "souscription au don mensuel": Coalesce(("subscription", spec_subscription), default=None), "images d'événements": ("event_images", T.all(), [spec_event_images]), "réponses à des formulaires": ( "form_submissions", T.all(), [spec_form_submissions], ), "libellés": ("tags", T.all(), [spec_tags]), } self.stdout.ending = "" json.dump(glom(p, spec_person), self.stdout, cls=DjangoJSONEncoder)
def test_check_ported_tests(): """ Tests ported from Check() to make sure all the functionality has an analogue. """ target = [{'id': 0}, {'id': 1}, {'id': 2}] # check that skipping non-passing values works assert glom(target, [Coalesce(M(T['id']) == 0, default=SKIP)]) == [{'id': 0}] # TODO: should M(subspec, default='') work? I lean no. # NB: this is not a very idiomatic use of Match, just brought over for Check reasons assert glom(target, [Match({'id': And(int, M == 1)}, default=SKIP)]) == [{'id': 1}] assert glom(target, [Match({'id': And(int, M <= 1)}, default=STOP)]) == [{'id': 0}, {'id': 1}] # check that stopping chain execution on non-passing values works spec = (Or(Match(len), Val(STOP)), T[0]) assert glom('hello', spec, glom_debug=True) == 'h' assert glom('', spec) == '' # would fail with IndexError if STOP didn't work target = [1, u'a'] assert glom(target, [Match(unicode, default=SKIP)]) == ['a'] assert glom(target, Match([Or(unicode, int)])) == [1, 'a'] target = ['1'] assert glom(target, [(M(T), int)]) == [1] assert glom(target, M(T)) == ['1'] failing_checks = [({'a': {'b': 1}}, {'a': ('a', 'b', Match(str))}, '''expected type str, not int'''), # TODO: bbrepr at least, maybe include path like Check did ({'a': {'b': 1}}, {'a': ('a', Match({'b': str}))}, '''expected type str, not int'''), # TODO: include subspec path ('b') (1, Match(Or(unicode, bool))), (1, Match(unicode)), (1, Match(0)), (1, Match(Or(0, 2))), ('-3.14', Match(lambda x: int(x) > 0)), # ('-3.14', M(lambda x: int(x) > 0)), # TODO: M doesn't behave quite like Match because it's mode-free ] for fc in failing_checks: if len(fc) == 2: target, check = fc msg = None else: target, check, msg = fc with pytest.raises(MatchError) as exc_info: glom(target, check) if msg is not None: actual_msg = str(exc_info.value) assert actual_msg.find(msg) != -1 assert repr(exc_info.value) return
def _parse_api_response(resp_json): """Parse the json returned by the alerts API and extract relevant bits.""" service_subspec = { 'ServiceType': 'ServiceType', 'ServiceTypeDescription': 'ServiceTypeDescription', 'ServiceName': 'ServiceName', 'ServiceId': 'ServiceId' } spec = ( 'CTAAlerts.Alert', [ # for each entry in CTAAlerts.Alert pull out the following info { 'AlertId': ('AlertId', int), 'ShortDescription': 'ShortDescription', 'FullDescription': 'FullDescription.#cdata-section', 'Impact': 'Impact', 'SeverityScore': ('SeverityScore', int), 'LastSeen': Literal(pytz.UTC.localize(dt.datetime.utcnow()).isoformat()), 'EventStart': Coalesce(('EventStart', _chicago_to_utc), default=None), 'EventEnd': Coalesce(('EventEnd', _chicago_to_utc), default=None), # pull out each entry in ImpactedService if there are multiple 'ImpactedService': ( 'ImpactedService.Service', # some chicanery to make sure this entry is always a # list even though the API just returns a nested dict # when there's only one service lambda s: glom(s, [service_subspec]) if isinstance(s, list) else [glom(s, service_subspec)]) } ]) return glom(resp_json, spec)
def test_initial_integration(): class Example(object): pass example = Example() subexample = Example() subexample.name = 'good_name' example.mapping = {'key': subexample} val = { 'a': { 'b': 'c' }, # basic dictionary nesting 'example': example, # basic object 'd': { 'e': ['f'], # list in dictionary 'g': 'h' }, 'i': [{ 'j': 'k', 'l': 'm' }], # list of dictionaries 'n': 'o' } spec = { 'a': (Inspect(recursive=True), 'a', 'b'), # inspect just prints here 'name': 'example.mapping.key.name', # test object access 'e': 'd.e', # d.e[0] or d.e: (callable to fetch 0) 'i': ( 'i', [{ 'j': 'j' }] ), # TODO: support True for cases when the value should simply be mapped into the field name? 'n': ('n', lambda n: n.upper()), 'p': Coalesce('xxx', 'yyy', default='zzz') } ret = glom(val, spec) print('in: ', val) print('got:', ret) expected = { 'a': 'c', 'name': 'good_name', 'e': ['f'], 'i': [{ 'j': 'k' }], 'n': 'O', 'p': 'zzz' } print('exp:', expected) assert ret == expected
def serialize_notifications(notifications): # All fields are either spec = [{ "id": "id", "status": "status", "content": Coalesce(T.html_content(), T.announcement.html_content(), skip="", default=""), "icon": Coalesce("icon", "announcement.icon", skip="", default=""), "link": Coalesce("link", "announcement.link", skip="", default=""), "created": (Coalesce("announcement.start_date", "created"), T.isoformat()), }] return glom(notifications, spec)
def test_coalesce(): val = { 'a': { 'b': 'c' }, # basic dictionary nesting 'd': { 'e': ['f'], # list in dictionary 'g': 'h' }, 'i': [{ 'j': 'k', 'l': 'm' }], # list of dictionaries 'n': 'o' } assert glom(val, 'a.b') == 'c' assert glom(val, Coalesce('xxx', 'yyy', 'a.b')) == 'c' try: glom(val, Coalesce('xxx', 'yyy')) except CoalesceError as ce: msg = str(ce) assert "'xxx'" in msg assert "'yyy'" in msg assert msg.count('PathAccessError') == 2 else: assert False, 'expected a CoalesceError' # check that defaulting works assert glom(val, Coalesce('xxx', 'yyy', default='zzz')) == 'zzz' # check that arbitrary values can be skipped assert glom(val, Coalesce('xxx', 'yyy', 'a.b', default='zzz', skip='c')) == 'zzz' # check that arbitrary exceptions can be ignored assert glom(val, Coalesce(lambda x: 1 / 0, 'a.b', skip_exc=ZeroDivisionError)) == 'c'
def test_coalesce(): val = { 'a': { 'b': 'c' }, # basic dictionary nesting 'd': { 'e': ['f'], # list in dictionary 'g': 'h' }, 'i': [{ 'j': 'k', 'l': 'm' }], # list of dictionaries 'n': 'o' } assert glom(val, 'a.b') == 'c' assert glom(val, Coalesce('xxx', 'yyy', 'a.b')) == 'c' with pytest.raises(CoalesceError) as exc_info: glom(val, Coalesce('xxx', 'yyy')) msg = exc_info.exconly() assert "'xxx'" in msg assert "'yyy'" in msg assert msg.count('PathAccessError') == 2 expected = "[PathAccessError(KeyError('xxx',), Path('xxx'), 0), PathAccessError(KeyError('yyy',), Path('yyy'), 0)], [])" received = repr(exc_info.value) assert expected.replace(',', '') in received.replace( ',', '') # normalize commas for py3.7+ repr change # check that defaulting works spec = Coalesce('xxx', 'yyy', default='zzz') assert glom(val, spec) == 'zzz' assert repr(spec) == "Coalesce('xxx', 'yyy', default='zzz')" # check that default_factory works sentinel_list = [] factory = lambda: sentinel_list assert glom(val, Coalesce('xxx', 'yyy', default_factory=factory)) is sentinel_list with pytest.raises(ValueError): Coalesce('x', 'y', default=1, default_factory=list) # check that arbitrary values can be skipped assert glom(val, Coalesce('xxx', 'yyy', 'a.b', default='zzz', skip='c')) == 'zzz' # check that arbitrary exceptions can be ignored assert glom(val, Coalesce(lambda x: 1 / 0, 'a.b', skip_exc=ZeroDivisionError)) == 'c'
def field_expression(f): if isinstance(f, PhoneNumberField): return (f.name, Coalesce("as_e164", default="")) elif isinstance(f, GeometryField): return (f.name, str) elif isinstance(f, CountryField): return (f.name, str) elif isinstance(f, StdImageField): return (f.name, lambda u: settings.FRONT_DOMAIN + settings.MEDIA_URL + u.url) elif f.name == "password": return Literal("(caché)") return f.name
def render_annotations(annos): for anno in annos: textcontent = '\n'.join( glom( anno, Coalesce( (('passport.annotation', [ 'lemma', ]), flatten), default=[], ))) textcontent = re.sub(r'\n', '<br/>', textcontent) anno['body'] = mark_safe(tag_transcription(textcontent)) anno['name'] = mark_safe(tag_transcription(anno.get('name', ''))) return annos
def test_sum_integers(): target = list(range(5)) assert glom(target, Sum()) == 10 assert glom(target, Sum(init=lambda: 2)) == 12 target = [] assert glom(target, Sum()) == 0 target = [{"num": 3}, {"num": 2}, {"num": -1}] assert glom(target, Sum(['num'])) == 4 target = target + [{}] # add a non-compliant dict assert glom(target, Sum([Coalesce('num', default=0)])) == 4 repr(Sum())
def lemma_details_page(request, lemma_id): lemma = store.get('lemma', lemma_id) bibl = glom( lemma, Coalesce( ( ( 'passport.bibliography.0.bibliographical_text_field.0', lambda x: re.sub(r';\s*([A-Z])', r'|\1', x).split( '|' ) # Workaround: splitten nur wenn nach ";" kein Großbuchstabe folgt #lambda x: x.split(';') ), [str.strip]), default=[])) lemma['relations'] = { predicate: [ relation for relation in relations if relation.get('eclass') == 'BTSLemmaEntry' ] for predicate, relations in lemma.get('relations', {}).items() } return render( request, 'details/lemma.html', { 'lemma': lemma, 'bibl': bibl, 'ext': { provider: format_ext_refs(provider, refs) for provider, refs in ( lemma.get('externalReferences') or {}).items() }, 'coins': coins_openurl_kev(lemma), 'occurrences': { 'corpus': occurrence_count(lemma_id), }, 'annotations': render_annotations(lemma_annotations(lemma_id)), 'tlaVersion': tlaVersion, 'tlaTitle': tlaTitle, 'tlaVersion': tlaVersion, 'tlaIssue': tlaIssue, 'tlaReleaseDate': tlaReleaseDate, 'tlaEditor': tlaEditor, 'tlaBaseURL': tlaBaseURL, 'dateToday': datetime.now().strftime("%d.%m.%Y"), })
def test_coalesce_stack(): val = { 'a': { 'b': 'c' }, # basic dictionary nesting 'd': { 'e': ['f'], # list in dictionary 'g': 'h' }, 'i': [{ 'j': 'k', 'l': 'm' }], # list of dictionaries 'n': 'o' } actual = _make_stack(Coalesce('xxx', 'yyy'), target=val) expected = """\ Traceback (most recent call last): File "test_error.py", line ___, in _make_stack glom(target, spec) File "core.py", line ___, in glom raise err glom.core.CoalesceError: error raised while processing, details below. Target-spec trace (most recent last): - Target: {'a': {'b': 'c'}, 'd': {'e': ['f'], 'g': 'h'}, 'i': [{'j... (len=4) + Spec: Coalesce('xxx', 'yyy') |\\ Spec: 'xxx' |X glom.core.PathAccessError: could not access 'xxx', part 0 of Path('xxx'), got error: KeyError('xxx') |\\ Spec: 'yyy' |X glom.core.PathAccessError: could not access 'yyy', part 0 of Path('yyy'), got error: KeyError('yyy') glom.core.CoalesceError: no valid values found. Tried ('xxx', 'yyy') and got (PathAccessError, PathAccessError) (at path []) """ if _PY2: # see https://github.com/pytest-dev/pytest/issues/1347 assert len(actual.split("\n")) == len(expected.split("\n")) else: assert actual == expected
data = {"student": {"info": [{"name": "张三"}, {"name": "李四"}]}} info = glom(data, ("student.info", ["name"])) print(info) info = glom(data, {"info": ("student.info", ["name"])}) print(info) data_1 = {"school": {"student": [{"name": "张三"}, {"name": "李四"}]}} data_2 = {"school": {"teacher": [{"name": "王老师"}, {"name": "赵老师"}]}} spec_1 = {"name": ("school.student", ["name"])} spec_2 = {"name": ("school.teacher", ["name"])} print(glom(data_1, spec_1)) print(glom(data_2, spec_2)) spec = {"name": (Coalesce("school.student", "school.teacher"), ["name"])} print(glom(data_1, spec)) print(glom(data_2, spec)) data = { "school": { "student": [{ "name": "张三", "age": 18 }, { "name": "李四", "age": 20 }] } }
["Regulation", "regulation", "red"], ["Likelihood", "likelihood_label", "orange"], ["Likelihood Value", "likelihood_score", "orange"], ["Impact", "impact_label", "orange"], ["Impact Value", "impact_score", "orange"], ["Assessment State", "assessment_status"], ["Assessment Progress", "assessment_progress"], ["Report order status", "subscription_status"], ["Report tier", "subscription_tier"], ["Report available", "subscription_available"], ["Industry", "industry"], ] TP_MAPPING = { "likelihood_label": Coalesce("inherent_risk.likelihood_label", default=None), "likelihood_score": Coalesce("inherent_risk.likelihood_score", default=None), "impact_label": Coalesce("inherent_risk.impact_label", default=None), "impact_score": Coalesce("inherent_risk.impact_score", default=None), "assessment_status": Coalesce("assessment.status", default=None), "assessment_progress": Coalesce("assessment.progress", default=None), "subscription_status": Coalesce("subscription.status", default=None), "subscription_tier": Coalesce("subscription.tier", default=None), "subscription_available":
'power{}'.format(i): ('channelPowers.{}.power'.format(i), Check(type=int)), 'export_energy{}'.format(i): ('channelPowers.{}.exportEnergy'.format(i), Check(type=int)), 'import_energy{}'.format(i): ('channelPowers.{}.importEnergy'.format(i), Check(type=int)), 'phase_id{}'.format(i): ('channelPowers.{}.phaseId'.format(i), Check(type=int)), 'current{}'.format(i): ('channelPowers.{}.current'.format(i), Check(type=int)) }) # Voltages for i in range(3): spec.update({ 'volt{}'.format(i): (Coalesce('voltages.{}.voltage'.format(i), default=0), Check(type=int)), 'phase_id{}'.format(i): (Coalesce('voltages.{}.phaseId'.format(i), default=0), Check(type=int)) }) def _external_to_internal(external_dict: dict) -> dict: """ Transform incoming message to a format that the db can understand. :param external_dict: external message format :return: internal message format """ return glom(external_dict, spec) def transform_smappee(message: dict) -> Smappee:
import json import requests import dicttoxml from tqdm import tqdm from glom import glom, Coalesce, OMIT from xml.dom.minidom import parseString # yapf: disable TP_MAPPING = { "id": "id", "name": "name", "primary_url": "primary_url", "industry": "industry", "custom_id": "custom_id", "custom_metadata": "custom_metadata", "report_id": Coalesce("residual_risk.id", default=None), "report_type": Coalesce("residual_risk.report_type", default=None), "report_release_date": Coalesce("residual_risk.date", default=None), "report_tier": Coalesce("residual_risk.tier", default=None), "likelihood_label": Coalesce("inherent_risk.likelihood_label", default=None), "likelihood_score": Coalesce("inherent_risk.likelihood_score", default=None), "impact_label": Coalesce("inherent_risk.impact_label", default=None), "impact_score": Coalesce("inherent_risk.impact_score", default=None), "assessment_status": Coalesce("assessment.status", default=None), "assessment_progress": Coalesce("assessment.progress", default=None), "subscription_status": Coalesce("subscription.status", default=None), "subscription_tier": Coalesce("subscription.tier", default=None), "subscription_available": Coalesce("subscription.is_report_available", default=None), "residual_risk_outcomes": Coalesce("residual_risk.residual_risk_outcomes", default=[]), "scores": Coalesce(("residual_risk.scores", [{ "name": "name",
"""Module with data schemas.""" from app.models import enums from app.utils import datetime from glom import Coalesce, OMIT, Call, T DEFAULT_PAGE = 1 DEFAULT_LIMIT = 10 CALL_RECORD_REQUEST = { "body": { "type": ("type", enums.CallRecordType), "timestamp": ("timestamp", float), "call_id": ("call_id", int), "source": Coalesce("source", str, default=OMIT), "destination": Coalesce("destination", str, default=OMIT) } } CALL_RECORD_RESPONSE = { "id": "id", "type": "type", "timestamp": "timestamp", "call_id": ("call_id", int), "source": Coalesce("source", default=OMIT), "destination": Coalesce("destination", default=OMIT) } PHONE_BILL_REQUEST = { "params": { "phone_number": "phone_number", "period": (Coalesce("period", default='')),
["Impact", "impact_label", "orange"], ["Impact Value", "impact_score", "orange"], ["Assessment State", "assessment_status"], ["Assessment Progress", "assessment_progress"], ["Requested Completion Date", "assessment_requested_completion_date"], ["Assessment Completion Date", "assessment_completion_date"], ["Report order status", "subscription_status"], ["Report tier", "subscription_tier"], ["Report validated", "subscription_validated"], ["Report available", "subscription_available"], ["Industry", "industry"], ["Tags", "tags"], ] TP_MAPPING = { "likelihood_label": Coalesce("inherent_risk.likelihood_label", default=None), "likelihood_score": Coalesce("inherent_risk.likelihood_score", default=None), "impact_label": Coalesce("inherent_risk.impact_label", default=None), "impact_score": Coalesce("inherent_risk.impact_score", default=None), "assessment_status": Coalesce("assessment.status", default=None), "assessment_progress": Coalesce("assessment.progress", default=None), "assessment_requested_completion_date": Coalesce("assessment.requested_completion_date", default=None), "assessment_completion_date": Coalesce("assessment.completion_date", default=None), "subscription_status": Coalesce("subscription.status", default=None), "subscription_tier": Coalesce("subscription.tier", default=None), "subscription_available": Coalesce("subscription.is_report_available", default=None), "subscription_validated": Coalesce("subscription.is_validated", default=None), "tags": (Coalesce("tags", default=[]), ",".join), } GAPS_COLUMNS = [
def generer_fichier_deputes( deputes_path, groupes_path, partis_path, deputes_groupes_path, deputes_partis_path, dest, ): deputes = pd.read_csv(deputes_path) groupes = pd.read_csv(groupes_path) deputes_groupes = pd.read_csv(deputes_groupes_path).join( groupes.set_index("code")[["nom", "sigle"]], on="code") deputes_groupes = ( deputes_groupes[deputes_groupes.date_fin.isnull()].sort_values( ["code_depute", "relation"]).drop_duplicates( "code_depute", keep="last") # garder "P" (président) plutôt que "M" (membre) .set_index("code_depute")) deputes_groupes[ "groupe"] = deputes_groupes.nom + " (" + deputes_groupes.sigle + ")" partis = pd.read_csv(partis_path) deputes_partis = pd.read_csv(deputes_partis_path).join( partis.set_index("code")[["nom", "sigle"]], on="code") deputes_partis = deputes_partis[ deputes_partis.date_fin.isnull()].set_index("code_depute") deputes_partis = deputes_partis.nom + " (" + deputes_partis.sigle + ")" deputes_partis.name = "parti" deputes = deputes.join(deputes_groupes[["groupe", "relation"]], on=["code"]).join(deputes_partis, on=["code"]) with lzma.open(dest, "wt") as f, id_from_file( "circonscriptions_legislatives.csv") as id_circos, id_from_file( "deputes.csv") as id_deputes: spec = { "id": Invoke(id_deputes).specs(code=T.code), "circonscription_id": Invoke(id_circos).specs(code=T.circonscription), **{ c: getattr(T, c) for c in [ "code", "nom", "prenom", "sexe", "date_naissance", "legislature", "date_debut_mandat", ] }, "groupe": Coalesce(T.groupe, skip=pd.isna, default=""), "parti": Coalesce(T.parti, skip=pd.isna, default=""), "date_fin_mandat": Coalesce(T.date_fin_mandat, skip=pd.isna, default=NULL), "relation": Coalesce(T.relation, skip=pd.isna, default=""), "profession": Val(NULL), } w = csv.DictWriter(f, fieldnames=spec) w.writeheader() w.writerows(glom(deputes.itertuples(), Iter(spec)))