def test_inspect():
# test repr
assert repr(Inspect()) == '<INSPECT>'
target = {'a': {'b': 'c'}}
import pdb
# test breakpoint
assert Inspect(breakpoint=True).breakpoint == pdb.set_trace
with pytest.raises(TypeError):
Inspect(breakpoint='lol')
tracker = []
spec = {'a': Inspect('a.b', echo=False, breakpoint=lambda: tracker.append(True))}
glom(target, spec)
assert len(tracker) == 1
# test post_mortem
assert Inspect(post_mortem=True).post_mortem == pdb.post_mortem
with pytest.raises(TypeError):
Inspect(post_mortem='lol')
tracker = []
spec = {'a': Inspect('nope.nope', post_mortem=lambda: tracker.append(True))}
assert glom(target, spec, default='default') == 'default'
assert len(tracker) == 1
def test_skip():
assert OMIT is SKIP # backwards compat
target = {'a': {'b': 'c'}, # basic dictionary nesting
'd': {'e': ['f'], # list in dictionary
'g': 'h'},
'i': [{'j': 'k', 'l': 'm'}], # list of dictionaries
'n': 'o'}
res = glom(target, {'a': 'a.b',
'z': Coalesce('x', 'y', default=SKIP)})
assert res['a'] == 'c' # sanity check
assert 'x' not in target
assert 'y' not in target
assert 'z' not in res
# test that skip works on lists
target = range(7)
res = glom(target, [lambda t: t if t % 2 else SKIP])
assert res == [1, 3, 5]
# test that skip works on chains (enable conditional applications of transforms)
target = range(7)
# double each value if it's even, but convert all values to floats
res = glom(target, [(lambda x: x * 2 if x % 2 == 0 else SKIP, float)])
assert res == [0.0, 1.0, 4.0, 3.0, 8.0, 5.0, 12.0]
def test_bypass_getitem():
target = list(range(3)) * 3
with pytest.raises(PathAccessError):
glom.glom(target, 'count')
res = glom.glom(target, lambda list_obj: list_obj.count(1))
assert res == 3
def test_scope():
assert glom(None, S['foo'], scope={'foo': 'bar'}) == 'bar'
target = range(3)
spec = [(S, lambda S: S['multiplier'] * S[T])]
scope = {'multiplier': 2}
assert glom(target, spec, scope=scope) == [0, 2, 4]
scope = {'multiplier': 2.5}
assert glom(target, spec, scope=scope) == [0.0, 2.5, 5.0]
def test_empty_path_access():
target = {}
assert glom(target, Path()) is target
assert glom(target, (Path(), Path(), Path())) is target
dup_dict = glom(target, {'target': Path(),
'target2': Path()})
dup_dict['target'] is target
dup_dict['target2'] is target
def get_gc_info():
import gc
ret = {}
ret['is_enabled'] = gc.isenabled()
ret['thresholds'] = gc.get_threshold()
ret['counts'] = glom(gc, T.get_count(), skip_exc=Exception)
ret['obj_count'] = glom(gc, T.get_objects().__len__(), skip_exc=Exception)
return ret
def test_literal():
expected = {'value': 'c',
'type': 'a.b'}
target = {'a': {'b': 'c'}}
val = glom(target, {'value': 'a.b',
'type': Literal('a.b')})
assert val == expected
assert glom(None, Literal('success')) == 'success'
assert repr(Literal(3.14)) == 'Literal(3.14)'
def test_spec():
assert glom(5, T) == 5 # check assumption about echo behavior
echo = Spec(T)
assert echo.glom(5) == 5
assert glom(5, echo) == 5
echo2 = Spec(echo)
assert echo2.glom(5) == 5
with pytest.raises(TypeError, match='expected spec to be'):
glom({}, object())
return
def test_stop():
# test that stop works on iterables
target = iter([0, 1, 2, STOP, 3, 4])
assert glom(target, [T]) == [0, 1, 2]
# test that stop works on chains (but doesn't stop iteration up the stack)
target = ['a', ' b', ' c ', ' ', ' done']
assert glom(target, [(lambda x: x.strip(),
lambda x: x if x else STOP,
lambda x: x[0])]) == ['a', 'b', 'c', '', 'd']
return
def test_iter_set():
some_ints = set(range(5))
some_floats = glom.glom(some_ints, [float])
assert sorted(some_floats) == [0.0, 1.0, 2.0, 3.0, 4.0]
# now without defaults
glommer = Glommer(register_default_types=False)
glommer.register(set, iterate=iter)
some_floats = glom.glom(some_ints, [float])
assert sorted(some_floats) == [0.0, 1.0, 2.0, 3.0, 4.0]
def test_spec_and_recursion():
assert repr(Spec('a.b.c')) == "Spec('a.b.c')"
# Call doesn't normally recurse, but Spec can make it do so
assert glom(
['a', 'b', 'c'],
Call(list, args=(
Spec(Call(reversed, args=(Spec(T),))),)
)) == ['c', 'b', 'a']
assert glom(['cat', {'cat': 1}], T[1][T[0]]) == 1
assert glom(
[['ab', 'cd', 'ef'], ''.join],
Call(T[1], args=(Spec((T[0], [T[1:]])),))) == 'bdf'
def test_coalesce():
val = {'a': {'b': 'c'}, # basic dictionary nesting
'd': {'e': ['f'], # list in dictionary
'g': 'h'},
'i': [{'j': 'k', 'l': 'm'}], # list of dictionaries
'n': 'o'}
assert glom(val, 'a.b') == 'c'
assert glom(val, Coalesce('xxx', 'yyy', 'a.b')) == 'c'
with pytest.raises(CoalesceError) as exc_info:
glom(val, Coalesce('xxx', 'yyy'))
msg = exc_info.exconly()
assert "'xxx'" in msg
assert "'yyy'" in msg
assert msg.count('PathAccessError') == 2
assert "[PathAccessError(KeyError('xxx',), Path('xxx'), 0), PathAccessError(KeyError('yyy',), Path('yyy'), 0)], [])" in repr(exc_info.value)
# check that defaulting works
assert glom(val, Coalesce('xxx', 'yyy', default='zzz')) == 'zzz'
# check that default_factory works
sentinel_list = []
factory = lambda: sentinel_list
assert glom(val, Coalesce('xxx', 'yyy', default_factory=factory)) is sentinel_list
with pytest.raises(ValueError):
Coalesce('x', 'y', default=1, default_factory=list)
# check that arbitrary values can be skipped
assert glom(val, Coalesce('xxx', 'yyy', 'a.b', default='zzz', skip='c')) == 'zzz'
# check that arbitrary exceptions can be ignored
assert glom(val, Coalesce(lambda x: 1/0, 'a.b', skip_exc=ZeroDivisionError)) == 'c'
def test_top_level_default():
expected = object()
val = glom({}, 'a.b.c', default=expected)
assert val is expected
val = glom({}, lambda x: 1/0, skip_exc=ZeroDivisionError)
assert val is None
val = glom({}, lambda x: 1/0, skip_exc=ZeroDivisionError, default=expected)
assert val is expected
with pytest.raises(KeyError):
# p degenerate case if you ask me
glom({}, 'x', skip_exc=KeyError, default=glom_core._MISSING)
return
def get_pyvm_info():
ret = {}
ret['executable'] = sys.executable
ret['is_64bit'] = IS_64BIT
ret['version'] = sys.version
ret['compiler'] = platform.python_compiler()
ret['build_date'] = platform.python_build()[1]
ret['version_info'] = list(sys.version_info)
ret['have_ucs4'] = getattr(sys, 'maxunicode', 0) > 65536
ret['have_readline'] = HAVE_READLINE
ret['active_thread_count'] = glom(sys, T._current_frames().__len__(), skip_exc=Exception)
ret['recursion_limit'] = sys.getrecursionlimit()
ret['gc'] = glom(None, Call(get_gc_info), skip_exc=Exception) # effectively try/except:pass
ret['check_interval'] = sys.getcheckinterval()
return ret
def to_dict(self, *a, **kw):
ret = super(ContextualInternalServerError, self).to_dict(*a, **kw)
del ret['exc_info']
exc_info = getattr(self, 'exc_info', None)
if not exc_info:
return ret
exc_tb = exc_info.tb_info.to_dict()
for i, frame in enumerate(exc_tb['frames']):
if self.hide_internal_frames:
if not frame['line'] and frame['module_path'] == '<string>':
frame['is_hidden'] = True
elif frame['module_name'] == 'clastic.sinter' and \
frame['func_name'] == 'inject':
frame['is_hidden'] = True
frame['id'] = i
pre_start_lineno = glom(frame, T['pre_lines'][0]['lineno'], default=1)
frame['pre_start_lineno'] = pre_start_lineno
frame['post_start_lineno'] = frame['lineno'] + 1
last_frame = glom(exc_tb, T['frames'][-1], default=None)
eid = {'is_email': False,
'clastic_version': _version.__version__,
'exc_type': exc_info.exc_type,
'exc_value': exc_info.exc_msg,
'exc_tb': exc_tb,
'last_frame': last_frame,
'exc_tb_str': str(exc_info.tb_info),
'server_time': str(datetime.datetime.now()),
'server_time_utc': str(datetime.datetime.utcnow()),
'python': {'executable': sys.executable,
'version': sys.version.replace('\n', ' '),
'path': sys.path}}
request = self.request
if request:
eid['req'] = {'path': request.path,
'full_url': request.url,
'method': request.method,
'abs_path': request.path,
'url_params': request.args,
'cookies': request.cookies,
'headers': request.headers,
'files': request.files}
ret.update(eid)
return ret
def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
# This is the CPU that the product was built for
processed_crash['cpu_arch'] = glom(
processed_crash, 'json_dump.system_info.cpu_arch', default=''
)
# NOTE(willkg): "cpu_name" is deprecated and we can remove it in July 2019
processed_crash['cpu_name'] = glom(
processed_crash, 'json_dump.system_info.cpu_arch', default=''
)
# This is the CPU info of the machine the product was running on
processed_crash['cpu_info'] = glom(
processed_crash, 'json_dump.system_info.cpu_info', default=''
)
processed_crash['cpu_count'] = glom(
processed_crash, 'json_dump.system_info.cpu_count', default=0
)
def test_scope_spec():
scope_spec = Spec(S)
assert scope_spec.glom(5, scope={'cat': 1})['cat'] == 1
cat_scope_spec = Spec(scope_spec, scope={'cat': 1})
assert 'cat' in repr(cat_scope_spec)
assert cat_scope_spec.glom(5)['cat'] == 1
# test that Spec overrides the scope for its sub-tree
assert glom(5, cat_scope_spec, scope={'cat': 2})['cat'] == 1
def test_python_native():
# 4
target = {'system': {'planets': [{'name': 'earth', 'moons': 1},
{'name': 'jupiter', 'moons': 69}]}}
output = glom(target, {'moon_count': ('system.planets', ['moons'], sum)})
assert output == {'moon_count': 70}
# 5
spec = T['system']['planets'][-1].values()
output = glom(target, spec)
assert set(output) == set(['jupiter', 69]) # for ordering reasons
with pytest.raises(glom_core.GlomError):
spec = T['system']['comets'][-1].values()
output = glom(target, spec)
def action(self, raw_crash, processed_crash, notes):
notes.append(
'Signature replaced with a JIT Crash Category, '
'was: "{}"'.format(processed_crash.get('signature', ''))
)
processed_crash['signature'] = "jit | {}".format(
glom(processed_crash, 'classifications.jit.category')
)
return True
def test_iter_str():
# check that strings are not iterable by default, one of the most
# common sources of bugs
glom_buddy = 'kurt'
with pytest.raises(UnregisteredTarget):
glom.glom(glom_buddy, {'name': [glom_buddy]})
# also check that someone can override this
glommer = Glommer()
glommer.register(str, iterate=iter)
res = glommer.glom(glom_buddy, {'name_chars_for_some_reason': [str]})
assert len(res['name_chars_for_some_reason']) == 4
# the better way, for any dissenter reading this
assert glom.glom(glom_buddy, {'name_chars': list}) == {'name_chars': ['k', 'u', 'r', 't']}
def action(self, crash_data, result):
# If this is a Java crash, then generate a Java signature
if crash_data.get('java_stack_trace'):
result.debug(self.name, 'Using JavaSignatureTool')
signature, notes, debug_notes = self.java_signature_tool.generate(
crash_data['java_stack_trace'],
delimiter=': '
)
for note in notes:
result.info(self.name, note)
for note in debug_notes:
result.debug(self.name, note)
result.set_signature(self.name, signature)
return True
result.debug(self.name, 'Using CSignatureTool')
try:
# First, we need to figure out which thread to look at. If it's a
# chrome hang (1), then use thread 0. Otherwise, use the crashing
# thread specified in the crash data.
if crash_data.get('hang_type', None) == 1:
crashing_thread = 0
else:
crashing_thread = self._get_crashing_thread(crash_data)
# If we have a thread to look at, pull the frames for that.
# Otherwise we don't have frames to use.
if crashing_thread is not None:
signature_list = self._create_frame_list(
glom(crash_data, 'threads.%d' % crashing_thread, default={}),
crash_data.get('os') == 'Windows NT'
)
else:
signature_list = []
except (KeyError, IndexError) as exc:
result.note('No crashing frames found because of %s', exc)
signature_list = []
signature, notes, debug_notes = self.c_signature_tool.generate(
signature_list,
crash_data.get('hang_type'),
crash_data.get('crashing_thread')
)
if signature_list:
result.extra['proto_signature'] = ' | '.join(signature_list)
for note in notes:
result.info(self.name, note)
for note in debug_notes:
result.debug(self.name, note)
if signature:
result.set_signature(self.name, signature)
return True
def action(self, raw_crash, processed_crash, notes):
# If this is a Java crash, then generate a Java signature
if raw_crash.get('JavaStackTrace', None):
signature, signature_notes = self.java_signature_tool.generate(
raw_crash['JavaStackTrace'],
delimiter=': '
)
processed_crash['signature'] = signature
if signature_notes:
notes.extend(signature_notes)
return True
# This isn't a Java crash, so figure out what we need and then generate a C signature
crashed_thread = self._get_crashing_thread(processed_crash)
try:
if processed_crash.get('hang_type', None) == 1:
# Force the signature to come from thread 0
signature_list = self._create_frame_list(
glom(processed_crash, 'json_dump.threads.0'),
glom(processed_crash, 'json_dump.system_info.os', default=None) == 'Windows NT'
)
elif crashed_thread is not None:
signature_list = self._create_frame_list(
glom(processed_crash, 'json_dump.threads.%d' % crashed_thread),
glom(processed_crash, 'json_dump.system_info.os', default=None) == 'Windows NT'
)
else:
signature_list = []
except (KeyError, IndexError) as exc:
notes.append('No crashing frames found because of %s' % exc)
signature_list = []
signature, signature_notes = self.c_signature_tool.generate(
signature_list,
processed_crash.get('hang_type', None),
crashed_thread,
)
processed_crash['proto_signature'] = ' | '.join(signature_list)
processed_crash['signature'] = signature
if signature_notes:
notes.extend(signature_notes)
return True
def test_beyond_access():
# 1
target = {'galaxy': {'system': {'planet': 'jupiter'}}}
spec = 'galaxy.system.planet'
output = glom(target, spec)
assert output == 'jupiter'
# 2
target = {'system': {'planets': [{'name': 'earth'}, {'name': 'jupiter'}]}}
output = glom(target, ('system.planets', ['name']))
assert output == ['earth', 'jupiter']
# 3
target = {'system': {'planets': [{'name': 'earth', 'moons': 1},
{'name': 'jupiter', 'moons': 69}]}}
spec = {'names': ('system.planets', ['name']),
'moons': ('system.planets', ['moons'])}
output = glom(target, spec)
assert output == {'names': ['earth', 'jupiter'], 'moons': [1, 69]}
def get_proc_info():
ret = {}
ret['pid'] = os.getpid()
_user_t, _sys_t = os.times()[:2]
ret['cpu_times'] = {'user_time': _user_t, 'sys_time': _sys_t}
ret['cwd'] = os.getcwdu()
ret['umask'] = os.umask(os.umask(2)) # have to set to get
ret['umask_str'] = '{0:03o}'.format(ret['umask'])
ret['owner'] = glom(globals(), Coalesce(T['getpass'].getuser(),
T['os'].getuid()),
skip_exc=Exception)
# use 0 to get current niceness, seems to return process group's nice level
unix_only_vals = glom(os, {'ppid': T.getppid(),
'pgid': T.getpgrp(),
'niceness': T.nice(0)},
skip_exc=AttributeError)
ret.update(unix_only_vals)
ret['rusage'] = get_rusage_dict()
ret['rlimit'] = get_rlimit_dict()
return ret
def get_host_info():
ret = {}
now = datetime.datetime.utcnow()
ret['hostname'] = socket.gethostname()
ret['hostfqdn'] = socket.getfqdn()
ret['uname'] = platform.uname()
ret['cpu_count'] = CPU_COUNT
ret['platform'] = platform.platform()
ret['platform_terse'] = platform.platform(terse=True)
ret['load_avgs'] = glom(os, T.getloadavg(), skip_exc=AttributeError)
ret['utc_time'] = str(now)
return ret
def test_t_picklability():
import pickle
class TargetType(object):
def __init__(self):
self.attribute = lambda: None
self.attribute.method = lambda: {'key': lambda x: x * 2}
spec = T.attribute.method()['key'](x=5)
rt_spec = pickle.loads(pickle.dumps(spec))
assert repr(spec) == repr(rt_spec)
assert glom(TargetType(), spec) == 10
s_spec = S.attribute
assert repr(s_spec) == repr(pickle.loads(pickle.dumps(s_spec)))
def convert_data(input_data, output_folder, items_selector, key_selector, spec):
with open(os.path.join(PROJECT_FOLDER, input_data)) as f:
data = yaml.load(f.read())
items_folder = os.path.join(PROJECT_FOLDER, output_folder)
if os.path.exists(items_folder):
shutil.rmtree(items_folder)
os.mkdir(items_folder)
for item in items_selector(data):
key = key_selector(item)
path = os.path.join(items_folder, f"{key}.yml")
with open(path, 'w') as f:
f.write(yaml.dump(
glom.glom(item, spec),
default_flow_style=False
))
def test_call_and_target():
class F(object):
def __init__(s, a, b, c): s.a, s.b, s.c = a, b, c
call_f = Call(F, kwargs=dict(a=T, b=T, c=T))
assert repr(call_f)
val = glom(1, call_f)
assert (val.a, val.b, val.c) == (1, 1, 1)
class F(object):
def __init__(s, a): s.a = a
val = glom({'one': F('two')}, Call(F, args=(T['one'].a,)))
assert val.a == 'two'
assert glom({'a': 1}, Call(F, kwargs=T)).a == 1
assert glom([1], Call(F, args=T)).a == 1
assert glom(F, T(T)).a == F
assert glom([F, 1], T[0](T[1]).a) == 1
assert glom([[1]], T[0][0][0][UP]) == 1
assert glom([[1]], T[0][UP][UP][UP]) == [[1]] # tops out at just T
assert list(glom({'a': 'b'}, Call(T.values))) == ['b']
with pytest.raises(TypeError, match='expected func to be a callable or T'):
Call(func=object())
return
def glom_cli(target, spec, indent, debug, inspect):
"""Command-line interface to the glom library, providing nested data
access and data restructuring with the power of Python.
"""
if debug or inspect:
stdin_open = not sys.stdin.closed
spec = Inspect(spec,
echo=inspect,
recursive=inspect,
breakpoint=inspect and stdin_open,
post_mortem=debug and stdin_open)
try:
result = glom(target, spec)
except GlomError as ge:
print('%s: %s' % (ge.__class__.__name__, ge))
return 1
if not indent:
indent = None
print(json.dumps(result, indent=indent, sort_keys=True))
return
def test_initial_integration():
class Example(object):
pass
example = Example()
subexample = Example()
subexample.name = 'good_name'
example.mapping = {'key': subexample}
val = {'a': {'b': 'c'}, # basic dictionary nesting
'example': example, # basic object
'd': {'e': ['f'], # list in dictionary
'g': 'h'},
'i': [{'j': 'k', 'l': 'm'}], # list of dictionaries
'n': 'o'}
spec = {'a': (Inspect(recursive=True), 'a', 'b'), # inspect just prints here
'name': 'example.mapping.key.name', # test object access
'e': 'd.e', # d.e[0] or d.e: (callable to fetch 0)
'i': ('i', [{'j': 'j'}]), # TODO: support True for cases when the value should simply be mapped into the field name?
'n': ('n', lambda n: n.upper()),
'p': Coalesce('xxx',
'yyy',
default='zzz')}
ret = glom(val, spec)
print('in: ', val)
print('got:', ret)
expected = {'a': 'c',
'name': 'good_name',
'e': ['f'],
'i': [{'j': 'k'}],
'n': 'O',
'p': 'zzz'}
print('exp:', expected)
assert ret == expected
async def get_emails_imap(
self,
username,
password,
imap_server,
foldername,
amount,
unread,
fields,
include_raw_body,
include_attachment_data,
upload_email_shuffle,
upload_attachments_shuffle,
):
def path_to_dict(path, value=None):
def pack(parts):
return (
{parts[0]: pack(parts[1:]) if len(parts) > 1 else value}
if len(parts) > 1
else {parts[0]: value}
)
return pack(path.split("."))
def merge(d1, d2):
for k in d2:
if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], dict):
merge(d1[k], d2[k])
else:
d1[k] = d2[k]
if type(amount) == str:
try:
amount = int(amount)
except ValueError:
return "Amount needs to be a number, not %s" % amount
try:
email = imaplib.IMAP4_SSL(imap_server)
except ConnectionRefusedError as error:
try:
email = imaplib.IMAP4(imap_server)
email.starttls()
except socket.gaierror as error:
return "Can't connect to IMAP server %s: %s" % (imap_server, error)
except socket.gaierror as error:
return "Can't connect to IMAP server %s: %s" % (imap_server, error)
try:
email.login(username, password)
except imaplib.IMAP4.error as error:
return "Failed to log into %s: %s" % (username, error)
email.select(foldername)
unread = True if unread.lower().strip() == "true" else False
try:
# IMAP search queries, e.g. "seen" or "read"
# https://www.rebex.net/secure-mail.net/features/imap-search.aspx
mode = "(UNSEEN)" if unread else "ALL"
thistype, data = email.search(None, mode)
except imaplib.IMAP4.error as error:
return "Couldn't find folder %s." % (foldername)
email_ids = data[0]
id_list = email_ids.split()
if id_list == None:
return "Couldn't retrieve email. Data: %s" % data
try:
print("LIST: ", len(id_list))
except TypeError:
return "Error getting email. Data: %s" % data
include_raw_body = True if include_raw_body.lower().strip() == "true" else False
include_attachment_data = (
True if include_attachment_data.lower().strip() == "true" else False
)
upload_email_shuffle = (
True if upload_email_shuffle.lower().strip() == "true" else False
)
upload_attachments_shuffle = (
True if upload_attachments_shuffle.lower().strip() == "true" else False
)
# Convert <amount> of mails in json
emails = []
ep = eml_parser.EmlParser(
include_attachment_data=include_attachment_data
or upload_attachments_shuffle,
include_raw_body=include_raw_body,
)
try:
for i in range(len(id_list) - 1, len(id_list) - amount - 1, -1):
resp, data = email.fetch(id_list[i], "(RFC822)")
error = None
if resp != "OK":
print("Failed getting %s" % id_list[i])
continue
if data == None:
continue
output_dict = {}
parsed_eml = ep.decode_email_bytes(data[0][1])
if fields and fields.strip() != "":
for field in fields.split(","):
field = field.strip()
merge(
output_dict,
path_to_dict(
field,
glom(parsed_eml, field, default=None),
),
)
else:
output_dict = parsed_eml
fields = "ALL"
# Add message-id as top returned field
output_dict["message-id"] = parsed_eml["header"]["header"][
"message-id"
][0]
if upload_email_shuffle:
email_up = [{"filename": "email.msg", "data": data[0][1]}]
email_id = self.set_files(email_up)
output_dict["email_uid"] = email_id[0]
if upload_attachments_shuffle:
atts_up = [
{
"filename": x["filename"],
"data": base64.b64decode(x["raw"]),
}
for x in parsed_eml["attachment"]
]
atts_ids = self.set_files(atts_up)
output_dict["attachments_uids"] = atts_ids
emails.append(output_dict)
except Exception as err:
return "Error during email processing: {}".format(err)
return json.dumps(emails, default=default)
def test_add_code():
b = dp.templates.add_code(md_block, "print(1)")
assert isinstance(b, dp.Select)
assert glom(b, ("blocks", ["_tag"])) == ["Text", "Code"]
assert "print(1)" in element_to_str(b)
async def get_emails(
self,
username,
password,
server,
build,
account,
verifyssl,
foldername,
amount,
unread,
fields,
include_raw_body,
include_attachment_data,
):
def path_to_dict(path, value=None):
def pack(parts):
return ({
parts[0]: pack(parts[1:]) if len(parts) > 1 else value
} if len(parts) > 1 else {
parts[0]: value
})
return pack(path.split("."))
def merge(d1, d2):
for k in d2:
if k in d1 and isinstance(d1[k], dict) and isinstance(
d2[k], dict):
merge(d1[k], d2[k])
else:
d1[k] = d2[k]
# Authenticate
auth = await self.authenticate(username, password, server, build,
account, verifyssl)
if auth["error"]:
return auth["error"]
account = auth["account"]
# Parse email folder
folder = await self.parse_folder(account, foldername)
if folder["error"]:
return folder["error"]
folder = folder["folder"]
if type(amount) == str:
try:
amount = int(amount)
except ValueError:
return {
"account": None,
"error": "Amount needs to be a number, not %s" % amount,
}
# Get input from gui
unread = True if unread.lower().strip() == "true" else False
include_raw_body = True if include_raw_body.lower().strip(
) == "true" else False
include_attachment_data = (True
if include_attachment_data.lower().strip()
== "true" else False)
# Convert <amount> of mails in json
emails = []
ep = eml_parser.EmlParser(
include_attachment_data=include_attachment_data,
include_raw_body=include_raw_body,
)
try:
for email in folder.filter(is_read=not unread).order_by(
"-datetime_received")[:amount]:
output_dict = {}
parsed_eml = ep.decode_email_bytes(email.mime_content)
if fields and fields.strip() != "":
for field in fields.split(","):
field = field.strip()
merge(
output_dict,
path_to_dict(
field,
glom(parsed_eml, field, default=None),
),
)
else:
output_dict = parsed_eml
emails.append(output_dict)
except Exception as err:
return "Error during email processing: {}".format(err)
return json.dumps(emails, default=default)
def deserialize(o):
return glom(o, mk_inv_spec(o))
def test_good_error():
target = {'data': [0, 1, 2]}
with pytest.raises(GlomError):
glom(target, ('data.3'))
def get_fiats():
url = f"{cmc_url}/fiat/map"
parameters = {"sort": "name"}
with api_request(cmc_session, url, parameters) as data:
return str(glom(data, ("data", ["symbol"]))).replace("'", '"')
def test_nested_dict():
assert glom({1: 2}, Match({A.t: S.t})) == {1: 1}
def get_replicas(service: Service) -> int:
if not service.spec.mode or "Global" in service.spec.mode:
return 1
return glom(service.spec.mode, "Replicated.Replicas", default=0)
def get_variable_from_projectrc(variable: str) -> str:
projectrc = configuration.load_yaml_file(file=PROJECTRC, is_optional=False)
return glom(projectrc,
f"project_configuration.variables.env.{variable}",
default="")
def glom_event(self, spec: TType) -> Any:
return glom(self.event, spec, default=None)
def latexToPdf(compilerName, directory, main_resource, options={}):
bibtexCommand = glom.glom(options,
"bibliography.command",
default="bibtex")
if bibtexCommand not in AVAILABLE_BIBLIOGRAPHY_COMMANDS:
raise ValueError("Invalid bibtex command")
if compilerName not in AVAILABLE_LATEX_COMPILERS:
raise ValueError("Invalid compiler")
# TODO Choose appropriate options following the compiler.
# Copy files to tmp directory.
# Should already be an absolute path (in our usage), but just to be sure.
directory = os.path.abspath(directory)
# TODO Uses workspace.filesystem module to these get paths.
input_path = "{}/{}".format(directory, main_resource["build_path"])
output_path = "{}/output.pdf".format(directory)
log_dir = "{}/latex.out".format(directory)
logger.info("Compiling %s from %s", main_resource["build_path"], directory)
# Use https://github.com/aclements/latexrun
# to manage multiple runs of Latex compiler for us.
# (Cross-references, page numbers, etc.)
# TODO Put on pip
# TODO Fix this lame subprocessing with parh orgy.
if compilerName in ["context"]:
# TODO Patch latexrun to support context?
# Here do not support multi runs or bibtex/biber commands.
# --> do not pass nonstopmode
# --> parse jobName / output files from Context output
# Or use another more universal Latex runner?
# https://mg.readthedocs.io/latexmk.html
# Alternative: support many runners.
# Arara https://github.com/cereda/arara
# https://github.com/wtsnjp/llmk
command = [
compilerName,
input_path,
]
output_path = "{}/{}".format(
directory, main_resource["build_path"].replace(".tex", ".pdf"))
log_dir = "{}/{}".format(
directory, main_resource["build_path"].replace(".tex", ".log"))
else:
command = [
"python",
"{}/latexonhttp/latexrun.py".format(os.getcwd()),
"--latex-cmd={}".format(compilerName),
"-O={}".format(log_dir),
"-o={}".format(output_path),
# Return all logs.
"-W=all"
# TODO Add -halt-on-error --interaction=nonstopmode
# TODO Let user choose DVI (or other supported) output.
'--latex-args="--output-format=pdf"',
"--bibtex-cmd={}".format(bibtexCommand),
input_path,
]
logger.debug(command)
commandOutput = run_command(directory, command)
if commandOutput["return_code"] == 0 and compilerName in [
"platex", "uplatex"
]:
# We need a dvipdfmx pass.
# https://tex.stackexchange.com/questions/295414/what-is-uptex-uplatex
# TODO Use ptex2pdf?
# https://github.com/texjporg/ptex2pdf
command = [
"dvipdfmx",
"{}/{}".format(log_dir,
main_resource["build_path"].replace(".tex",
".dvi")),
]
output_path = "{}/{}".format(
log_dir, main_resource["build_path"].replace(".tex", ".pdf"))
logger.debug(command)
run_command(log_dir, command)
# TODO Check for compilation errors.
# commandOutput['return_code'] is not 0
# Return both generated PDF and compile logs.
# TODO Uses workspace.filesystem module read file back?
pdf = None
if os.path.isfile(output_path):
with open(output_path, "rb") as f:
pdf = f.read()
# TODO Returns paths instead of data?
return {
"pdf": pdf,
"output_path": main_resource["output_path"],
"logs": commandOutput["stdout"],
}
def get_attribute(self, value: str) -> Any:
"""Get attribute from configurations."""
return glom(self._config, value, default="")
def isnode(path):
return _is_node(path, path[-1], glom(conf, ".".join(path)))
async def test_client_part_of_app():
app = GithubApp(
user_agent=TEST_USER_AGENT,
app_id=TEST_APP_ID,
private_key=TEST_PRIVATE_KEY,
webhook_secret=TEST_WEBHOOK_SECRET,
)
assert app.app_client.app is app
# github actually won't let any app client access the /rate_limit
# endpoint, because app credentials are so locked down. But we can look up
# information about ourself!
data = await app.app_client.getitem(
"/app", accept=accept_format(version="machine-man-preview"))
assert glom(data, "name") == "snekomatic-test"
# We can get an installation token
token = await app.token_for(TEST_INSTALLATION_ID)
# They're cached
token2 = await app.token_for(TEST_INSTALLATION_ID)
assert token == token2
# And the client works too:
i_client = app.client_for(TEST_INSTALLATION_ID)
assert i_client.app is app
assert i_client.installation_id == TEST_INSTALLATION_ID
data = await i_client.getitem("/rate_limit")
assert "rate" in data
# Now we'll cheat and trick the app into thinking that the token is
# expiring, and check that the client automatically renews it.
soon = pendulum.now().add(seconds=10)
app._installation_tokens[TEST_INSTALLATION_ID].expires_at = soon
# The client still works...
i_client = app.client_for(TEST_INSTALLATION_ID)
data = await i_client.getitem("/rate_limit")
assert "rate" in data
# ...but the token has changed.
assert token != await app.token_for(TEST_INSTALLATION_ID)
# And let's do that again, but this time we'll have two tasks try to fetch
# the token at the same time.
soon = pendulum.now().add(seconds=10)
app._installation_tokens[TEST_INSTALLATION_ID].expires_at = soon
tokens = []
async def get_token():
tokens.append(await app.token_for(TEST_INSTALLATION_ID))
async with trio.open_nursery() as nursery:
nursery.start_soon(get_token)
nursery.start_soon(get_token)
# They both end up with the same token, demonstrating that they didn't do
# two independent fetches
assert len(tokens) == 2
assert len(set(tokens)) == 1
def test_enum_input_value_resolution(mocker):
# Add test example flags, switches and samples
type_defs = """
type Query
"""
schema = make_executable_schema(config.type_defs + [type_defs],
config.all_app_types)
for i in range(2):
baker.make(get_waffle_flag_model(),
name="flag%s" % i,
everyone=bool(i % 2),
note="a flag")
for i in range(2):
baker.make(Switch,
name="switch%s" % i,
active=bool(i % 2),
note="a switch")
for i in range(2):
baker.make(Sample, name="sample%s" % i, percent="100", note="a sample")
result = graphql_sync(
schema,
"""
query {
waffle {
flags {
...BaseWaffleItem
}
switches {
...BaseWaffleItem
}
samples {
...BaseWaffleItem
}
all {
...BaseWaffleItem
}
flagDefault
switchDefault
sampleDefault
}
}
fragment BaseWaffleItem on WaffleItem {
__typename
id
name
active
note
}
""",
)
assert result.errors is None
waffle = result.data.get("waffle")
assert glom(waffle, "flags.0.active") == False
assert glom(waffle, "flags.1.active") == True
assert glom(waffle, "switches.0.active") == False
assert glom(waffle, "switches.1.active") == True
assert glom(waffle, "samples.0.active") == True
assert glom(waffle, "samples.1.active") == True
assert glom(waffle, "flags.0.name") == "flag0"
assert glom(waffle, "flags.1.name") == "flag1"
assert glom(waffle, "all.0.name") == "switch0"
assert glom(waffle, "all.0.__typename") == "WaffleSwitch"
assert glom(waffle, "all.5.name") == "flag1"
assert glom(waffle, "all.5.__typename") == "WaffleFlag"
assert glom(waffle, "flagDefault") == False
assert glom(waffle, "switchDefault") == False
assert glom(waffle, "sampleDefault") == True
async def get_emails(
self,
username,
password,
server,
build,
account,
verifyssl,
foldername,
amount,
unread,
fields,
include_raw_body,
include_attachment_data,
upload_email_shuffle,
upload_attachments_shuffle,
):
def path_to_dict(path, value=None):
def pack(parts):
return ({
parts[0]: pack(parts[1:]) if len(parts) > 1 else value
} if len(parts) > 1 else {
parts[0]: value
})
return pack(path.split("."))
def merge(d1, d2):
for k in d2:
if k in d1 and isinstance(d1[k], dict) and isinstance(
d2[k], dict):
merge(d1[k], d2[k])
else:
d1[k] = d2[k]
# Authenticate
auth = await self.authenticate(username, password, server, build,
account, verifyssl)
if auth["error"]:
return auth["error"]
account = auth["account"]
# Parse email folder
folder = await self.parse_folder(account, foldername)
if folder["error"]:
return folder["error"]
folder = folder["folder"]
if type(amount) == str:
try:
amount = int(amount)
except ValueError:
return {
"account": None,
"error": "Amount needs to be a number, not %s" % amount,
}
# Get input from gui
unread = True if unread.lower().strip() == "true" else False
include_raw_body = True if include_raw_body.lower().strip(
) == "true" else False
include_attachment_data = (True
if include_attachment_data.lower().strip()
== "true" else False)
upload_email_shuffle = (True if upload_email_shuffle.lower().strip()
== "true" else False)
upload_attachments_shuffle = (
True
if upload_attachments_shuffle.lower().strip() == "true" else False)
# Convert <amount> of mails in json
emails = []
ep = eml_parser.EmlParser(
include_attachment_data=include_attachment_data,
include_raw_body=include_raw_body,
)
try:
for email in folder.filter(is_read=not unread).order_by(
"-datetime_received")[:amount]:
output_dict = {}
parsed_eml = ep.decode_email_bytes(email.mime_content)
if fields and fields.strip() != "":
for field in fields.split(","):
field = field.strip()
merge(
output_dict,
path_to_dict(
field,
glom(parsed_eml, field, default=None),
),
)
else:
output_dict = parsed_eml
# Add message-id as top returned field
output_dict["message_id"] = parsed_eml["header"]["header"][
"message-id"][0]
if upload_email_shuffle:
email_up = [{
"filename": "email.msg",
"data": email.mime_content
}]
email_id = self.set_files(email_up)
output_dict["email_fileid"] = email_id[0]
if upload_attachments_shuffle:
atts_up = [{
"filename": attachment.name,
"data": attachment.content
} for attachment in email.attachments
if type(attachment) == FileAttachment]
atts_ids = self.set_files(atts_up)
output_dict["attachments_uids"] = atts_ids
try:
if len(output_dict["body"]) > 1:
output_dict["body"][0]["raw_body"] = output_dict[
"body"][1]["content"]
if len(output_dict["body"]) > 0:
output_dict["body"] = output_dict["body"][0]
except KeyError as e:
print("KeyError: %s" % e)
emails.append(output_dict)
except Exception as err:
return "Error during email processing: {}".format(err)
return json.dumps(emails, default=default)
def get_cryptos(limit=100):
url = f"{cmc_url}/cryptocurrency/map"
parameters = {"limit": str(limit), "sort": "cmc_rank"}
with api_request(cmc_session, url, parameters) as data:
return str(glom(data, ("data", ["symbol"]))).replace("'", '"')
def test_line_trace():
stacklifier = ([{'data': S}], )
scope = glom([1], stacklifier)[0]['data'][ROOT][LAST_CHILD_SCOPE]
fmtd_stack = format_oneline_trace(scope)
assert fmtd_stack == '/tuple!list/list<0>/dict!int/S'
def convert(fsym, tsym):
url = f"{cryptoc_url}/data/price"
parameters = {"fsym": fsym, "tsyms": tsym}
with api_request(cryptoc_session, url, parameters) as data:
return str(glom(data, tsym))
def _uses_another_glom():
try:
ret = glom(['Nested'], {'internal': ['val']})
except Exception as exc:
raise
def test_check_ported_tests():
"""
Tests ported from Check() to make sure all the functionality has an analogue.
"""
target = [{'id': 0}, {'id': 1}, {'id': 2}]
# check that skipping non-passing values works
assert glom(target, [Coalesce(M(T['id']) == 0, default=SKIP)]) == [{'id': 0}]
# TODO: should M(subspec, default='') work? I lean no.
# NB: this is not a very idiomatic use of Match, just brought over for Check reasons
assert glom(target, [Match({'id': And(int, M == 1)}, default=SKIP)]) == [{'id': 1}]
assert glom(target, [Match({'id': And(int, M <= 1)}, default=STOP)]) == [{'id': 0}, {'id': 1}]
# check that stopping chain execution on non-passing values works
spec = (Or(Match(len), Val(STOP)), T[0])
assert glom('hello', spec, glom_debug=True) == 'h'
assert glom('', spec) == '' # would fail with IndexError if STOP didn't work
target = [1, u'a']
assert glom(target, [Match(unicode, default=SKIP)]) == ['a']
assert glom(target, Match([Or(unicode, int)])) == [1, 'a']
target = ['1']
assert glom(target, [(M(T), int)]) == [1]
assert glom(target, M(T)) == ['1']
failing_checks = [({'a': {'b': 1}}, {'a': ('a', 'b', Match(str))},
'''expected type str, not int'''), # TODO: bbrepr at least, maybe include path like Check did
({'a': {'b': 1}}, {'a': ('a', Match({'b': str}))},
'''expected type str, not int'''), # TODO: include subspec path ('b')
(1, Match(Or(unicode, bool))),
(1, Match(unicode)),
(1, Match(0)),
(1, Match(Or(0, 2))),
('-3.14', Match(lambda x: int(x) > 0)),
# ('-3.14', M(lambda x: int(x) > 0)),
# TODO: M doesn't behave quite like Match because it's mode-free
]
for fc in failing_checks:
if len(fc) == 2:
target, check = fc
msg = None
else:
target, check, msg = fc
with pytest.raises(MatchError) as exc_info:
glom(target, check)
if msg is not None:
actual_msg = str(exc_info.value)
assert actual_msg.find(msg) != -1
assert repr(exc_info.value)
return
def test_defaults():
assert glom(1, Match(2, default=3)) == 3
assert glom(1, Or(M == 2, default=3)) == 3
assert glom(1, And(M == 2, default=3)) == 3
def test_clamp():
assert glom(range(10), [(M < 7) | Val(7)]) == [0, 1, 2, 3, 4, 5, 6, 7, 7, 7]
assert glom(range(10), [(M < 7) | Val(SKIP)]) == [0, 1, 2, 3, 4, 5, 6]
def get(self, key, default: Any = None) -> Any:
return glom(self, key, default=default)
def test_list_path_access():
assert glom(list(range(10)), Path(1)) == 1
def test_path():
_obj = object()
target = {'a': {'b.b': [None, {_obj: [None, None, 'd']}]}}
assert glom(target, Path('a', 'b.b', 1, _obj, -1)) == 'd'
from glom import glom
from pandas.io.json import json_normalize
import numpy as np
import urllib3
from urllib3 import request
remote = True
input_file = '.\data\election_2020\pa_president.json'
output_file = '.\\results\election_2020\pa_president_computed.csv'
rest_url = 'https://static01.nyt.com/elections-assets/2020/data/api/2020-11-03/race-page/pennsylvania/president.json'
if remote:
http = urllib3.PoolManager()
res = http.request('GET', rest_url)
elect_df_from_rest = pd.read_json(res.data)
elect_df = pd.DataFrame(glom(elect_df_from_rest,
'data.races.0.timeseries'))
else:
elect_df_from_csv = pd.read_json(input_file)
elect_df = pd.DataFrame(glom(elect_df_from_csv, 'data.races.0.timeseries'))
elect_df = elect_df.join(json_normalize(elect_df.vote_shares))
#add the third party share of votes with formula (1 - (trumpd + bidenj))
elect_df['third_party'] = 1 - elect_df['trumpd'] - elect_df['bidenj']
elect_df.loc[0, 'third_party'] = 0.0
#compute the votes at each timestamp
elect_df['trumpd_votes'] = (elect_df['votes'] * elect_df['trumpd']).apply(
np.ceil)
elect_df['bidenj_votes'] = (elect_df['votes'] * elect_df['bidenj']).apply(
np.ceil)
elect_df['third_party_votes'] = (elect_df['votes'] *
def get_workspace_id(region, org_name, workspace_name):
"""Return the workspace id."""
response = get_workspace(region, org_name, workspace_name)
return glom(response, 'data.id', default=False)
def serialize(o):
return glom(o, spec)
def test_t_dict_key():
target = {'a': 'A'}
assert glom(target, {T['a']: 'a'}) == {'A': 'A'}