def merge_params_with_request_args_while_deep_merging_dict_struct(kwargs):
params_from_request_args = _serializable_params(request.args)
merged_params = merge(kwargs, params_from_request_args)
if kwargs.get('dict_struct') and params_from_request_args.get('dict_struct'):
merged_params['dict_struct'] = merge(
kwargs['dict_struct'], params_from_request_args.get('dict_struct'))
return merged_params
def merge_params_with_request_args_while_deep_merging_dict_struct(kwargs):
params_from_request_args = _serializable_params(request.args)
merged_params = merge(kwargs, params_from_request_args)
if kwargs.get('dict_struct') and params_from_request_args.get('dict_struct'):
merged_params['dict_struct'] = merge(
kwargs['dict_struct'], params_from_request_args.get('dict_struct'))
return merged_params
def convert_timestamp_indexed_df_to_dt(
df, dt_id=None, types_of_fields=None,
timestamp_col_name_format="%b %Y"):
df = df.replace(np.nan, 0, regex=True)
field_types = {
field: field_type
for field_type, fields in types_of_fields.items()
for field in fields
} if types_of_fields else {}
if isinstance(df.index, pd.DatetimeIndex):
dt_columns = [
{"name": "Period", "id": "period"}] + [
{
"name": col_name.replace("_", " ").capitalize(),
"id": col_name
} for col_name in df.columns
]
dt_rows = [
merge({
k: format_value(v, field_types.get(k))
for k, v in row.items()
}, {
"period": idx.strftime(
timestamp_col_name_format),
}) for idx, row in zip(
df.index, df.to_dict('records'))
]
else:
print("in else block")
dt_columns = [
{"name": "Metric", "id": "metric"}] + [
{
"name": i.strftime(timestamp_col_name_format),
"id": i.strftime(timestamp_col_name_format)
} for i in df.columns
]
dt_rows = [merge({
k.strftime(
timestamp_col_name_format): format_value(
v, field_types.get(idx))
for k, v in row.items()}, {
"metric": idx.replace("_", " ").capitalize()}
) for idx, row in zip(df.index, df.to_dict('records'))]
return dash_table.DataTable(
id=dt_id or 'table',
style_data={'whiteSpace': 'normal'},
css=[{
'selector': '.dash-cell div.dash-cell-value',
'rule': 'display: inline; white-space: inherit;\
overflow: inherit; text-overflow: inherit;'
}],
columns=dt_columns,
data=dt_rows,
)
def _prepare_data_for_saving(cls, kwargs):
"""Returns a preprocessed dictionary of parameters.
Use this to filter the kwargs passed to `new`, `create`,
`build` methods.
Args:
**kwargs: a dictionary of parameters
"""
# kwargs.pop('csrf_token', None)
for attr, val in kwargs.items():
if cls.is_the_primary_key(
attr) and cls._prevent_primary_key_initialization_:
del kwargs[attr]
continue
if val == "":
# Making an assumption that there is no good usecase
# for setting an empty string. This will help prevent
# cases where empty string is sent because of client
# not clearing form fields to null
kwargs[attr] = None
continue
if attr in class_mapper(
cls).relationships and attr not in cls._no_overwrite_:
rel = class_mapper(cls).relationships[attr]
if rel.uselist:
if isinstance(val, list):
if all(isinstance(v, dict) for v in val):
rel_cls = cls.mapped_rel_class(attr)
kwargs[attr] = rel_cls.update_or_new_all(
list_of_kwargs=val,
keys=[rel_cls.primary_key_name()])
elif isinstance(val, dict):
rel_cls = cls.mapped_rel_class(attr)
mapping_col = rel.collection_class().keyfunc.name
list_of_kwargs = [
merge(v, {mapping_col: k}) for k, v in val.items()
]
kwargs[attr] = {
getattr(obj, mapping_col): obj
for obj in rel_cls.update_or_new_all(
list_of_kwargs=list_of_kwargs,
keys=[rel_cls.primary_key_name()])
}
elif isinstance(val, dict):
rel_cls = cls.mapped_rel_class(attr)
kwargs[attr] = rel_cls.update_or_new(
**merge(val, {'keys': [rel_cls.primary_key_name()]}))
return kwargs
def jsoned(struct, wrap=True, meta=None, struct_key='result'):
""" Provides a json dump of the struct
Args:
struct: The data to dump
wrap (bool, optional): Specify whether to wrap the
struct in an enclosing dict
struct_key (str, optional): The string key which will
contain the struct in the result dict
meta (dict, optional): An optional dictonary to merge
with the output dictionary.
Examples:
>>> jsoned([3,4,5])
... '{"status": "success", "result": [3, 4, 5]}'
>>> jsoned([3,4,5], wrap=False)
... '[3, 4, 5]'
"""
if wrap:
output = {'status': 'success', struct_key: struct}
if meta:
output = merge(output, meta)
return _json.dumps(output,
default=json_encoder)
else:
return _json.dumps(struct,
default=json_encoder)
def serializable_list(olist,
attrs_to_serialize=None,
rels_to_expand=None,
group_listrels_by=None,
rels_to_serialize=None,
key_modifications=None,
groupby=None,
keyvals_to_merge=None):
if groupby:
return deep_group(olist,
keys=groupby,
serializer='todict',
serializer_kwargs={
'rels_to_serialize': rels_to_serialize,
'rels_to_expand': rels_to_expand,
'attrs_to_serialize': attrs_to_serialize,
'group_listrels_by': group_listrels_by,
'key_modifications': key_modifications
})
else:
result_list = map(
lambda o: serialized_obj(o,
attrs_to_serialize=attrs_to_serialize,
rels_to_expand=rels_to_expand,
group_listrels_by=group_listrels_by,
rels_to_serialize=rels_to_serialize,
key_modifications=key_modifications),
olist)
if keyvals_to_merge:
result_list = [
merge(obj_dict, kvdict)
for obj_dict, kvdict in zip(result_list, keyvals_to_merge)
]
return result_list
def params_for_serialization(attrs_to_serialize=None,
rels_to_expand=None,
rels_to_serialize=None,
group_listrels_by=None,
dict_struct=None,
preserve_order=None,
groupby=None,
check_groupby=False):
params = _serializable_params(request.args, check_groupby=check_groupby)
if attrs_to_serialize is not None and 'attrs_to_serialize' not in params:
params['attrs_to_serialize'] = attrs_to_serialize
if rels_to_expand is not None and 'rels_to_expand' not in params:
params['rels_to_expand'] = rels_to_expand
if rels_to_serialize is not None and 'rels_to_serialize' not in params:
params['rels_to_serialize'] = rels_to_serialize
if group_listrels_by is not None and 'group_listrels_by' not in params:
params['group_listrels_by'] = group_listrels_by
if preserve_order is not None and 'preserve_order' not in params:
params['preserve_order'] = preserve_order
if groupby is not None and 'groupby' not in params:
params['groupby'] = groupby
if dict_struct is not None:
if 'dict_struct' not in params:
params['dict_struct'] = dict_struct
else:
params['dict_struct'] = merge(dict_struct, params['dict_struct'])
return params
def jsoned(struct, wrap=True, meta=None, struct_key='result'):
""" Provides a json dump of the struct
Args:
struct: The data to dump
wrap (bool, optional): Specify whether to wrap the
struct in an enclosing dict
struct_key (str, optional): The string key which will
contain the struct in the result dict
meta (dict, optional): An optional dictonary to merge
with the output dictionary.
Examples:
>>> jsoned([3,4,5])
... '{"status": "success", "result": [3, 4, 5]}'
>>> jsoned([3,4,5], wrap=False)
... '[3, 4, 5]'
"""
if wrap:
output = {'status': 'success', struct_key: struct}
if meta:
output = merge(output, meta)
return _json.dumps(output, default=json_encoder)
else:
return _json.dumps(struct, default=json_encoder)
def template_response_from_obj(obj,
template,
keys_to_be_replaced={},
merge_keyvals={}):
for k in keys_to_be_replaced:
if k in obj:
obj[keys_to_be_replaced[k]] = obj.pop(k)
return render_template(template, **merge(obj, merge_keyvals))
def upload(self, url, file_key, file_path, **kwargs):
buffered = kwargs.pop('buffered', True)
content_type = kwargs.pop('content_type', 'multipart/form-data')
kwargs['data'] = merge(
kwargs['data'],
{file_key: (open(file_path, 'rb'), file_path)})
return self.post(url, buffered=buffered, content_type=content_type,
**kwargs)
def upload(self, url, file_key, file_path, **kwargs):
buffered = kwargs.pop('buffered', True)
content_type = kwargs.pop('content_type', 'multipart/form-data')
kwargs['data'] = merge(
kwargs['data'],
{file_key: (open(file_path, 'rb'), file_path)})
return self.post(url, buffered=buffered, content_type=content_type,
**kwargs)
def jsoned(struct, wrap=True, meta=None):
if wrap:
output = {'status': 'success', 'result': struct}
if meta:
output = merge(output, meta)
return _json.dumps(output, default=_json_encoder)
else:
return _json.dumps(struct, default=_json_encoder)
def serializable_list(olist,
attrs_to_serialize=None,
rels_to_expand=None,
group_listrels_by=None,
rels_to_serialize=None,
key_modifications=None,
groupby=None,
keyvals_to_merge=None):
"""
Converts a list of model instances to a list of dictionaries
using their `todict` method.
Args:
olist (list): The list of instances to convert
attrs_to_serialize (list, optional): To be passed as an argument
to the `todict` method
rels_to_expand (list, optional): To be passed as an argument
to the `todict` method
group_listrels_by (dict, optional): To be passed as an argument
to the `todict` method
rels_to_serialize (list, optional): To be passed as an argument
to the `todict` method
key_modifications (dict, optional): To be passed as an argument
to the `todict` method
groupby (list, optional): An optional list of keys based on which
the result list will be hierarchially grouped ( and converted
into a dict)
keyvals_to_merge (list of dicts, optional): A list of parameters
to be merged with each dict of the output list
"""
if groupby:
return deep_group(olist,
keys=groupby,
serializer='todict',
serializer_kwargs={
'rels_to_serialize': rels_to_serialize,
'rels_to_expand': rels_to_expand,
'attrs_to_serialize': attrs_to_serialize,
'group_listrels_by': group_listrels_by,
'key_modifications': key_modifications
})
else:
result_list = map(
lambda o: serialized_obj(o,
attrs_to_serialize=attrs_to_serialize,
rels_to_expand=rels_to_expand,
group_listrels_by=group_listrels_by,
rels_to_serialize=rels_to_serialize,
key_modifications=key_modifications),
olist)
if keyvals_to_merge:
result_list = [
merge(obj_dict, kvdict)
for obj_dict, kvdict in zip(result_list, keyvals_to_merge)
]
return result_list
def convert_result_to_response_structure(result,
meta={},
attrs_to_serialize=None,
rels_to_expand=None,
rels_to_serialize=None,
group_listrels_by=None,
dict_struct=None,
preserve_order=None,
groupby=None):
params_to_be_serialized = params_for_serialization(
attrs_to_serialize=attrs_to_serialize,
rels_to_expand=rels_to_expand,
rels_to_serialize=rels_to_serialize,
group_listrels_by=group_listrels_by,
dict_struct=dict_struct,
preserve_order=preserve_order,
groupby=groupby,
check_groupby=True)
if isinstance(result, Pagination):
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', PER_PAGE_ITEMS_COUNT))
if result.total != 0 and int(page) > result.pages:
return {
"status": "failure",
"error": "PAGE_NOT_FOUND",
"total_pages": result.pages
}
pages_meta = {
'total_pages': result.pages,
'total_items': result.total,
'page': page,
'per_page': per_page,
'curr_page_first_item_index': (page - 1) * per_page + 1,
'curr_page_last_item_index': min(page * per_page, result.total)
}
if isinstance(meta, dict) and len(meta.keys()) > 0:
pages_meta = merge(pages_meta, meta)
return structured(serializable_list(result.items,
**params_to_be_serialized),
meta=pages_meta)
if isinstance(meta, dict) and len(meta.keys()) > 0:
kwargs = merge(params_to_be_serialized, {'meta': meta})
else:
kwargs = params_to_be_serialized
return as_dict_list(result, **kwargs)
def structured(struct, wrap=True, meta=None, struct_key='result', pre_render_callback=None):
output = struct
if wrap:
output = {'status': 'success', struct_key: struct}
if meta:
output = merge(output, meta)
if pre_render_callback and callable(pre_render_callback):
output = pre_render_callback(output)
return output
def structured(struct, wrap=True, meta=None, struct_key=None, pre_render_callback=None):
if struct_key is None:
struct_key = "result"
output = struct
if wrap:
output = {'status': 'success', struct_key: struct}
if meta:
output = merge(output, meta)
if pre_render_callback and callable(pre_render_callback):
output = pre_render_callback(output)
return output
def _preprocess_params(cls, kwargs):
"""Returns a preprocessed dictionary of parameters.
Use this to filter the kwargs passed to `new`, `create`,
`build` methods.
Args:
**kwargs: a dictionary of parameters
"""
# kwargs.pop('csrf_token', None)
for attr, val in kwargs.items():
if cls.is_the_primary_key(attr) and cls._prevent_primary_key_initialization_:
del kwargs[attr]
continue
if val == "":
# Making an assumption that there is no good usecase
# for setting an empty string. This will help prevent
# cases where empty string is sent because of client
# not clearing form fields to null
kwargs[attr] = None
continue
if attr in class_mapper(cls).relationships and attr not in cls._no_overwrite_:
rel = class_mapper(cls).relationships[attr]
if rel.uselist:
if isinstance(val, list):
if all(isinstance(v, dict) for v in val):
rel_cls = cls.mapped_rel_class(attr)
kwargs[attr] = rel_cls.update_or_new_all(
list_of_kwargs=val, keys=[rel_cls.primary_key_name()])
elif isinstance(val, dict):
rel_cls = cls.mapped_rel_class(attr)
mapping_col = rel.collection_class().keyfunc.name
list_of_kwargs = [merge(v, {mapping_col: k}) for k, v in val.items()]
kwargs[attr] = {getattr(obj, mapping_col): obj for obj in rel_cls.update_or_new_all(
list_of_kwargs=list_of_kwargs, keys=[rel_cls.primary_key_name()])}
elif isinstance(val, dict):
rel_cls = cls.mapped_rel_class(attr)
kwargs[attr] = rel_cls.update_or_new(
**merge(val, {'keys': [rel_cls.primary_key_name()]}))
return kwargs
def serializable_list(
olist, attrs_to_serialize=None, rels_to_expand=None,
group_listrels_by=None, rels_to_serialize=None,
key_modifications=None, groupby=None, keyvals_to_merge=None):
"""
Converts a list of model instances to a list of dictionaries
using their `todict` method.
Args:
olist (list): The list of instances to convert
attrs_to_serialize (list, optional): To be passed as an argument
to the `todict` method
rels_to_expand (list, optional): To be passed as an argument
to the `todict` method
group_listrels_by (dict, optional): To be passed as an argument
to the `todict` method
rels_to_serialize (list, optional): To be passed as an argument
to the `todict` method
key_modifications (dict, optional): To be passed as an argument
to the `todict` method
groupby (list, optional): An optional list of keys based on which
the result list will be hierarchially grouped ( and converted
into a dict)
keyvals_to_merge (list of dicts, optional): A list of parameters
to be merged with each dict of the output list
"""
if groupby:
return deep_group(
olist, keys=groupby, serializer='todict',
serializer_kwargs={
'rels_to_serialize': rels_to_serialize,
'rels_to_expand': rels_to_expand,
'attrs_to_serialize': attrs_to_serialize,
'group_listrels_by': group_listrels_by,
'key_modifications': key_modifications
})
else:
result_list = map(
lambda o: serialized_obj(
o, attrs_to_serialize=attrs_to_serialize,
rels_to_expand=rels_to_expand,
group_listrels_by=group_listrels_by,
rels_to_serialize=rels_to_serialize,
key_modifications=key_modifications),
olist)
if keyvals_to_merge:
result_list = [merge(obj_dict, kvdict)
for obj_dict, kvdict in
zip(result_list, keyvals_to_merge)]
return result_list
def convert_result_to_response_structure(
result, meta={}, attrs_to_serialize=None, rels_to_expand=None,
rels_to_serialize=None, group_listrels_by=None,
dict_struct=None,
preserve_order=None, groupby=None):
params_to_be_serialized = params_for_serialization(
attrs_to_serialize=attrs_to_serialize, rels_to_expand=rels_to_expand,
rels_to_serialize=rels_to_serialize,
group_listrels_by=group_listrels_by,
dict_struct=dict_struct,
preserve_order=preserve_order, groupby=groupby,
check_groupby=True)
if isinstance(result, Pagination):
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', PER_PAGE_ITEMS_COUNT ))
if result.total != 0 and int(page) > result.pages:
return {
"status": "failure",
"error": "PAGE_NOT_FOUND",
"total_pages": result.pages
}
pages_meta = {
'total_pages': result.pages,
'total_items': result.total,
'page': page,
'per_page': per_page,
'curr_page_first_item_index': (page - 1) * per_page + 1,
'curr_page_last_item_index': min(page * per_page, result.total)
}
if isinstance(meta, dict) and len(meta.keys()) > 0:
pages_meta = merge(pages_meta, meta)
return structured(
serializable_list(result.items, **params_to_be_serialized),
meta=pages_meta)
if isinstance(meta, dict) and len(meta.keys()) > 0:
kwargs = merge(params_to_be_serialized, {'meta': meta})
else:
kwargs = params_to_be_serialized
return as_dict_list(result, **kwargs)
def construct_query_modifiers(query_modifiers=None,
allow_modification_via_requests=True):
default_query_modifiers = {
"page": None,
"per_page": 20,
"limit": None,
"offset": None,
"order_by": None,
"sort": "asc",
}
if query_modifiers is None:
query_modifiers = {}
query_modifiers = subdict(query_modifiers, QUERY_MODIFIERS)
query_modifiers = merge(default_query_modifiers, query_modifiers)
if allow_modification_via_requests:
query_modifiers = merge(query_modifiers,
fetch_query_modifiers_from_request())
for k in ['page', 'per_page', 'limit', 'offset']:
if k in query_modifiers:
query_modifiers[k] = null_safe_type_cast(int,
query_modifiers.get(k))
return query_modifiers
def prepare_data_sources(data_sources, app, engine_kwargs=None):
for db_name, db_dict in data_sources["sqla_compatible_dbs"].items():
engine_kwargs_for_db = merge(
fetch_nested_key_from_dict(engine_kwargs, '*') or {},
fetch_nested_key_from_dict(engine_kwargs, db_name) or {})
db_dict["sqla_db_uri"] = construct_sqla_db_uri(db_dict)
db_dict["engine"] = create_engine(db_dict["sqla_db_uri"],
**engine_kwargs_for_db)
if db_dict.get("automap_tables"):
db_dict["metadata"] = MetaData()
db_dict["metadata"].reflect(db_dict["engine"])
elif db_dict.get("automap_orm"):
db_dict["base"] = automap_base()
db_dict["base"].prepare(db_dict["engine"], reflect=True)
db_dict["query_builder"] = SqlaQueryBuilder(
db_dict["engine"],
flask_app=app,
timedelta_mins_from_utc=app.config["TIMEDELTA_MINS"])
def params_for_serialization(
attrs_to_serialize=None, rels_to_expand=None,
rels_to_serialize=None, group_listrels_by=None,
dict_struct=None,
preserve_order=None, groupby=None, check_groupby=False):
params = _serializable_params(request.args, check_groupby=check_groupby)
if attrs_to_serialize is not None and 'attrs_to_serialize' not in params:
params['attrs_to_serialize'] = attrs_to_serialize
if rels_to_expand is not None and 'rels_to_expand' not in params:
params['rels_to_expand'] = rels_to_expand
if rels_to_serialize is not None and 'rels_to_serialize' not in params:
params['rels_to_serialize'] = rels_to_serialize
if group_listrels_by is not None and 'group_listrels_by' not in params:
params['group_listrels_by'] = group_listrels_by
if preserve_order is not None and 'preserve_order' not in params:
params['preserve_order'] = preserve_order
if groupby is not None and 'groupby' not in params:
params['groupby'] = groupby
if dict_struct is not None:
if 'dict_struct' not in params:
params['dict_struct'] = dict_struct
else:
params['dict_struct'] = merge(dict_struct, params['dict_struct'])
return params
def run_interactive_shell(app, db):
app.config['WTF_CSRF_ENABLED'] = False
# Needed for making the console work in app request context
ctx = app.test_request_context()
ctx.push()
# app.preprocess_request()
# The test client. You can do .get and .post on all endpoints
client = app.test_client()
get = client.get
post = client.post
put = client.put
patch = client.patch
delete = client.delete
# Helper method for sending JSON POST.
def load_json(resp):
return json.loads(resp.data)
def jpost(url, data, raw=False):
response = client.post(url,
data=json.dumps(data),
content_type="application/json")
if raw:
return response
return load_json(response)
def jput(url, data, raw=False):
response = client.put(url,
data=json.dumps(data),
content_type="application/json")
if raw:
return response
return load_json(response)
def jpatch(url, data, raw=False):
response = client.patch(url,
data=json.dumps(data),
content_type="application/json")
if raw:
return response
return load_json(response)
def jget(url, **kwargs):
return load_json(get(url, **kwargs))
# Use this in your code as `with login() as c:` and you can use
# all the methods defined on `app.test_client`
@contextmanager
def login(email=None, password=None):
client.post('/login', data={'email': email, 'password': password})
yield
client.get('/logout', follow_redirects=True)
q = db.session.query
add = db.session.add
addall = db.session.add_all
commit = db.session.commit
delete = db.session.delete
sitemap = app.url_map._rules_by_endpoint
routes = {}
endpoints = {}
pprint = PrettyPrinter(indent=4).pprint
for rule in app.url_map._rules:
routes[rule.rule] = rule.endpoint
endpoints[rule.endpoint] = rule.rule
try:
import IPython
IPython.embed()
except:
import code
code.interact(local=merge(locals(), globals()))
def serializable_list(
olist, attrs_to_serialize=None, rels_to_expand=None,
group_listrels_by=None, rels_to_serialize=None,
key_modifications=None, groupby=None, keyvals_to_merge=None,
preserve_order=False, dict_struct=None, dict_post_processors=None):
"""
Converts a list of model instances to a list of dictionaries
using their `todict` method.
Args:
olist (list): The list of instances to convert
attrs_to_serialize (list, optional): To be passed as an argument
to the `todict` method
rels_to_expand (list, optional): To be passed as an argument
to the `todict` method
group_listrels_by (dict, optional): To be passed as an argument
to the `todict` method
rels_to_serialize (list, optional): To be passed as an argument
to the `todict` method
key_modifications (dict, optional): To be passed as an argument
to the `todict` method
groupby (list, optional): An optional list of keys based on which
the result list will be hierarchially grouped ( and converted
into a dict)
keyvals_to_merge (list of dicts, optional): A list of parameters
to be merged with each dict of the output list
"""
if groupby:
if preserve_order:
result = json_encoder(deep_group(
olist, keys=groupby, serializer='todict',
preserve_order=preserve_order,
serializer_kwargs={
'rels_to_serialize': rels_to_serialize,
'rels_to_expand': rels_to_expand,
'attrs_to_serialize': attrs_to_serialize,
'group_listrels_by': group_listrels_by,
'key_modifications': key_modifications,
'dict_struct': dict_struct,
'dict_post_processors': dict_post_processors
}))
else:
result = deep_group(
olist, keys=groupby, serializer='todict',
preserve_order=preserve_order,
serializer_kwargs={
'rels_to_serialize': rels_to_serialize,
'rels_to_expand': rels_to_expand,
'attrs_to_serialize': attrs_to_serialize,
'group_listrels_by': group_listrels_by,
'key_modifications': key_modifications,
'dict_struct': dict_struct,
'dict_post_processors': dict_post_processors
})
return result
else:
result_list = [serialized_obj(
o, attrs_to_serialize=attrs_to_serialize,
rels_to_expand=rels_to_expand,
group_listrels_by=group_listrels_by,
rels_to_serialize=rels_to_serialize,
key_modifications=key_modifications,
dict_struct=dict_struct,
dict_post_processors=dict_post_processors) for o in olist]
if keyvals_to_merge:
result_list = [merge(obj_dict, kvdict)
for obj_dict, kvdict in
zip(result_list, keyvals_to_merge)]
return result_list
def template_response_from_obj(
obj, template, keys_to_be_replaced={}, merge_keyvals={}):
for k in keys_to_be_replaced:
if k in obj:
obj[keys_to_be_replaced[k]] = obj.pop(k)
return render_template(template, **merge(obj, merge_keyvals))
