def merge_params_with_request_args_while_deep_merging_dict_struct(kwargs): params_from_request_args = _serializable_params(request.args) merged_params = merge(kwargs, params_from_request_args) if kwargs.get('dict_struct') and params_from_request_args.get('dict_struct'): merged_params['dict_struct'] = merge( kwargs['dict_struct'], params_from_request_args.get('dict_struct')) return merged_params
def convert_timestamp_indexed_df_to_dt( df, dt_id=None, types_of_fields=None, timestamp_col_name_format="%b %Y"): df = df.replace(np.nan, 0, regex=True) field_types = { field: field_type for field_type, fields in types_of_fields.items() for field in fields } if types_of_fields else {} if isinstance(df.index, pd.DatetimeIndex): dt_columns = [ {"name": "Period", "id": "period"}] + [ { "name": col_name.replace("_", " ").capitalize(), "id": col_name } for col_name in df.columns ] dt_rows = [ merge({ k: format_value(v, field_types.get(k)) for k, v in row.items() }, { "period": idx.strftime( timestamp_col_name_format), }) for idx, row in zip( df.index, df.to_dict('records')) ] else: print("in else block") dt_columns = [ {"name": "Metric", "id": "metric"}] + [ { "name": i.strftime(timestamp_col_name_format), "id": i.strftime(timestamp_col_name_format) } for i in df.columns ] dt_rows = [merge({ k.strftime( timestamp_col_name_format): format_value( v, field_types.get(idx)) for k, v in row.items()}, { "metric": idx.replace("_", " ").capitalize()} ) for idx, row in zip(df.index, df.to_dict('records'))] return dash_table.DataTable( id=dt_id or 'table', style_data={'whiteSpace': 'normal'}, css=[{ 'selector': '.dash-cell div.dash-cell-value', 'rule': 'display: inline; white-space: inherit;\ overflow: inherit; text-overflow: inherit;' }], columns=dt_columns, data=dt_rows, )
def _prepare_data_for_saving(cls, kwargs): """Returns a preprocessed dictionary of parameters. Use this to filter the kwargs passed to `new`, `create`, `build` methods. Args: **kwargs: a dictionary of parameters """ # kwargs.pop('csrf_token', None) for attr, val in kwargs.items(): if cls.is_the_primary_key( attr) and cls._prevent_primary_key_initialization_: del kwargs[attr] continue if val == "": # Making an assumption that there is no good usecase # for setting an empty string. This will help prevent # cases where empty string is sent because of client # not clearing form fields to null kwargs[attr] = None continue if attr in class_mapper( cls).relationships and attr not in cls._no_overwrite_: rel = class_mapper(cls).relationships[attr] if rel.uselist: if isinstance(val, list): if all(isinstance(v, dict) for v in val): rel_cls = cls.mapped_rel_class(attr) kwargs[attr] = rel_cls.update_or_new_all( list_of_kwargs=val, keys=[rel_cls.primary_key_name()]) elif isinstance(val, dict): rel_cls = cls.mapped_rel_class(attr) mapping_col = rel.collection_class().keyfunc.name list_of_kwargs = [ merge(v, {mapping_col: k}) for k, v in val.items() ] kwargs[attr] = { getattr(obj, mapping_col): obj for obj in rel_cls.update_or_new_all( list_of_kwargs=list_of_kwargs, keys=[rel_cls.primary_key_name()]) } elif isinstance(val, dict): rel_cls = cls.mapped_rel_class(attr) kwargs[attr] = rel_cls.update_or_new( **merge(val, {'keys': [rel_cls.primary_key_name()]})) return kwargs
def jsoned(struct, wrap=True, meta=None, struct_key='result'): """ Provides a json dump of the struct Args: struct: The data to dump wrap (bool, optional): Specify whether to wrap the struct in an enclosing dict struct_key (str, optional): The string key which will contain the struct in the result dict meta (dict, optional): An optional dictonary to merge with the output dictionary. Examples: >>> jsoned([3,4,5]) ... '{"status": "success", "result": [3, 4, 5]}' >>> jsoned([3,4,5], wrap=False) ... '[3, 4, 5]' """ if wrap: output = {'status': 'success', struct_key: struct} if meta: output = merge(output, meta) return _json.dumps(output, default=json_encoder) else: return _json.dumps(struct, default=json_encoder)
def serializable_list(olist, attrs_to_serialize=None, rels_to_expand=None, group_listrels_by=None, rels_to_serialize=None, key_modifications=None, groupby=None, keyvals_to_merge=None): if groupby: return deep_group(olist, keys=groupby, serializer='todict', serializer_kwargs={ 'rels_to_serialize': rels_to_serialize, 'rels_to_expand': rels_to_expand, 'attrs_to_serialize': attrs_to_serialize, 'group_listrels_by': group_listrels_by, 'key_modifications': key_modifications }) else: result_list = map( lambda o: serialized_obj(o, attrs_to_serialize=attrs_to_serialize, rels_to_expand=rels_to_expand, group_listrels_by=group_listrels_by, rels_to_serialize=rels_to_serialize, key_modifications=key_modifications), olist) if keyvals_to_merge: result_list = [ merge(obj_dict, kvdict) for obj_dict, kvdict in zip(result_list, keyvals_to_merge) ] return result_list
def params_for_serialization(attrs_to_serialize=None, rels_to_expand=None, rels_to_serialize=None, group_listrels_by=None, dict_struct=None, preserve_order=None, groupby=None, check_groupby=False): params = _serializable_params(request.args, check_groupby=check_groupby) if attrs_to_serialize is not None and 'attrs_to_serialize' not in params: params['attrs_to_serialize'] = attrs_to_serialize if rels_to_expand is not None and 'rels_to_expand' not in params: params['rels_to_expand'] = rels_to_expand if rels_to_serialize is not None and 'rels_to_serialize' not in params: params['rels_to_serialize'] = rels_to_serialize if group_listrels_by is not None and 'group_listrels_by' not in params: params['group_listrels_by'] = group_listrels_by if preserve_order is not None and 'preserve_order' not in params: params['preserve_order'] = preserve_order if groupby is not None and 'groupby' not in params: params['groupby'] = groupby if dict_struct is not None: if 'dict_struct' not in params: params['dict_struct'] = dict_struct else: params['dict_struct'] = merge(dict_struct, params['dict_struct']) return params
def template_response_from_obj(obj, template, keys_to_be_replaced={}, merge_keyvals={}): for k in keys_to_be_replaced: if k in obj: obj[keys_to_be_replaced[k]] = obj.pop(k) return render_template(template, **merge(obj, merge_keyvals))
def upload(self, url, file_key, file_path, **kwargs): buffered = kwargs.pop('buffered', True) content_type = kwargs.pop('content_type', 'multipart/form-data') kwargs['data'] = merge( kwargs['data'], {file_key: (open(file_path, 'rb'), file_path)}) return self.post(url, buffered=buffered, content_type=content_type, **kwargs)
def jsoned(struct, wrap=True, meta=None): if wrap: output = {'status': 'success', 'result': struct} if meta: output = merge(output, meta) return _json.dumps(output, default=_json_encoder) else: return _json.dumps(struct, default=_json_encoder)
def serializable_list(olist, attrs_to_serialize=None, rels_to_expand=None, group_listrels_by=None, rels_to_serialize=None, key_modifications=None, groupby=None, keyvals_to_merge=None): """ Converts a list of model instances to a list of dictionaries using their `todict` method. Args: olist (list): The list of instances to convert attrs_to_serialize (list, optional): To be passed as an argument to the `todict` method rels_to_expand (list, optional): To be passed as an argument to the `todict` method group_listrels_by (dict, optional): To be passed as an argument to the `todict` method rels_to_serialize (list, optional): To be passed as an argument to the `todict` method key_modifications (dict, optional): To be passed as an argument to the `todict` method groupby (list, optional): An optional list of keys based on which the result list will be hierarchially grouped ( and converted into a dict) keyvals_to_merge (list of dicts, optional): A list of parameters to be merged with each dict of the output list """ if groupby: return deep_group(olist, keys=groupby, serializer='todict', serializer_kwargs={ 'rels_to_serialize': rels_to_serialize, 'rels_to_expand': rels_to_expand, 'attrs_to_serialize': attrs_to_serialize, 'group_listrels_by': group_listrels_by, 'key_modifications': key_modifications }) else: result_list = map( lambda o: serialized_obj(o, attrs_to_serialize=attrs_to_serialize, rels_to_expand=rels_to_expand, group_listrels_by=group_listrels_by, rels_to_serialize=rels_to_serialize, key_modifications=key_modifications), olist) if keyvals_to_merge: result_list = [ merge(obj_dict, kvdict) for obj_dict, kvdict in zip(result_list, keyvals_to_merge) ] return result_list
def convert_result_to_response_structure(result, meta={}, attrs_to_serialize=None, rels_to_expand=None, rels_to_serialize=None, group_listrels_by=None, dict_struct=None, preserve_order=None, groupby=None): params_to_be_serialized = params_for_serialization( attrs_to_serialize=attrs_to_serialize, rels_to_expand=rels_to_expand, rels_to_serialize=rels_to_serialize, group_listrels_by=group_listrels_by, dict_struct=dict_struct, preserve_order=preserve_order, groupby=groupby, check_groupby=True) if isinstance(result, Pagination): page = int(request.args.get('page', 1)) per_page = int(request.args.get('per_page', PER_PAGE_ITEMS_COUNT)) if result.total != 0 and int(page) > result.pages: return { "status": "failure", "error": "PAGE_NOT_FOUND", "total_pages": result.pages } pages_meta = { 'total_pages': result.pages, 'total_items': result.total, 'page': page, 'per_page': per_page, 'curr_page_first_item_index': (page - 1) * per_page + 1, 'curr_page_last_item_index': min(page * per_page, result.total) } if isinstance(meta, dict) and len(meta.keys()) > 0: pages_meta = merge(pages_meta, meta) return structured(serializable_list(result.items, **params_to_be_serialized), meta=pages_meta) if isinstance(meta, dict) and len(meta.keys()) > 0: kwargs = merge(params_to_be_serialized, {'meta': meta}) else: kwargs = params_to_be_serialized return as_dict_list(result, **kwargs)
def structured(struct, wrap=True, meta=None, struct_key='result', pre_render_callback=None): output = struct if wrap: output = {'status': 'success', struct_key: struct} if meta: output = merge(output, meta) if pre_render_callback and callable(pre_render_callback): output = pre_render_callback(output) return output
def structured(struct, wrap=True, meta=None, struct_key=None, pre_render_callback=None): if struct_key is None: struct_key = "result" output = struct if wrap: output = {'status': 'success', struct_key: struct} if meta: output = merge(output, meta) if pre_render_callback and callable(pre_render_callback): output = pre_render_callback(output) return output
def _preprocess_params(cls, kwargs): """Returns a preprocessed dictionary of parameters. Use this to filter the kwargs passed to `new`, `create`, `build` methods. Args: **kwargs: a dictionary of parameters """ # kwargs.pop('csrf_token', None) for attr, val in kwargs.items(): if cls.is_the_primary_key(attr) and cls._prevent_primary_key_initialization_: del kwargs[attr] continue if val == "": # Making an assumption that there is no good usecase # for setting an empty string. This will help prevent # cases where empty string is sent because of client # not clearing form fields to null kwargs[attr] = None continue if attr in class_mapper(cls).relationships and attr not in cls._no_overwrite_: rel = class_mapper(cls).relationships[attr] if rel.uselist: if isinstance(val, list): if all(isinstance(v, dict) for v in val): rel_cls = cls.mapped_rel_class(attr) kwargs[attr] = rel_cls.update_or_new_all( list_of_kwargs=val, keys=[rel_cls.primary_key_name()]) elif isinstance(val, dict): rel_cls = cls.mapped_rel_class(attr) mapping_col = rel.collection_class().keyfunc.name list_of_kwargs = [merge(v, {mapping_col: k}) for k, v in val.items()] kwargs[attr] = {getattr(obj, mapping_col): obj for obj in rel_cls.update_or_new_all( list_of_kwargs=list_of_kwargs, keys=[rel_cls.primary_key_name()])} elif isinstance(val, dict): rel_cls = cls.mapped_rel_class(attr) kwargs[attr] = rel_cls.update_or_new( **merge(val, {'keys': [rel_cls.primary_key_name()]})) return kwargs
def serializable_list( olist, attrs_to_serialize=None, rels_to_expand=None, group_listrels_by=None, rels_to_serialize=None, key_modifications=None, groupby=None, keyvals_to_merge=None): """ Converts a list of model instances to a list of dictionaries using their `todict` method. Args: olist (list): The list of instances to convert attrs_to_serialize (list, optional): To be passed as an argument to the `todict` method rels_to_expand (list, optional): To be passed as an argument to the `todict` method group_listrels_by (dict, optional): To be passed as an argument to the `todict` method rels_to_serialize (list, optional): To be passed as an argument to the `todict` method key_modifications (dict, optional): To be passed as an argument to the `todict` method groupby (list, optional): An optional list of keys based on which the result list will be hierarchially grouped ( and converted into a dict) keyvals_to_merge (list of dicts, optional): A list of parameters to be merged with each dict of the output list """ if groupby: return deep_group( olist, keys=groupby, serializer='todict', serializer_kwargs={ 'rels_to_serialize': rels_to_serialize, 'rels_to_expand': rels_to_expand, 'attrs_to_serialize': attrs_to_serialize, 'group_listrels_by': group_listrels_by, 'key_modifications': key_modifications }) else: result_list = map( lambda o: serialized_obj( o, attrs_to_serialize=attrs_to_serialize, rels_to_expand=rels_to_expand, group_listrels_by=group_listrels_by, rels_to_serialize=rels_to_serialize, key_modifications=key_modifications), olist) if keyvals_to_merge: result_list = [merge(obj_dict, kvdict) for obj_dict, kvdict in zip(result_list, keyvals_to_merge)] return result_list
def convert_result_to_response_structure( result, meta={}, attrs_to_serialize=None, rels_to_expand=None, rels_to_serialize=None, group_listrels_by=None, dict_struct=None, preserve_order=None, groupby=None): params_to_be_serialized = params_for_serialization( attrs_to_serialize=attrs_to_serialize, rels_to_expand=rels_to_expand, rels_to_serialize=rels_to_serialize, group_listrels_by=group_listrels_by, dict_struct=dict_struct, preserve_order=preserve_order, groupby=groupby, check_groupby=True) if isinstance(result, Pagination): page = int(request.args.get('page', 1)) per_page = int(request.args.get('per_page', PER_PAGE_ITEMS_COUNT )) if result.total != 0 and int(page) > result.pages: return { "status": "failure", "error": "PAGE_NOT_FOUND", "total_pages": result.pages } pages_meta = { 'total_pages': result.pages, 'total_items': result.total, 'page': page, 'per_page': per_page, 'curr_page_first_item_index': (page - 1) * per_page + 1, 'curr_page_last_item_index': min(page * per_page, result.total) } if isinstance(meta, dict) and len(meta.keys()) > 0: pages_meta = merge(pages_meta, meta) return structured( serializable_list(result.items, **params_to_be_serialized), meta=pages_meta) if isinstance(meta, dict) and len(meta.keys()) > 0: kwargs = merge(params_to_be_serialized, {'meta': meta}) else: kwargs = params_to_be_serialized return as_dict_list(result, **kwargs)
def construct_query_modifiers(query_modifiers=None, allow_modification_via_requests=True): default_query_modifiers = { "page": None, "per_page": 20, "limit": None, "offset": None, "order_by": None, "sort": "asc", } if query_modifiers is None: query_modifiers = {} query_modifiers = subdict(query_modifiers, QUERY_MODIFIERS) query_modifiers = merge(default_query_modifiers, query_modifiers) if allow_modification_via_requests: query_modifiers = merge(query_modifiers, fetch_query_modifiers_from_request()) for k in ['page', 'per_page', 'limit', 'offset']: if k in query_modifiers: query_modifiers[k] = null_safe_type_cast(int, query_modifiers.get(k)) return query_modifiers
def prepare_data_sources(data_sources, app, engine_kwargs=None): for db_name, db_dict in data_sources["sqla_compatible_dbs"].items(): engine_kwargs_for_db = merge( fetch_nested_key_from_dict(engine_kwargs, '*') or {}, fetch_nested_key_from_dict(engine_kwargs, db_name) or {}) db_dict["sqla_db_uri"] = construct_sqla_db_uri(db_dict) db_dict["engine"] = create_engine(db_dict["sqla_db_uri"], **engine_kwargs_for_db) if db_dict.get("automap_tables"): db_dict["metadata"] = MetaData() db_dict["metadata"].reflect(db_dict["engine"]) elif db_dict.get("automap_orm"): db_dict["base"] = automap_base() db_dict["base"].prepare(db_dict["engine"], reflect=True) db_dict["query_builder"] = SqlaQueryBuilder( db_dict["engine"], flask_app=app, timedelta_mins_from_utc=app.config["TIMEDELTA_MINS"])
def params_for_serialization( attrs_to_serialize=None, rels_to_expand=None, rels_to_serialize=None, group_listrels_by=None, dict_struct=None, preserve_order=None, groupby=None, check_groupby=False): params = _serializable_params(request.args, check_groupby=check_groupby) if attrs_to_serialize is not None and 'attrs_to_serialize' not in params: params['attrs_to_serialize'] = attrs_to_serialize if rels_to_expand is not None and 'rels_to_expand' not in params: params['rels_to_expand'] = rels_to_expand if rels_to_serialize is not None and 'rels_to_serialize' not in params: params['rels_to_serialize'] = rels_to_serialize if group_listrels_by is not None and 'group_listrels_by' not in params: params['group_listrels_by'] = group_listrels_by if preserve_order is not None and 'preserve_order' not in params: params['preserve_order'] = preserve_order if groupby is not None and 'groupby' not in params: params['groupby'] = groupby if dict_struct is not None: if 'dict_struct' not in params: params['dict_struct'] = dict_struct else: params['dict_struct'] = merge(dict_struct, params['dict_struct']) return params
def run_interactive_shell(app, db): app.config['WTF_CSRF_ENABLED'] = False # Needed for making the console work in app request context ctx = app.test_request_context() ctx.push() # app.preprocess_request() # The test client. You can do .get and .post on all endpoints client = app.test_client() get = client.get post = client.post put = client.put patch = client.patch delete = client.delete # Helper method for sending JSON POST. def load_json(resp): return json.loads(resp.data) def jpost(url, data, raw=False): response = client.post(url, data=json.dumps(data), content_type="application/json") if raw: return response return load_json(response) def jput(url, data, raw=False): response = client.put(url, data=json.dumps(data), content_type="application/json") if raw: return response return load_json(response) def jpatch(url, data, raw=False): response = client.patch(url, data=json.dumps(data), content_type="application/json") if raw: return response return load_json(response) def jget(url, **kwargs): return load_json(get(url, **kwargs)) # Use this in your code as `with login() as c:` and you can use # all the methods defined on `app.test_client` @contextmanager def login(email=None, password=None): client.post('/login', data={'email': email, 'password': password}) yield client.get('/logout', follow_redirects=True) q = db.session.query add = db.session.add addall = db.session.add_all commit = db.session.commit delete = db.session.delete sitemap = app.url_map._rules_by_endpoint routes = {} endpoints = {} pprint = PrettyPrinter(indent=4).pprint for rule in app.url_map._rules: routes[rule.rule] = rule.endpoint endpoints[rule.endpoint] = rule.rule try: import IPython IPython.embed() except: import code code.interact(local=merge(locals(), globals()))
def serializable_list( olist, attrs_to_serialize=None, rels_to_expand=None, group_listrels_by=None, rels_to_serialize=None, key_modifications=None, groupby=None, keyvals_to_merge=None, preserve_order=False, dict_struct=None, dict_post_processors=None): """ Converts a list of model instances to a list of dictionaries using their `todict` method. Args: olist (list): The list of instances to convert attrs_to_serialize (list, optional): To be passed as an argument to the `todict` method rels_to_expand (list, optional): To be passed as an argument to the `todict` method group_listrels_by (dict, optional): To be passed as an argument to the `todict` method rels_to_serialize (list, optional): To be passed as an argument to the `todict` method key_modifications (dict, optional): To be passed as an argument to the `todict` method groupby (list, optional): An optional list of keys based on which the result list will be hierarchially grouped ( and converted into a dict) keyvals_to_merge (list of dicts, optional): A list of parameters to be merged with each dict of the output list """ if groupby: if preserve_order: result = json_encoder(deep_group( olist, keys=groupby, serializer='todict', preserve_order=preserve_order, serializer_kwargs={ 'rels_to_serialize': rels_to_serialize, 'rels_to_expand': rels_to_expand, 'attrs_to_serialize': attrs_to_serialize, 'group_listrels_by': group_listrels_by, 'key_modifications': key_modifications, 'dict_struct': dict_struct, 'dict_post_processors': dict_post_processors })) else: result = deep_group( olist, keys=groupby, serializer='todict', preserve_order=preserve_order, serializer_kwargs={ 'rels_to_serialize': rels_to_serialize, 'rels_to_expand': rels_to_expand, 'attrs_to_serialize': attrs_to_serialize, 'group_listrels_by': group_listrels_by, 'key_modifications': key_modifications, 'dict_struct': dict_struct, 'dict_post_processors': dict_post_processors }) return result else: result_list = [serialized_obj( o, attrs_to_serialize=attrs_to_serialize, rels_to_expand=rels_to_expand, group_listrels_by=group_listrels_by, rels_to_serialize=rels_to_serialize, key_modifications=key_modifications, dict_struct=dict_struct, dict_post_processors=dict_post_processors) for o in olist] if keyvals_to_merge: result_list = [merge(obj_dict, kvdict) for obj_dict, kvdict in zip(result_list, keyvals_to_merge)] return result_list
def template_response_from_obj( obj, template, keys_to_be_replaced={}, merge_keyvals={}): for k in keys_to_be_replaced: if k in obj: obj[keys_to_be_replaced[k]] = obj.pop(k) return render_template(template, **merge(obj, merge_keyvals))