class OutputArgs(ma.Schema):
"""
Check read-in arguments for creating a job Output.
"""
class Meta:
"""
Ensure that it can only take the defined arguments
"""
strict = True
destination = Str(required=True)
type = Str(required=True)
name = Str(required=True)
label = Str(required=True)
filename = Str(required=True)
@validates_schema(pass_original=True)
def check_unknown_fields(self, data, original_data):
"""
Ensure no additional fields are passed
"""
unknown = set()
for output in original_data:
unknown = set(unknown).union(set(output)) - set(data)
if unknown:
raise BadRequestKeyError("Unknown field", unknown)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.meta = self.Meta()
for k in ("database", "schema"):
if getattr(self.meta, k) is None:
raise ValueError(
f"Meta value '{k}' must be provided for ModelAPIEndpoint")
schema = self.meta.schema
self._model_name = classname_for_table(schema.opts.model.__table__)
log.info(self._model_name)
d = "related models to nest within response [allowed_nests]"
e = "?nest=session,project"
des = create_params(d, e)
self.instance_schema = dict(
nest=NestedModelField(Str(), missing=[], description=des))
des_page = create_params("number of results per page", "?page=>i:5039")
des_per_page = create_params("number of results per page",
"?per_page=15")
des_all = create_params("return all results", "?all=true")
self.args_schema = dict(
**self.instance_schema,
page=Str(missing=None, description=des_page),
per_page=Int(missing=20, description=des_per_page),
all=Boolean(missing=False, description=des_all),
)
self._filters = []
self.register_filter(OpenSearchFilter)
self.register_filter(AuthorityFilter)
self.register_filter(FieldExistsFilter)
self.register_filter(FieldNotExistsFilter)
self.register_filter(EmbargoFilter)
self.register_filter(DateFilter)
self.register_filter(DOIFilter)
self.register_filter(CoordinateFilter)
self.register_filter(GeometryFilter)
self.register_filter(TextSearchFilter)
self.register_filter(AgeRangeFilter)
self.register_filter(IdListFilter)
self.register_filter(TagsFilter)
def __init__(self) -> None:
super().__init__()
self.arg_schema = dict(
page=Str(missing=None, description="Page"),
per_page=Int(missing=20, description="Number to show"),
all=Boolean(missing=False, description="Return all results."),
)
self.fields = {}
self.basic_info = {}
self.schema = None
class AuthorityFilter(BaseFilter):
params = dict(authority=Str(missing=None))
help = dict(authority="Authority")
def should_apply(self):
return hasattr(self.model, "authority")
def apply(self, args, query):
if args["authority"] is None:
return query
return query.filter(self.model.authority == args["authority"])
class OpenSearchEndpoint(HTTPEndpoint):
args_schema = dict(
page=Str(missing=None, description="Page"),
per_page=Int(missing=20, description="Number to show"),
all=Boolean(missing=False, description="Return all results."),
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
async def get(self, request):
db = app_context().database
query_params = request.query_params
if "query" not in query_params or "model" not in query_params:
return JSONResponse({
"Status":
"Success",
"Message":
"Provide a string as a query parameter or model",
"example":
"/api/v2/search?query=lava&model=sample",
})
params = {"query": query_params["query"]}
model = query_params["model"]
models = {
"sample": search_sample,
"project": search_project,
"session": search_session,
}
if model == "all":
sql_fn = search_total
query_res = db.exec_sql_query(fn=sql_fn, params=params)
json_res = [dict(model=x, data=y) for x, y in query_res]
json_res.reverse()
return JSONResponse(json_res)
else:
for key, value in models.items():
if model == key:
sql_fn = value
schema = getattr(db.interface, key)(many=True)
query_res = db.exec_sql_query(fn=sql_fn, params=params)
json_res = [dict(r) for r in query_res]
return APIResponse(json_res, total_count=len(json_res), schema=schema)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.meta = self.Meta()
if getattr(self.meta, "table") is None:
raise ValueError(
f"Meta value '{k}' must be provided for ViewAPIEndpoint")
self._model_name = classname_for_table(self.meta.table)
log.info(self._model_name)
self.args_schema = dict(page=Str(missing=None),
per_page=Int(missing=20),
all=Boolean(missing=False))
class JobPatchArgs(ma.Schema):
"""
Read in the arguments for patching a job
"""
class Meta:
"""
Ensure that other fields are not provided
"""
strict = True
name = Str()
description = Str()
values = Nested(JobArgumentArgs, many=True)
@validates_schema(pass_original=True)
def check_unknown_fields(self, data, original_data):
"""
Ensure no additional fields are passed
"""
unknown = set(original_data) - set(self.fields)
if unknown:
raise BadRequestKeyError("Unknown field {}".format(unknown))
class JobArgs(ma.Schema):
"""
Class to read in arguments to create a new job
"""
class Meta:
"""
Ensure that it can only take the defined arguments
"""
strict = True
case_id = Int(required=True)
author = Str(required=True)
name = Str(required=True)
@validates_schema(pass_original=True)
def check_unknown_fields(self, data, original_data):
"""
Ensure no additional fields are passed
"""
unknown = set(original_data) - set(self.fields)
if unknown:
raise BadRequestKeyError("Unknown field", unknown)
class JobArgumentArgs(ma.Schema):
"""
Class to read in the arguments for a single Job argument value
"""
class Meta:
"""
Ensure that other fields are not provided
"""
strict = True
name = Str(required=True)
value = Str(required=True)
units = Str(required=True)
@validates_schema(pass_original=True, pass_many=True)
def check_unknown_fields(self, data, original_data, many):
"""
Ensure no additional fields are passed
"""
if not many:
self.check_unknown_field(data, original_data)
return
if len(data) != len(original_data):
raise BadRequestKeyError(
"Could not parse all fields, {}".format(original_data))
for index in range(0, len(data)):
self.check_unknown_field(data[index], original_data[index])
def check_unknown_field(self, data, original_data):
"""
Check that a single instance of a field validates
"""
unknown = set(original_data) - set(self.fields)
if unknown:
raise BadRequestKeyError("Unknown field {}".format(unknown))
class StatusPatchSchema(ma.Schema):
"""
Read in arguments for setting a status value
"""
class Meta:
"""
Ensure that other fields are not provided
"""
strict = True
status = Str(validate=lambda s: s.upper() in JobStatus.__members__.keys())
@validates_schema(pass_original=True)
def check_unknown_fields(self, data, original_data):
"""
Ensure no additional fields are passed
"""
unknown = set(original_data) - set(self.fields)
if len(unknown) > 0:
raise BadRequestKeyError("Unknown field {}".format(unknown))
class SearchArgs(ma.Schema):
"""
Read in arguments for searching
"""
class Meta:
"""
Ensure that other fields are not provided
"""
strict = True
name = Str(missing=None, strict=True)
exact = Bool(missing=False, strict=True, required=False)
@validates_schema(pass_original=True)
def check_unknown_fields(self, data, original_data):
"""
Ensure no additional fields are passed
"""
unknown = set(original_data) - set(self.fields)
if len(unknown) > 0:
raise BadRequestKeyError("Unknown field {}".format(unknown))
class DataFileListEndpoint(HTTPEndpoint):
"""A simple demonstration endpoint for paginating a select statement. Extremely quick, but somewhat hand-constructed."""
args_schema = dict(
page=Str(missing=None, description="Page"),
per_page=Int(missing=20, description="Number to show"),
all=Boolean(missing=False, description="Return all results."),
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
db = get_database()
self.schema = db.interface.data_file
self.model = db.model.data_file
self._filters = []
self.register_filter(OpenSearchFilter)
self.register_filter(AuthorityFilter)
self.register_filter(FieldExistsFilter)
self.register_filter(FieldNotExistsFilter)
self.register_filter(EmbargoFilter)
self.register_filter(DateFilter)
self.register_filter(TextSearchFilter)
self.register_filter(AgeRangeFilter)
self.register_filter(IdListFilter)
self.register_filter(TagsFilter)
def register_filter(self, _filter: BaseFilter):
"""Register a filter specification to control parametrization of the
model query"""
f = _filter(self.model, schema=self.schema)
if not f.should_apply():
return
if params := getattr(f, "params"):
self.args_schema.update(**params)
self._filters.append(f)
def params(self):
d = "filter by [field] existence, will return results where passed fields are null"
e = ["?not_has=name", "?not_has=name,location"]
des = create_params(d, e)
return {self.key: DelimitedList(Str(), missing=[], description=des)}
query = query.limit(max_n_results)
return query, new_offset
DEF_MAX_N_RESULTS = 2048
ORDER_BY_OPTIONS = [
'newest',
'oldest',
None,
]
ORDER_TOPICS_OPTIONS = ORDER_BY_OPTIONS + ['newest_last_post']
DEF_GET_COLL_ARGS = {
'fields': DelimitedList(Str()),
'offset': Int(validate=lambda n: n >= 0),
'max_n_results': Int(validate=lambda n: n >= 0, missing=DEF_MAX_N_RESULTS),
'order': Str(validate=validate.OneOf(ORDER_BY_OPTIONS), missing='newest'),
}
GET_TOPICS_ARGS = DEF_GET_COLL_ARGS.copy()
GET_TOPICS_ARGS.update({
'order':
Str(validate=validate.OneOf(ORDER_TOPICS_OPTIONS),
missing='newest_last_post'),
'statuses':
DelimitedList(Str()),
})
def params(self):
d = "A string to match text field of the data model"
e = ["?like=basalt"]
des = create_params(d, e)
return {self.key: Str(description=des)}
def params(self):
d = "A query string to search by"
e = ["?search=basalt"]
des = create_params(d, e)
return {self.key: Str(description=des)}
def params(self):
d = "Age of Geologic material, passed in thousand years (ka), can take either range or single number"
e = ["?age=1200"]
des = create_params(d, e)
return {self.key: Str(description=des)}
def params(self):
d = "A string of Well Know Text for a Polygon, circle or box, will return all data located WITHIN the geometry provided. NOTE: do NOT add SRID, assumed SRID of 4326, WGS84 Longitude/Latitude"
e = ["?geometry=POLYGON((0 0,180 90,180 0,0 90,0 0))"]
des = create_params(d, e)
return {self.key: Str(description=des)}
from flask_restful import Resource, marshal_with
from webargs.fields import Integer, Str, Boolean, Dict, List, Nested
from webargs.flaskparser import use_args
from ..db import Logbook
from ..actions import new_logbook, edit_logbook
from . import fields, send_signal
logbook_args = {
"parent_id":
Integer(allow_none=True),
"name":
Str(),
"description":
Str(allow_none=True),
"template":
Str(allow_none=True),
"attributes":
List(
Nested({
"name":
Str(),
"type":
Str(validate=lambda t: t in
["text", "number", "boolean", "option", "multioption"]),
"required":
Boolean(),
"options":
List(Str())
})),
"metadata":
"""
facets = utils.bootstrap_facets()
# Bootstrap URL institution(s).
facets['institution'] = utils.institution_facets(
include=args['institution_id']
)
return render_template('home.html', facets=facets)
@app.route('/api/ranks')
@use_args(dict(
query = Str(missing=None),
size = Int(missing=200),
page = Int(missing=1),
corpus = List(Str(), missing=None),
field_id = List(Int(), missing=None),
subfield_id = List(Int(), missing=None),
institution_id = List(Int(), missing=None),
state = List(Str(), missing=None),
country = List(Str(), missing=None),
))
def api_ranks(args):
"""
Ranking API.
class TextsRes(Resource):
POST_MAX_N_TAGS = 16
POST_ARGS = {
'tags': DelimitedList(Str()),
'content': Str(),
}
TAGS_BLACKLIST = {}
GET_MAX_N_RESULTS = 1000
GET_ARGS = {
'any_tags': DelimitedList(Str()),
'all_tags': DelimitedList(Str()),
'no_tags': DelimitedList(Str()),
'date_from': Date(),
'date_to': Date(),
'max_n_results': \
Integer(validate=lambda n: n >= 0, missing=GET_MAX_N_RESULTS),
'offset': \
Integer(validate=lambda n: n >= 0, missing=0),
'fields': DelimitedList(Str()),
}
@staticmethod
def parse_post_args(req):
args = parser.parse(TextsRes.POST_ARGS, req)
if 'tags' in args:
if len(args['tags']) > TextsRes.POST_MAX_N_TAGS:
raise ValidationError('too many tags (limit={})'.format(
TextsRes.POST_MAX_N_TAGS))
for tag in args['tags']:
if tag in TextsRes.TAGS_BLACKLIST:
raise ValidationError('"{}" is blacklisted'.format(tag))
args['tags'] = serialize_tags(get_tags(args['tags']))
return args
@staticmethod
def filter_texts(args):
#sorting in decreasing order by creation time
query = Text.query.order_by(-Text.created_at)
if 'date_to' in args:
query = query.filter(func.DATE(Text.created_at) <= args['date_to'])
if 'date_from' in args:
query = query.filter(
func.DATE(Text.created_at) >= args['date_from'])
if 'all_tags' in args:
tags = Tag.query.filter(Tag.content.in_(args['all_tags'])).all()
if len(tags) < len(args['all_tags']):
return [], None
#dirty hack TODO: get a better solution
for t in tags:
query = query.filter(Text.tags.contains(t))
elif 'any_tags' in args:
query = query.join(Tag, Text.tags).join(
Tag.query.join(Text, Tag.texts).filter(
Tag.content.in_(args['any_tags'])))
if 'no_tags' in args:
tags = Tag.query.filter(Tag.content.in_(args['no_tags'])).all()
#new dirty hack TODO: get a better solution
for t in tags:
query = query.filter(~Text.tags.contains(t))
query = query.offset(args['offset'])
if query.count() <= args['max_n_results']:
offset = None
else:
offset = args['offset'] + args['max_n_results']
texts = query.limit(args['max_n_results']).all()
return texts, offset
@staticmethod
def parse_get_args(req):
args = parser.parse(TextsRes.GET_ARGS,
req,
locations=('querystring', ))
return args
@jwt_required
def post(self):
"""
Create new text resource.
.. :quickref: Text creation; Create new text.
**Example request**:
.. sourcecode:: http
POST /texts HTTP/1.1
Host: example.com
Accept: application/json
Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhb
**Example response**:
.. sourcecode:: http
HTTP/1.1 201 CREATED
Content-Type: application/json
{
"text": {
"content": "Bom dia!",
"text_id": 1,
"tags": [
"bomdia"
],
"updated_at": null,
"created_at": "2018-09-15T22:53:26+00:00"
},
}
:reqheader Authorization: access token of logged in user (required)
:form content: the text contents (required)
:form tags: comma-separated list of tags
:resheader Content-Type: application/json
:status 201: resource created
:returns: :class:`memedata.models.Text`
"""
try:
args = TextsRes.parse_post_args(request)
text = TextSchema().load(args)
except ValidationError as e:
return mk_errors(400, fmt_validation_error_messages(e.messages))
db.session.add(text)
db.session.commit()
return TextSchema().dump(text), 201
@jwt_required
def get(self):
"""
Return collection of texts.
.. :quickref: Get Texts; Get collection of texts.
**Example request**:
.. sourcecode:: http
GET /texts HTTP/1.1
Host: example.com
Accept: application/json
Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhb
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"texts": [
{
"content": "Bom dia!",
"text_id": 1,
"tags": [
"bomdia"
],
"updated_at": null,
"created_at": "2018-09-15T22:53:26+00:00"
},
{
"content": "Eu adoro as manhãs",
"text_id": 32,
"tags": [
"jesus",
"sexta"
],
"updated_at": null,
"created_at": "2018-09-15T22:53:26+00:00"
},
],
"offset": 2
}
:reqheader Authorization: access token of logged in user (required)
:query string fields: comma-separated list of fields to get for each \
text.
:query string date_from: only texts created after specified date \
(inclusive).
:query string date_to: only texts created before specified date.
:query string any_tags: texts with at least one tags in specified list.
:query string all_tags: texts only containing all specified tags.
:query string no_tags: texts only not containing any of specified tags.
:query int offset: pagination offset to start getting results
:query int max_n_results: maximum number of results to return.
:resheader Content-Type: application/json
:status 200: texts found
:returns: :class:`memedata.models.Text`
"""
try:
args = TextsRes.parse_get_args(request)
except ValidationError as e:
return mk_errors(400, fmt_validation_error_messages(e.messages))
texts, offset = TextsRes.filter_texts(args)
objs = TextSchema(many=True).dump(texts)
objs = filter_fields(objs, args.get('fields'))
objs['offset'] = offset
return objs
mk_message,
fmt_validation_error_messages,
flatten,
filter_fields,
generate_hash,
verify_hash,
)
from memedata.models import User, RevokedToken
from memedata.database import db
from memedata.extensions import jwt
from memedata import config
_USER_PASS_ARGS = {
'username':
Str(required=True),
'password':
Str(validate=validate.Length(min=config.min_password_len), required=True),
}
def check_priviledges():
if not get_jwt_identity() in config.superusers:
abort(401, 'unauthorized user')
class Login(Resource):
def post(self):
"""
Login into system.
from dha_poc.rec_engine.engine import StandardEngine
from dha_poc.ads_creator.creator import StandardAdGenerator, AdDataBundle
from dha_poc.url_manager.manager import ImageURLManager, RecommendationContext
from dha_poc.data.formatter import HotelDataFormatter
from dha_poc.data.hotel_api import HotelDataInterface
from webargs import core
from webargs.flaskparser import use_args
from webargs.fields import Str, Int, Bool
from dha_poc.util import zipcode_regex, coordinates_regex, sort_criteria, dimensions_regex
from dha_poc.data.hotel_attributes import available_languages
parser = core.Parser()
query_args = {
'zipcode':
Str(required=True,
validate=lambda z: zipcode_regex.match(str(z)) is not None),
'email':
Str(),
'price':
Int(validate=lambda n: 0 <= n <= 10e6),
'title':
Str(validate=lambda s: len(s) < 1000),
'language':
Str(validate=lambda s: len(s) == 2 and s.lower() in available_languages),
'currency':
Str(validate=lambda s: len(s) == 3),
'coord':
Str(validate=lambda s: coordinates_regex.match(s)),
'event_date':
Str(),
'num':
from flask_restful import Resource, marshal_with
from webargs.fields import Integer, Str, Boolean, Dict, List, Nested
from webargs.flaskparser import use_args
from ..db import Logbook
from ..actions import new_logbook, edit_logbook
from . import fields, send_signal
logbook_args = {
"parent_id": Integer(allow_none=True),
"name": Str(),
"description": Str(allow_none=True),
"template": Str(allow_none=True),
"attributes": List(Nested({
"name": Str(),
"type": Str(
validate=lambda t: t in ["text",
"number",
"boolean",
"option",
"multioption"]),
"required": Boolean(),
"options": List(Str(), missing=None)
})),
"metadata": Dict(),
"archived": Boolean(missing=False)
}
class LogbooksResource(Resource):
def params(self):
d = "Filter by date or date range where the format is YYYY-MM-DD"
e = ["?date_range=2013-03-23,2013-05-01"]
des = create_params(d, e)
return {self.key: DelimitedList(Str(), description=des)}
class TextRes(Resource):
GET_ARGS = {
'fields': DelimitedList(Str()),
}
@staticmethod
def get_text(text_id):
text = Text.query.get(text_id)
if text is None:
abort(mk_errors(404, '{} doest not exist'.format(text_id)))
return text
@staticmethod
def parse_get_args(req):
args = parser.parse(TextRes.GET_ARGS, req, locations=('querystring', ))
return args
@jwt_required
def get(self, text_id):
"""
Return text resource.
.. :quickref: Get Text; Get text.
**Example request**:
.. sourcecode:: http
GET /texts/1 HTTP/1.1
Host: example.com
Accept: application/json
Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhb
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"text": {
"content": "Bom dia!",
"text_id": 1,
"tags": [
"bomdia"
],
"updated_at": null,
"created_at": "2018-09-15T22:53:26+00:00"
}
}
:reqheader Authorization: access token of logged in user (required)
:param int text_id: id of text resource.
:resheader Content-Type: application/json
:status 200: text found
:returns: :class:`memedata.models.Text`
"""
text = TextRes.get_text(text_id)
try:
args = TextRes.parse_get_args(request)
except ValidationError as e:
return mk_errors(400, fmt_validation_error_messages(e.messages))
obj = TextSchema().dump(text)
return filter_fields(obj, args.get('fields'))
@jwt_required
def put(self, text_id):
"""
Update text resource.
.. :quickref: Update Text; Update text.
**Example request**:
.. sourcecode:: http
PUT /texts/1 HTTP/1.1
Host: example.com
Accept: application/json
Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhb
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"text": {
"content": "Updated text",
"text_id": 1,
"tags": [
"bomdia"
],
"updated_at": "2018-09-16T23:00:13+00:00",
"created_at": "2018-09-15T22:53:26+00:00"
}
}
:reqheader Authorization: access token of logged in user (required)
:param int text_id: id of text resource.
:form content: the text contents
:form tags: comma-separated list of tags
:resheader Content-Type: application/json
:status 200: text updated
:returns: :class:`memedata.models.Text`
"""
text = TextRes.get_text(text_id)
try:
schema = TextSchema()
args = TextsRes.parse_post_args(request)
text = schema.load(args, instance=text, partial=True)
except ValidationError as e:
return mk_errors(400, fmt_validation_error_messages(e.messages))
db.session.add(text)
db.session.commit()
return schema.dump(text)
@jwt_required
def delete(self, text_id):
"""
Delete text resource.
.. :quickref: Delete Text; Remove text.
**Example request**:
.. sourcecode:: http
DELETE /texts/1 HTTP/1.1
Host: example.com
Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhb
**Example response**:
.. sourcecode:: http
HTTP/1.1 204 NO CONTENT
:reqheader Authorization: access token of logged in user (required)
:param int text_id: id of text resource.
:status 204: text deleted
"""
text = TextRes.get_text(text_id)
db.session.delete(text)
db.session.commit()
return '', 204
def params(self):
d = "Search for field by DOI string, can be whole string or anypart of doi"
e = ["?doi_like=10.10", "?doi_like=10.1130/B31239.1"]
des = create_params(d, e)
return {self.key: Str(description=des)}
LocalDateTime)
from webargs.flaskparser import use_args
from ..db import Entry, Logbook, EntryLock
from ..attachments import handle_img_tags
from ..export import export_entries_as_pdf
from ..actions import new_entry, edit_entry
from . import fields, send_signal
entry_args = {
"id":
Integer(allow_none=True),
"logbook_id":
Integer(allow_none=True),
"title":
Str(allow_none=True),
"content":
Str(),
"content_type":
Str(missing="text/html"),
"authors":
List(Nested({
"name": Str(),
"login": Str(allow_none=True),
"email": Email(allow_none=True)
}),
validate=lambda a: len(a) > 0),
"created_at":
LocalDateTime(),
"last_changed_at":
LocalDateTime(allow_none=True),
def params(self):
d = "Option to filter by coordinates, pass a bounding box, minLong, minLat, maxLong, maxLat"
e = ["?coordinates=0,0,180,90"]
des = create_params(d, e)
return {self.key: DelimitedList(Str(), description=des)}
