def _check_id_validity(self, node_id):
"""
Checks whether id corresponds to an object of the expected type,
whenever type is a valid column of the database (ex. for nodes,
but not for users)
:param node_id: id (or id starting pattern)
:return: True if node_id valid, False if invalid. If True, sets the id
filter attribute correctly
:raise RestValidationError: if no node is found or id pattern does
not identify a unique node
"""
from aiida.common.exceptions import MultipleObjectsError, NotExistent
from aiida.orm.utils.loaders import IdentifierType, get_loader
loader = get_loader(self._aiida_class)
if self._has_uuid:
# For consistency check that id is a string
if not isinstance(node_id, six.string_types):
raise RestValidationError('parameter id has to be a string')
identifier_type = IdentifierType.UUID
qbobj, _ = loader.get_query_builder(
node_id, identifier_type, sub_classes=(self._aiida_class, ))
else:
# Similarly, check that id is an integer
if not isinstance(node_id, int):
raise RestValidationError('parameter id has to be an integer')
identifier_type = IdentifierType.ID
qbobj, _ = loader.get_query_builder(
node_id, identifier_type, sub_classes=(self._aiida_class, ))
# For efficiency I don't go further than two results
qbobj.limit(2)
try:
pk = qbobj.one()[0].pk
except MultipleObjectsError:
raise RestInputValidationError(
'More than one node found. Provide longer starting pattern for id.'
)
except NotExistent:
raise RestInputValidationError(
"either no object's id starts"
" with '{}' or the corresponding object"
' is not of type aiida.orm.{}'.format(node_id,
self._aiida_type))
else:
# create a permanent filter
self._id_filter = {'id': {'==': pk}}
return True
def validate_request(
self, limit=None, offset=None, perpage=None, page=None, query_type=None, is_querystring_defined=False
):
# pylint: disable=fixme,no-self-use,too-many-arguments,too-many-branches
"""
Performs various checks on the consistency of the request.
Add here all the checks that you want to do, except validity of the page
number that is done in paginate().
Future additional checks must be added here
"""
# TODO Consider using **kwargs so to make easier to add more validations
# 1. perpage incompatible with offset and limits
if perpage is not None and (limit is not None or offset is not None):
raise RestValidationError('perpage key is incompatible with limit and offset')
# 2. /page/<int: page> in path is incompatible with limit and offset
if page is not None and (limit is not None or offset is not None):
raise RestValidationError('requesting a specific page is incompatible with limit and offset')
# 3. perpage requires that the path contains a page request
if perpage is not None and page is None:
raise RestValidationError(
'perpage key requires that a page is '
'requested (i.e. the path must contain '
'/page/)'
)
# 4. No querystring if query type = projectable_properties'
if query_type in ('projectable_properties',) and is_querystring_defined:
raise RestInputValidationError('projectable_properties requests do not allow specifying a query string')
def validate_request(limit=None,
offset=None,
perpage=None,
page=None,
query_type=None,
is_querystring_defined=False):
"""
Performs various checks on the consistency of the request.
Add here all the checks that you want to do, except validity of the page
number that is done in paginate().
Future additional checks must be added here
"""
# TODO Consider using **kwargs so to make easier to add more validations
# 1. perpage incompatible with offset and limits
if perpage is not None and (limit is not None or offset is not None):
raise RestValidationError("perpage key is incompatible with "
"limit and offset")
# 2. /page/<int: page> in path is incompatible with limit and offset
if page is not None and (limit is not None or offset is not None):
raise RestValidationError("requesting a specific page is "
"incompatible "
"with limit and offset")
# 3. perpage requires that the path contains a page request
if perpage is not None and page is None:
raise RestValidationError("perpage key requires that a page is "
"requested (i.e. the path must contain "
"/page/)")
# 4. if resource_type=='schema'
if query_type == 'schema' and is_querystring_defined:
raise RestInputValidationError("schema requests do not allow "
"specifying a query string")
def validate_time(s, loc, toks):
datetime_string = toks[0]
# Check the precision
precision = len(datetime_string.replace('T', ':').split(':'))
# Parse
try:
dt = dtparser.parse(datetime_string)
except ValueError:
raise RestInputValidationError("time value has wrong format. The "
"right format is "
"<date>T<time><offset>, "
"where <date> is expressed as "
"[YYYY]-[MM]-[DD], "
"<time> is expressed as [HH]:[MM]:["
"SS], "
"<offset> is expressed as +/-[HH]:["
"MM] "
"given with "
"respect to UTC")
if dt.tzinfo is not None:
tzoffset_minutes = int(
dt.tzinfo.utcoffset(None).total_seconds() / 60)
return datetime_precision(
dt.replace(tzinfo=FixedOffsetTimezone(offset=tzoffset_minutes,
name=None)), precision)
else:
return datetime_precision(
dt.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)),
precision)
def get_all_download_formats(full_type=None):
"""
returns dict of possible node formats for all available node types
"""
all_formats = {}
if full_type:
try:
node_cls = load_entry_point_from_full_type(full_type)
except (TypeError, ValueError):
raise RestInputValidationError(f'The full type {full_type} is invalid.')
except EntryPointError:
raise RestFeatureNotAvailable('The download formats for this node type are not available.')
try:
available_formats = node_cls.get_export_formats()
all_formats[full_type] = available_formats
except AttributeError:
pass
else:
entry_point_group = 'aiida.data'
for name in get_entry_point_names(entry_point_group):
try:
node_cls = load_entry_point(entry_point_group, name)
available_formats = node_cls.get_export_formats()
except (AttributeError, LoadingEntryPointError):
continue
if available_formats:
full_type = construct_full_type(node_cls.class_node_type, '')
all_formats[full_type] = available_formats
return all_formats
def get_visualization_data(node, format=None):
"""
Returns: data in specified format. If format is not specified returns data
in xsf format in order to visualize the structure with JSmol.
"""
response = {}
response["str_viz_info"] = {}
if format is None:
format = 'xsf'
if format in node.get_export_formats():
try:
response["str_viz_info"]["data"] = node._exportstring(
format)[0]
response["str_viz_info"]["format"] = format
except LicensingException as e:
response = e.message
else:
raise RestInputValidationError(
"The format {} is not supported.".format(format))
# Add extra information
response["dimensionality"] = node.get_dimensionality()
response["pbc"] = node.pbc
response["formula"] = node.get_formula()
return response
def get_downloadable_data(node, format=None):
"""
Generic function extented for kpoints data. Currently
it is not implemented.
:param node: node object that has to be visualized
:param format: file extension format
:returns: raise RestFeatureNotAvailable exception
"""
response = {}
if node.folder.exists():
folder_node = node._get_folder_pathsubfolder
filename = node.filename
try:
content = NodeTranslator.get_file_content(
folder_node, filename)
except IOError as e:
error = "Error in getting {} content".format(filename)
raise RestInputValidationError(error)
response["status"] = 200
response["data"] = content
response["filename"] = filename
else:
response["status"] = 200
response["data"] = "file does not exist"
return response
def get_retrieved_outputs(node, filename=None, rtype=None):
"""
Get the retrieved output files for job calculation
:param node: aiida node
:return: the retrieved output files for job calculation
"""
if node.dbnode.type.startswith("calculation.job."):
retrieved_folder = node.out.retrieved
response = {}
if retrieved_folder is None:
response["status"] = 200
response["data"] = "This node does not have retrieved folder"
return response
output_folder = retrieved_folder._get_folder_pathsubfolder
if filename is not None:
if rtype is None:
rtype = "download"
if rtype == "download":
try:
content = NodeTranslator.get_file_content(
output_folder, filename)
except IOError as e:
error = "Error in getting {} content".format(filename)
raise RestInputValidationError(error)
response["status"] = 200
response["data"] = content
response["filename"] = filename.replace("/", "_")
else:
raise RestInputValidationError("rtype is not supported")
return response
# if filename is not provided, return list of all retrieved files
retrieved = CalculationTranslator.get_files_list(output_folder)
return retrieved
return []
def _load_and_verify(self, node_id=None):
"""Load node and verify it is of the required type"""
from aiida.orm import load_node
node = load_node(node_id)
if not isinstance(node, self.trans._aiida_class): # pylint: disable=protected-access,isinstance-second-argument-not-valid-type
raise RestInputValidationError(
f'node {node_id} is not of the required type {self.trans._aiida_class}' # pylint: disable=protected-access
)
return node
def _load_and_verify(self, node_id=None):
"""Load node and verify it is of the required type"""
from aiida.orm import load_node
node = load_node(node_id)
if not isinstance(node, self.trans._aiida_class): # pylint: disable=protected-access
raise RestInputValidationError(
'node {} is not of the required type {}'.format(
node_id, self.trans._aiida_class) # pylint: disable=protected-access
)
return node
def set_query(self, filters=None, orders=None, projections=None, id=None):
"""
Adds filters, default projections, order specs to the query_help,
and initializes the qb object
:param filters: dictionary with the filters
:param orders: dictionary with the order for each tag
:param orders: dictionary with the projections
:param id: id of a specific node
:type id: int
"""
tagged_filters = {}
## Check if filters are well defined and construct an ad-hoc filter
# for id_query
if id is not None:
self._is_id_query = True
if self._result_type == self.__label__ and len(filters) > 0:
raise RestInputValidationError("selecting a specific id does "
"not "
"allow to specify filters")
try:
self._check_id_validity(id)
except RestValidationError as e:
raise RestValidationError(e.message)
else:
tagged_filters[self.__label__] = self._id_filter
if self._result_type is not self.__label__:
tagged_filters[self._result_type] = filters
else:
tagged_filters[self.__label__] = filters
## Add filters
self.set_filters(tagged_filters)
## Add projections
if projections is None:
self.set_default_projections()
else:
tagged_projections = {self._result_type: projections}
self.set_projections(tagged_projections)
##Add order_by
if orders is not None:
tagged_orders = {self._result_type: orders}
self.set_order(tagged_orders)
## Initialize the query_object
self.init_qb()
def set_query(self, filters=None, orders=None, projections=None, pk=None):
"""
Adds filters, default projections, order specs to the query_help,
and initializes the qb object
:param filters: dictionary with the filters
:param orders: dictionary with the order for each tag
:param pk (integer): pk of a specific node
"""
tagged_filters = {}
## Check if filters are well defined and construct an ad-hoc filter
# for pk_query
if pk is not None:
self._is_pk_query = True
if self._result_type == self.__label__ and len(filters) > 0:
raise RestInputValidationError("selecting a specific pk does "
"not "
"allow to specify filters")
elif not self._check_pk_validity(pk):
raise RestValidationError(
"either the selected pk does not exist "
"or the corresponding object is not of "
"type aiida.orm.{}".format(self._aiida_type))
else:
tagged_filters[self.__label__] = {'id': {'==': pk}}
if self._result_type is not self.__label__:
tagged_filters[self._result_type] = filters
else:
tagged_filters[self.__label__] = filters
## Add filters
self.set_filters(tagged_filters)
## Add projections
if projections is None:
self.set_default_projections()
else:
tagged_projections = {self._result_type: projections}
self.set_projections(tagged_projections)
##Add order_by
if orders is not None:
tagged_orders = {self._result_type: orders}
self.set_order(tagged_orders)
## Initialize the query_object
self.init_qb()
def get_downloadable_data(node, download_format=None):
"""
Return content in the specified format for download
:param node: node representing cif file to be downloaded
:param download_format: export format
:returns: content of the node in the specified format for download
"""
response = {}
if download_format is None:
raise RestInputValidationError(
'Please specify the download format. '
'The available download formats can be '
'queried using the /nodes/download_formats/ endpoint.')
elif download_format in node.get_export_formats():
try:
response['data'] = node._exportcontent(download_format)[0] # pylint: disable=protected-access
response['status'] = 200
try:
response['filename'] = node.filename
except AttributeError:
response['filename'] = f'{node.uuid}.{download_format}'
except LicensingException as exc:
response['status'] = 500
response['data'] = str(exc)
return response
else:
raise RestInputValidationError(
'The format {} is not supported. '
'The available download formats can be '
'queried using the /nodes/download_formats/ endpoint.'.format(
download_format))
def get_repo_list(node, filename=''):
"""
Every node in AiiDA is having repo folder.
This function returns the metadata using list_objects() method
:param node: node object
:param filename: folder name (optional)
:return: folder list
"""
try:
flist = node.list_objects(filename)
except IOError:
raise RestInputValidationError(f'{filename} is not a directory in this repository')
response = []
for fobj in flist:
response.append({'name': fobj.name, 'type': fobj.file_type.name})
return response
def get_repo_contents(node, filename=''):
"""
Every node in AiiDA is having repo folder.
This function returns the metadata using get_object() method
:param node: node object
:param filename: folder or file name (optional)
:return: file content in bytes to download
"""
if filename:
try:
data = node.get_object_content(filename, mode='rb')
return data
except IOError:
raise RestInputValidationError('No such file is present')
raise RestValidationError('filename is not provided')
def get_all_download_formats(full_type=None):
"""
returns dict of possible node formats for all available node types
"""
from aiida.plugins.entry_point import load_entry_point, get_entry_point_names
from aiida.restapi.common.identifiers import load_entry_point_from_full_type, construct_full_type
from aiida.common import EntryPointError
from aiida.common.exceptions import LoadingEntryPointError
from aiida.restapi.common.exceptions import RestFeatureNotAvailable, RestInputValidationError
all_formats = {}
if full_type:
try:
node_cls = load_entry_point_from_full_type(full_type)
except (TypeError, ValueError):
raise RestInputValidationError(
'The full type {} is invalid.'.format(full_type))
except EntryPointError:
raise RestFeatureNotAvailable(
'The download formats for this node type are not available.'
)
try:
available_formats = node_cls.get_export_formats()
all_formats[full_type] = available_formats
except AttributeError:
pass
else:
entry_point_group = 'aiida.data'
for name in get_entry_point_names(entry_point_group):
try:
node_cls = load_entry_point(entry_point_group, name)
except LoadingEntryPointError:
pass
else:
node_cls.get_export_formats()
try:
available_formats = node_cls.get_export_formats()
if available_formats:
full_type = construct_full_type(
node_cls.class_node_type, '')
all_formats[full_type] = available_formats
except AttributeError:
pass
return all_formats
def validate_time(toks):
"""
Function to convert datetime string into datetime object. The format is
compliant with ParseAction requirements
:param toks: datetime string passed in tokens
:return: datetime object
"""
datetime_string = toks[0]
# Check the precision
precision = len(datetime_string.replace('T', ':').split(':'))
# Parse
try:
dtobj = dtparser.parse(datetime_string)
except ValueError:
raise RestInputValidationError(
'time value has wrong format. The '
'right format is '
'<date>T<time><offset>, '
'where <date> is expressed as '
'[YYYY]-[MM]-[DD], '
'<time> is expressed as [HH]:[MM]:['
'SS], '
'<offset> is expressed as +/-[HH]:['
'MM] '
'given with '
'respect to UTC')
if dtobj.tzinfo is not None and dtobj.utcoffset() is not None:
tzoffset_minutes = int(dtobj.utcoffset().total_seconds() // 60)
return DatetimePrecision(
dtobj.replace(tzinfo=FixedOffsetTimezone(
offset=tzoffset_minutes, name=None)), precision)
return DatetimePrecision(
dtobj.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)),
precision)
def set_query(self,
filters=None,
orders=None,
projections=None,
query_type=None,
node_id=None,
attributes=None,
attributes_filter=None,
extras=None,
extras_filter=None):
# pylint: disable=too-many-arguments,unused-argument,too-many-locals,too-many-branches
"""
Adds filters, default projections, order specs to the query_help,
and initializes the qb object
:param filters: dictionary with the filters
:param orders: dictionary with the order for each tag
:param projections: dictionary with the projection. It is discarded
if query_type=='attributes'/'extras'
:param query_type: (string) specify the result or the content ("attr")
:param id: (integer) id of a specific node
:param filename: name of the file to return its content
:param attributes: flag to show attributes in nodes endpoint
:param attributes_filter: list of node attributes to query
:param extras: flag to show attributes in nodes endpoint
:param extras_filter: list of node extras to query
"""
tagged_filters = {}
## Check if filters are well defined and construct an ad-hoc filter
# for id_query
if node_id is not None:
self._is_id_query = True
if self._result_type == self.__label__ and filters:
raise RestInputValidationError(
'selecting a specific id does not allow to specify filters'
)
try:
self._check_id_validity(node_id)
except RestValidationError as exc:
raise RestValidationError(str(exc))
else:
tagged_filters[self.__label__] = self._id_filter
if self._result_type is not self.__label__:
tagged_filters[self._result_type] = filters
else:
tagged_filters[self.__label__] = filters
## Add filters
self.set_filters(tagged_filters)
## Add projections
if projections is None:
if attributes is None and extras is None:
self.set_default_projections()
else:
default_projections = self.get_default_projections()
if attributes is True:
if attributes_filter is None:
default_projections.append('attributes')
else:
## Check if attributes_filter is not a list
if not isinstance(attributes_filter, list):
attributes_filter = [attributes_filter]
for attr in attributes_filter:
default_projections.append('attributes.' +
str(attr))
elif attributes is not None and attributes is not False:
raise RestValidationError(
'The attributes filter is false by default and can only be set to true.'
)
if extras is True:
if extras_filter is None:
default_projections.append('extras')
else:
## Check if extras_filter is not a list
if not isinstance(extras_filter, list):
extras_filter = [extras_filter]
for extra in extras_filter:
default_projections.append('extras.' + str(extra))
elif extras is not None and extras is not False:
raise RestValidationError(
'The extras filter is false by default and can only be set to true.'
)
self.set_projections({self.__label__: default_projections})
else:
tagged_projections = {self._result_type: projections}
self.set_projections(tagged_projections)
##Add order_by
if orders is not None:
tagged_orders = {self._result_type: orders}
self.set_order(tagged_orders)
## Initialize the query_object
self.init_qb()
def parse_query_string(self, query_string):
# pylint: disable=too-many-locals
"""
Function that parse the querystring, extracting infos for limit, offset,
ordering, filters, attribute and extra projections.
:param query_string (as obtained from request.query_string)
:return: parsed values for the querykeys
"""
from pyparsing import Word, alphas, nums, alphanums, printables, \
ZeroOrMore, OneOrMore, Suppress, Optional, Literal, Group, \
QuotedString, Combine, \
StringStart as SS, StringEnd as SE, \
WordEnd as WE, \
ParseException
from pyparsing import pyparsing_common as ppc
from dateutil import parser as dtparser
from psycopg2.tz import FixedOffsetTimezone
## Define grammar
# key types
key = Word(f'{alphas}_', f'{alphanums}_')
# operators
operator = (Literal('=like=') | Literal('=ilike=') | Literal('=in=')
| Literal('=notin=') | Literal('=') | Literal('!=')
| Literal('>=') | Literal('>') | Literal('<=')
| Literal('<'))
# Value types
value_num = ppc.number
value_bool = (
Literal('true')
| Literal('false')).addParseAction(lambda toks: bool(toks[0]))
value_string = QuotedString('"', escQuote='""')
value_orderby = Combine(Optional(Word('+-', exact=1)) + key)
## DateTimeShift value. First, compose the atomic values and then
# combine
# them and convert them to datetime objects
# Date
value_date = Combine(
Word(nums, exact=4) + Literal('-') + Word(nums, exact=2) +
Literal('-') + Word(nums, exact=2))
# Time
value_time = Combine(
Literal('T') + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)) +
Optional(Literal(':') + Word(nums, exact=2)))
# Shift
value_shift = Combine(
Word('+-', exact=1) + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)))
# Combine atomic values
value_datetime = Combine(
value_date + Optional(value_time) + Optional(value_shift) +
WE(printables.replace('&', ''))
# To us the
# word must end with '&' or end of the string
# Adding WordEnd only here is very important. This makes atomic
# values for date, time and shift not really
# usable alone individually.
)
########################################################################
def validate_time(toks):
"""
Function to convert datetime string into datetime object. The format is
compliant with ParseAction requirements
:param toks: datetime string passed in tokens
:return: datetime object
"""
datetime_string = toks[0]
# Check the precision
precision = len(datetime_string.replace('T', ':').split(':'))
# Parse
try:
dtobj = dtparser.parse(datetime_string)
except ValueError:
raise RestInputValidationError(
'time value has wrong format. The '
'right format is '
'<date>T<time><offset>, '
'where <date> is expressed as '
'[YYYY]-[MM]-[DD], '
'<time> is expressed as [HH]:[MM]:['
'SS], '
'<offset> is expressed as +/-[HH]:['
'MM] '
'given with '
'respect to UTC')
if dtobj.tzinfo is not None and dtobj.utcoffset() is not None:
tzoffset_minutes = int(dtobj.utcoffset().total_seconds() // 60)
return DatetimePrecision(
dtobj.replace(tzinfo=FixedOffsetTimezone(
offset=tzoffset_minutes, name=None)), precision)
return DatetimePrecision(
dtobj.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)),
precision)
########################################################################
# Convert datetime value to datetime object
value_datetime.setParseAction(validate_time)
# More General types
value = (value_string | value_bool | value_datetime | value_num
| value_orderby)
# List of values (I do not check the homogeneity of the types of values,
# query builder will do it somehow)
value_list = Group(value + OneOrMore(Suppress(',') + value) +
Optional(Suppress(',')))
# Fields
single_field = Group(key + operator + value)
list_field = Group(key + (Literal('=in=') | Literal('=notin=')) +
value_list)
orderby_field = Group(key + Literal('=') + value_list)
field = (list_field | orderby_field | single_field)
# Fields separator
separator = Suppress(Literal('&'))
# General query string
general_grammar = SS() + Optional(field) + ZeroOrMore(
separator + field) + \
Optional(separator) + SE()
## Parse the query string
try:
fields = general_grammar.parseString(query_string)
# JQuery adds _=timestamp a parameter to not use cached data/response.
# To handle query, remove this "_" parameter from the query string
# For more details check issue #789
# (https://github.com/aiidateam/aiida-core/issues/789) in aiida-core
field_list = [
entry for entry in fields.asList() if entry[0] != '_'
]
except ParseException as err:
raise RestInputValidationError(
'The query string format is invalid. '
"Parser returned this massage: \"{"
"}.\" Please notice that the column "
'number '
'is counted from '
'the first character of the query '
'string.'.format(err))
## return the translator instructions elaborated from the field_list
return self.build_translator_parameters(field_list)
def build_translator_parameters(self, field_list):
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
"""
Takes a list of elements resulting from the parsing the query_string and
elaborates them in order to provide translator-compliant instructions
:param field_list: a (nested) list of elements resulting from parsing the query_string
:returns: the filters in the
"""
## Create void variables
filters = {}
orderby = []
limit = None
offset = None
perpage = None
filename = None
download_format = None
download = True
attributes = None
attributes_filter = None
extras = None
extras_filter = None
full_type = None
# io tree limit parameters
tree_in_limit = None
tree_out_limit = None
## Count how many time a key has been used for the filters
# and check if reserved keyword have been used twice
field_counts = {}
for field in field_list:
field_key = field[0]
if field_key not in field_counts.keys():
field_counts[field_key] = 1
# Store the information whether membership operator is used
# is_membership = (field[1] is '=in=')
else:
# Check if the key of a filter using membership operator is used
# in multiple filters
# if is_membership is True or field[1] is '=in=':
# raise RestInputValidationError("If a key appears in "
# "multiple filters, "
# "those cannot use "
# "membership opertor '=in='")
field_counts[field_key] = field_counts[field_key] + 1
## Check the reserved keywords
if 'limit' in field_counts.keys() and field_counts['limit'] > 1:
raise RestInputValidationError(
'You cannot specify limit more than once')
if 'offset' in field_counts.keys() and field_counts['offset'] > 1:
raise RestInputValidationError(
'You cannot specify offset more than once')
if 'perpage' in field_counts.keys() and field_counts['perpage'] > 1:
raise RestInputValidationError(
'You cannot specify perpage more than once')
if 'orderby' in field_counts.keys() and field_counts['orderby'] > 1:
raise RestInputValidationError(
'You cannot specify orderby more than once')
if 'download' in field_counts.keys() and field_counts['download'] > 1:
raise RestInputValidationError(
'You cannot specify download more than once')
if 'download_format' in field_counts.keys(
) and field_counts['download_format'] > 1:
raise RestInputValidationError(
'You cannot specify download_format more than once')
if 'filename' in field_counts.keys() and field_counts['filename'] > 1:
raise RestInputValidationError(
'You cannot specify filename more than once')
if 'in_limit' in field_counts.keys() and field_counts['in_limit'] > 1:
raise RestInputValidationError(
'You cannot specify in_limit more than once')
if 'out_limit' in field_counts.keys(
) and field_counts['out_limit'] > 1:
raise RestInputValidationError(
'You cannot specify out_limit more than once')
if 'attributes' in field_counts.keys(
) and field_counts['attributes'] > 1:
raise RestInputValidationError(
'You cannot specify attributes more than once')
if 'attributes_filter' in field_counts.keys(
) and field_counts['attributes_filter'] > 1:
raise RestInputValidationError(
'You cannot specify attributes_filter more than once')
if 'extras' in field_counts.keys() and field_counts['extras'] > 1:
raise RestInputValidationError(
'You cannot specify extras more than once')
if 'extras_filter' in field_counts.keys(
) and field_counts['extras_filter'] > 1:
raise RestInputValidationError(
'You cannot specify extras_filter more than once')
if 'full_type' in field_counts.keys(
) and field_counts['full_type'] > 1:
raise RestInputValidationError(
'You cannot specify full_type more than once')
## Extract results
for field in field_list:
if field[0] == 'limit':
if field[1] == '=':
limit = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'limit'"
)
elif field[0] == 'offset':
if field[1] == '=':
offset = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'offset'"
)
elif field[0] == 'perpage':
if field[1] == '=':
perpage = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'perpage'"
)
elif field[0] == 'orderby':
if field[1] == '=':
# Consider value (gives string) and value_list (gives list of
# strings) cases
if isinstance(field[2], list):
orderby.extend(field[2])
else:
orderby.extend([field[2]])
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'orderby'"
)
elif field[0] == 'download':
if field[1] == '=':
download = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'download'"
)
elif field[0] == 'download_format':
if field[1] == '=':
download_format = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'download_format'"
)
elif field[0] == 'filename':
if field[1] == '=':
filename = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'filename'"
)
elif field[0] == 'full_type':
if field[1] == '=':
full_type = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'full_type'"
)
elif field[0] == 'in_limit':
if field[1] == '=':
tree_in_limit = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'in_limit'"
)
elif field[0] == 'out_limit':
if field[1] == '=':
tree_out_limit = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'out_limit'"
)
elif field[0] == 'attributes':
if field[1] == '=':
attributes = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'attributes'"
)
elif field[0] == 'attributes_filter':
if field[1] == '=':
attributes_filter = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'attributes_filter'"
)
elif field[0] == 'extras':
if field[1] == '=':
extras = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'extras'"
)
elif field[0] == 'extras_filter':
if field[1] == '=':
extras_filter = field[2]
else:
raise RestInputValidationError(
"only assignment operator '=' is permitted after 'extras_filter'"
)
else:
## Construct the filter entry.
field_key = field[0]
operator = field[1]
field_value = field[2]
if isinstance(field_value,
DatetimePrecision) and operator == '=':
filter_value = self.build_datetime_filter(field_value)
else:
filter_value = {self.op_conv_map[field[1]]: field_value}
# Here I treat the AND clause
if field_counts[field_key] > 1:
if field_key not in filters.keys():
filters.update({field_key: {'and': [filter_value]}})
else:
filters[field_key]['and'].append(filter_value)
else:
filters.update({field_key: filter_value})
# #Impose defaults if needed
# if limit is None:
# limit = self.limit_default
return (limit, offset, perpage, orderby, filters, download_format,
download, filename, tree_in_limit, tree_out_limit, attributes,
attributes_filter, extras, extras_filter, full_type)
def paginate(self, page, perpage, total_count):
"""
Calculates limit and offset for the reults of a query,
given the page and the number of restuls per page.
Moreover, calculates the last available page and raises an exception
if the
required page exceeds that limit.
If number of rows==0, only page 1 exists
:param page: integer number of the page that has to be viewed
:param perpage: integer defining how many results a page contains
:param total_count: the total number of rows retrieved by the query
:return: integers: limit, offset, rel_pages
"""
from math import ceil
## Type checks
# Mandatory params
try:
page = int(page)
except ValueError:
raise InputValidationError('page number must be an integer')
try:
total_count = int(total_count)
except ValueError:
raise InputValidationError('total_count must be an integer')
# Non-mandatory params
if perpage is not None:
try:
perpage = int(perpage)
except ValueError:
raise InputValidationError('perpage must be an integer')
else:
perpage = self.perpage_default
## First_page is anyway 1
first_page = 1
## Calculate last page
if total_count == 0:
last_page = 1
else:
last_page = int(ceil(total_count / perpage))
## Check validity of required page and calculate limit, offset,
# previous,
# and next page
if page > last_page or page < 1:
raise RestInputValidationError(
f'Non existent page requested. The page range is [{first_page} : {last_page}]'
)
limit = perpage
offset = (page - 1) * perpage
prev_page = None
if page > 1:
prev_page = page - 1
next_page = None
if page < last_page:
next_page = page + 1
rel_pages = dict(prev=prev_page,
next=next_page,
first=first_page,
last=last_page)
return (limit, offset, rel_pages)
def parse_path(self, path_string, parse_pk_uuid=None):
# pylint: disable=too-many-return-statements,too-many-branches, too-many-statements
"""
Takes the path and parse it checking its validity. Does not parse "io",
"content" fields. I do not check the validity of the path, since I assume
that this is done by the Flask routing methods.
:param path_string: the path string
:param parse_id_uuid: if 'pk' ('uuid') expects an integer (uuid starting pattern)
:return: resource_type (string)
page (integer)
node_id (string: uuid starting pattern, int: pk)
query_type (string))
"""
## Initialization
page = None
node_id = None
query_type = 'default'
path = self.split_path(self.strip_api_prefix(path_string))
## Pop out iteratively the "words" of the path until it is an empty
# list.
## This way it should be easier to plug in more endpoint logic
# Resource type
resource_type = path.pop(0)
if not path:
return (resource_type, page, node_id, query_type)
# Validate uuid or starting pattern of uuid.
# Technique: - take our UUID_REF and replace the first characters the
# string to be validated as uuid.
# - validate instead the newly built string
if parse_pk_uuid == 'pk':
raw_id = path[0]
try:
# Check whether it can be an integer
node_id = int(raw_id)
except ValueError:
pass
else:
path.pop(0)
elif parse_pk_uuid == 'uuid':
import uuid
raw_id = path[0]
maybe_uuid = raw_id + UUID_REF[len(raw_id):]
try:
_ = uuid.UUID(maybe_uuid, version=4)
except ValueError:
# assume that it cannot be an id and go to the next check
pass
else:
# It is a node_id so pop out the path element
node_id = raw_id
path.pop(0)
if not path:
return (resource_type, page, node_id, query_type)
if path[0] in [
'projectable_properties', 'statistics', 'full_types',
'full_types_count', 'download', 'download_formats', 'report',
'status', 'input_files', 'output_files'
]:
query_type = path.pop(0)
if path:
raise RestInputValidationError(
'Given url does not accept further fields')
elif path[0] in ['links', 'contents']:
path.pop(0)
query_type = path.pop(0)
elif path[0] in ['repo']:
path.pop(0)
query_type = f'repo_{path.pop(0)}'
if not path:
return (resource_type, page, node_id, query_type)
# Page (this has to be in any case the last field)
if path[0] == 'page':
path.pop(0)
if not path:
page = 1
return (resource_type, page, node_id, query_type)
page = int(path.pop(0))
else:
raise RestInputValidationError(
'The requested URL is not found on the server.')
return (resource_type, page, node_id, query_type)
def parse_path(path_string):
"""
Takes the path and parse it checking its validity. Does not parse "io",
"content" fields. I do not check the validity of the path, since I assume
that this is done by the Flask routing methods.
It assunme
:param path_string: the path string
:return:resource_type (string)
page (integer)
pk (integer)
result_type (string))
"""
## Initialization
page = None
pk = None
query_type = "default"
path = split_path(strip_prefix(path_string))
## Pop out iteratively the "words" of the path until it is an empty list.
## This way it should be easier to plug in more endpoint logic
# Resource type
resource_type = path.pop(0)
if not path:
return (resource_type, page, pk, query_type)
# Node_pk
try:
pk = int(path[0])
foo = path.pop(0)
except ValueError:
pass
if not path:
return (resource_type, page, pk, query_type)
# Result type (input, output, attributes, extras, schema)
if path[0] == 'schema':
query_type = path.pop(0)
if path:
raise RestInputValidationError(
"url requesting schema resources do not "
"admit further fields")
else:
return (resource_type, page, pk, query_type)
elif path[0] == 'statistics':
query_type = path.pop(0)
if path:
raise RestInputValidationError(
"url requesting statistics resources do not "
"admit further fields")
else:
return (resource_type, page, pk, query_type)
elif path[0] == "io" or path[0] == "content":
foo = path.pop(0)
query_type = path.pop(0)
if not path:
return (resource_type, page, pk, query_type)
# Page (this has to be in any case the last field)
if path[0] == "page":
do_paginate = True
foo = path.pop(0)
if not path:
page = 1
return (resource_type, page, pk, query_type)
else:
page = int(path.pop(0))
return (resource_type, page, pk, query_type)
def parse_query_string(query_string):
"""
Function that parse the querystring, extracting infos for limit, offset,
ordering, filters, attribute and extra projections.
:param query_string (as obtained from request.query_string)
:return: parsed values for the querykeys
"""
from pyparsing import Word, alphas, nums, alphanums, printables, \
ZeroOrMore, OneOrMore, Suppress, Optional, Literal, Group, \
QuotedString, Combine, \
StringStart as SS, StringEnd as SE, \
WordStart as WS, WordEnd as WE, \
ParseException
from pyparsing import pyparsing_common as ppc
from dateutil import parser as dtparser
from psycopg2.tz import FixedOffsetTimezone
## Define grammar
# key types
key = Word(alphas + '_', alphanums + '_')
# operators
operator = (Literal('=like=') | Literal('=ilike=') | Literal('=in=')
| Literal('=notin=') | Literal('=') | Literal('!=')
| Literal('>=') | Literal('>') | Literal('<=') | Literal('<'))
# Value types
valueNum = ppc.number
valueBool = (Literal('true')
| Literal('false')).addParseAction(lambda toks: bool(toks[0]))
valueString = QuotedString('"', escQuote='""')
valueOrderby = Combine(Optional(Word('+-', exact=1)) + key)
## DateTimeShift value. First, compose the atomic values and then combine
# them and convert them to datetime objects
# Date
valueDate = Combine(
Word(nums, exact=4) + Literal('-') + Word(nums, exact=2) +
Literal('-') + Word(nums, exact=2))
# Time
valueTime = Combine(
Literal('T') + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)) +
Optional(Literal(':') + Word(nums, exact=2)))
# Shift
valueShift = Combine(
Word('+-', exact=1) + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)))
# Combine atomic values
valueDateTime = Combine(
valueDate + Optional(valueTime) + Optional(valueShift) +
WE(printables.translate(None, '&')) # To us the
# word must end with '&' or end of the string
# Adding WordEnd only here is very important. This makes atomic
# values for date, time and shift not really
# usable alone individually.
)
############################################################################
# Function to convert datetime string into datetime object. The format is
# compliant with ParseAction requirements
def validate_time(s, loc, toks):
datetime_string = toks[0]
# Check the precision
precision = len(datetime_string.replace('T', ':').split(':'))
# Parse
try:
dt = dtparser.parse(datetime_string)
except ValueError:
raise RestInputValidationError("time value has wrong format. The "
"right format is "
"<date>T<time><offset>, "
"where <date> is expressed as "
"[YYYY]-[MM]-[DD], "
"<time> is expressed as [HH]:[MM]:["
"SS], "
"<offset> is expressed as +/-[HH]:["
"MM] "
"given with "
"respect to UTC")
if dt.tzinfo is not None:
tzoffset_minutes = int(
dt.tzinfo.utcoffset(None).total_seconds() / 60)
return datetime_precision(
dt.replace(tzinfo=FixedOffsetTimezone(offset=tzoffset_minutes,
name=None)), precision)
else:
return datetime_precision(
dt.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)),
precision)
########################################################################
# Convert datetime value to datetime object
valueDateTime.setParseAction(validate_time)
# More General types
value = (valueString | valueBool | valueDateTime | valueNum | valueOrderby)
# List of values (I do not check the homogeneity of the types of values,
# query builder will do it in a sense)
valueList = Group(value + OneOrMore(Suppress(',') + value) +
Optional(Suppress(',')))
# Fields
singleField = Group(key + operator + value)
listField = Group(key + (Literal('=in=') | Literal('=notin=')) + valueList)
orderbyField = Group(key + Literal('=') + valueList)
Field = (listField | orderbyField | singleField)
# Fields separator
separator = Suppress(Literal('&'))
# General query string
generalGrammar = SS() + Optional(Field) + ZeroOrMore(separator + Field) + \
Optional(separator) + SE()
## Parse the query string
try:
fields = generalGrammar.parseString(query_string)
field_dict = fields.asDict()
field_list = fields.asList()
except ParseException as e:
raise RestInputValidationError("The query string format is invalid. "
"Parser returned this massage: \"{"
"}.\" Please notice that the column "
"number "
"is counted from "
"the first character of the query "
"string.".format(e))
## return the translator instructions elaborated from the field_list
return build_translator_parameters(field_list)
def build_translator_parameters(field_list):
"""
Takes a list of elements resulting from the parsing the query_string and
elaborates them in order to provide translator-compliant instructions
:param field_list: a (nested) list of elements resulting from parsing the
query_string
:return: the filters in the
"""
## Create void variables
filters = {}
orderby = []
limit = None
offset = None
perpage = None
alist = None
nalist = None
elist = None
nelist = None
## Count how many time a key has been used for the filters and check if
# reserved keyword
# have been used twice,
field_counts = {}
for field in field_list:
field_key = field[0]
if field_key not in field_counts.keys():
field_counts[field_key] = 1
# Store the information whether membership operator is used
# is_membership = (field[1] is '=in=')
else:
# Check if the key of a filter using membership operator is used
# in multiple filters
# if is_membership is True or field[1] is '=in=':
# raise RestInputValidationError("If a key appears in "
# "multiple filters, "
# "those cannot use "
# "membership opertor '=in='")
field_counts[field_key] = field_counts[field_key] + 1
## Check the reserved keywords
if 'limit' in field_counts.keys() and field_counts['limit'] > 1:
raise RestInputValidationError("You cannot specify limit more than "
"once")
if 'offset' in field_counts.keys() and field_counts['offset'] > 1:
raise RestInputValidationError("You cannot specify offset more than "
"once")
if 'perpage' in field_counts.keys() and field_counts['perpage'] > 1:
raise RestInputValidationError("You cannot specify perpage more than "
"once")
if 'orderby' in field_counts.keys() and field_counts['orderby'] > 1:
raise RestInputValidationError("You cannot specify orderby more than "
"once")
if 'alist' in field_counts.keys() and field_counts['alist'] > 1:
raise RestInputValidationError("You cannot specify alist more than "
"once")
if 'nalist' in field_counts.keys() and field_counts['nalist'] > 1:
raise RestInputValidationError("You cannot specify nalist more than "
"once")
if 'elist' in field_counts.keys() and field_counts['elist'] > 1:
raise RestInputValidationError("You cannot specify elist more than "
"once")
if 'nelist' in field_counts.keys() and field_counts['nelist'] > 1:
raise RestInputValidationError("You cannot specify nelist more than "
"once")
## Extract results
for field in field_list:
if field[0] == 'limit':
if field[1] == '=':
limit = field[2]
else:
raise RestInputValidationError("only assignment operator '=' "
"is permitted after 'limit'")
elif field[0] == 'offset':
if field[1] == '=':
offset = field[2]
else:
raise RestInputValidationError("only assignment operator '=' "
"is permitted after 'offset'")
elif field[0] == 'perpage':
if field[1] == '=':
perpage = field[2]
else:
raise RestInputValidationError("only assignment operator '=' "
"is permitted after 'perpage'")
elif field[0] == 'alist':
if field[1] == '=':
alist = field[2]
else:
raise RestInputValidationError("only assignment operator '=' "
"is permitted after 'alist'")
elif field[0] == 'nalist':
if field[1] == '=':
nalist = field[2]
else:
raise RestInputValidationError("only assignment operator '=' "
"is permitted after 'nalist'")
elif field[0] == 'elist':
if field[1] == '=':
elist = field[2]
else:
raise RestInputValidationError("only assignment operator '=' "
"is permitted after 'elist'")
elif field[0] == 'nelist':
if field[1] == '=':
nelist = field[2]
else:
raise RestInputValidationError("only assignment operator '=' "
"is permitted after 'nelist'")
elif field[0] == 'orderby':
if field[1] == '=':
# Consider value (gives string) and valueList (gives list of
# strings) cases
if type(field[2]) == list:
orderby.extend(field[2])
else:
orderby.extend([field[2]])
else:
raise RestInputValidationError("only assignment operator '=' "
"is permitted after 'orderby'")
else:
## Construct the filter entry.
field_key = field[0]
operator = field[1]
field_value = field[2]
if isinstance(field_value, datetime_precision) and operator == '=':
filter_value = build_datetime_filter(field_value)
else:
filter_value = {op_conv_map[field[1]]: field_value}
# Here I treat the AND clause
if field_counts[field_key] > 1:
if field_key not in filters.keys():
filters.update({field_key: {'and': [filter_value]}})
else:
filters[field_key]['and'].append(filter_value)
else:
filters.update({field_key: filter_value})
return (limit, offset, perpage, orderby, filters, alist, nalist, elist,
nelist)
def parse_path(self, path_string, parse_pk_uuid=None):
"""
Takes the path and parse it checking its validity. Does not parse "io",
"content" fields. I do not check the validity of the path, since I
assume
that this is done by the Flask routing methods.
:param path_string: the path string
:param parse_id_uuid: if 'pk' ('uuid') expects an integer (uuid
starting pattern)
:return:resource_type (string)
page (integer)
id (string: uuid starting pattern, int: pk)
query_type (string))
"""
## Initialization
page = None
id = None
query_type = "default"
path = self.split_path(self.strip_prefix(path_string))
## Pop out iteratively the "words" of the path until it is an empty
# list.
## This way it should be easier to plug in more endpoint logic
# Resource type
resource_type = path.pop(0)
if not path:
return (resource_type, page, id, query_type)
# Validate uuid or starting pattern of uuid.
# Technique: - take our uuid_ref and replace the first characters the
# string to be validated as uuid.
# - validate instead the newly built string
if parse_pk_uuid == 'pk':
raw_id = path[0]
try:
# Check whether it can be an integer
id = int(raw_id)
except ValueError:
pass
else:
path.pop(0)
elif parse_pk_uuid == 'uuid':
import uuid
raw_id = path[0]
maybe_uuid = raw_id + uuid_ref[len(raw_id):]
try:
_ = uuid.UUID(maybe_uuid, version=4)
except ValueError:
# assume that it cannot be an id and go to the next check
pass
else:
# It is an id so pop out the path element
id = raw_id
path.pop(0)
if not path:
return (resource_type, page, id, query_type)
# Query type (input, output, attributes, extras, visualization,
# schema, statistics)
if path[0] == 'schema':
query_type = path.pop(0)
if path:
raise RestInputValidationError(
"url requesting schema resources do not "
"admit further fields")
else:
return (resource_type, page, id, query_type)
elif path[0] == 'statistics':
query_type = path.pop(0)
if path:
raise RestInputValidationError(
"url requesting statistics resources do not "
"admit further fields")
else:
return (resource_type, page, id, query_type)
elif path[0] == "io" or path[0] == "content":
path.pop(0)
query_type = path.pop(0)
if not path:
return (resource_type, page, id, query_type)
# Page (this has to be in any case the last field)
if path[0] == "page":
path.pop(0)
if not path:
page = 1
return (resource_type, page, id, query_type)
else:
page = int(path.pop(0))
return (resource_type, page, id, query_type)