def bulk_get_documents():
"""
Returns a list of records.
"""
ids = flask.request.json
if not ids:
raise UserError('No ids provided')
if not isinstance(ids, list):
raise UserError('ids is not a list')
with blueprint.index_driver.session as session:
# Comment it out to compare against the eager loading option.
#query = session.query(IndexRecord)
#query = query.filter(IndexRecord.did.in_(ids)
# Use eager loading.
query = session.query(IndexRecord)
query = query.options(
joinedload(IndexRecord.urls).joinedload(
IndexRecordUrl.url_metadata))
query = query.options(joinedload(IndexRecord.acl))
query = query.options(joinedload(IndexRecord.hashes))
query = query.options(joinedload(IndexRecord.index_metadata))
query = query.options(joinedload(IndexRecord.aliases))
query = query.filter(IndexRecord.did.in_(ids))
docs = [q.to_document_dict() for q in query]
return json.dumps(docs), 200
def validate_hashes(**hashes):
'''
Validate hashes against known and valid hashing algorithms.
'''
if not all(h in ACCEPTABLE_HASHES for h in hashes):
raise UserError('invalid hash types specified')
if not all(ACCEPTABLE_HASHES[h](v) for h, v in hashes.items()):
raise UserError('invalid hash values specified')
def get_index():
'''
Returns a list of records.
'''
limit = flask.request.args.get('limit')
try: limit = 100 if limit is None else int(limit)
except ValueError as err:
raise UserError('limit must be an integer')
if limit <= 0 or limit > 1024:
raise UserError('limit must be between 1 and 1024')
size = flask.request.args.get('size')
try: size = size if size is None else int(size)
except ValueError as err:
raise UserError('size must be an integer')
if size is not None and size < 0:
raise UserError('size must be > 0')
start = flask.request.args.get('start')
urls = flask.request.args.getlist('url')
hashes = flask.request.args.getlist('hash')
hashes = {h:v for h,v in map(lambda x: x.split(':', 1), hashes)}
validate_hashes(**hashes)
hashes = hashes if hashes else None
if limit < 0 or limit > 1024:
raise UserError('limit must be between 0 and 1024')
ids = blueprint.index_driver.ids(
start=start,
limit=limit,
size=size,
urls=urls,
hashes=hashes,
)
base = {
'ids': ids,
'limit': limit,
'start': start,
'size': size,
'urls': urls,
'hashes': hashes,
}
return flask.jsonify(base), 200
def put_index_record(record):
'''
Update an existing record.
'''
try: jsonschema.validate(flask.request.json, PUT_RECORD_SCHEMA)
except jsonschema.ValidationError as err:
raise UserError(err)
rev = flask.request.args.get('rev')
size = flask.request.json.get('size')
urls = flask.request.json.get('urls')
hashes = flask.request.json.get('hashes')
did, rev = blueprint.index_driver.update(record, rev,
size=size,
urls=urls,
hashes=hashes,
)
ret = {
'did': record,
'rev': rev,
}
return flask.jsonify(ret), 200
def query_urls(self, exclude=None, include=None, versioned=None, offset=0, limit=1000, fields="did,urls", **kwargs):
if kwargs:
raise UserError("Unexpected query parameter(s) {}".format(kwargs.keys()))
versioned = versioned.lower() in ["true", "t", "yes", "y"] if versioned else None
with self.driver.session as session:
# special database specific functions dependent of the selected dialect
q_func = driver_query_map.get(session.bind.dialect.name)
query = session.query(IndexRecordUrl.did, q_func['string_agg'](IndexRecordUrl.url, ","))
# add version filter if versioned is not None
if versioned is True: # retrieve only those with a version number
query = query.outerjoin(IndexRecord)
query = query.filter(IndexRecord.version.isnot(None))
elif versioned is False: # retrieve only those without a version number
query = query.outerjoin(IndexRecord)
query = query.filter(~IndexRecord.version.isnot(None))
query = query.group_by(IndexRecordUrl.did)
# add url filters
if include and exclude:
query = query.having(and_(~q_func['string_agg'](IndexRecordUrl.url, ",").contains(exclude),
q_func['string_agg'](IndexRecordUrl.url, ",").contains(include)))
elif include:
query = query.having(q_func['string_agg'](IndexRecordUrl.url, ",").contains(include))
elif exclude:
query = query.having(~q_func['string_agg'](IndexRecordUrl.url, ",").contains(exclude))
print(query)
# [('did', 'urls')]
record_list = query.order_by(IndexRecordUrl.did.asc()).offset(offset).limit(limit).all()
return self._format_response(fields, record_list)
def put_alias_record(record):
'''
Create or replace an existing record.
'''
try: jsonschema.validate(flask.request.json, PUT_RECORD_SCHEMA)
except jsonschema.ValidationError as err:
raise UserError(err)
rev = flask.request.args.get('rev')
size = flask.request.json.get('size')
hashes = flask.request.json.get('hashes')
release = flask.request.json.get('release')
metastring = flask.request.json.get('metadata')
host_authorities = flask.request.json.get('host_authorities')
keeper_authority = flask.request.json.get('keeper_authority')
record, rev = blueprint.alias_driver.upsert(record, rev,
size=size,
hashes=hashes,
release=release,
metastring=metastring,
host_authorities=host_authorities,
keeper_authority=keeper_authority,
)
ret = {
'name': record,
'rev': rev,
}
return flask.jsonify(ret), 200
def put_index_record(record):
'''
Update an existing record.
'''
try:
jsonschema.validate(flask.request.json, PUT_RECORD_SCHEMA)
except jsonschema.ValidationError as err:
raise UserError(err)
rev = flask.request.args.get('rev')
file_name = flask.request.json.get('file_name')
version = flask.request.json.get('version')
urls = flask.request.json.get('urls')
did, baseid, rev = blueprint.index_driver.update(
record,
rev,
file_name=file_name,
version=version,
urls=urls,
)
ret = {
'did': did,
'baseid': baseid,
'rev': rev,
}
return flask.jsonify(ret), 200
def put_alias_record(record):
"""
Create or replace an existing record.
"""
try:
jsonschema.validate(flask.request.json, PUT_RECORD_SCHEMA)
except jsonschema.ValidationError as err:
raise UserError(err)
rev = flask.request.args.get("rev")
size = flask.request.json.get("size")
hashes = flask.request.json.get("hashes")
release = flask.request.json.get("release")
metastring = flask.request.json.get("metadata")
host_authorities = flask.request.json.get("host_authorities")
keeper_authority = flask.request.json.get("keeper_authority")
record, rev = blueprint.alias_driver.upsert(
record,
rev,
size=size,
hashes=hashes,
release=release,
metastring=metastring,
host_authorities=host_authorities,
keeper_authority=keeper_authority,
)
ret = {"name": record, "rev": rev}
return flask.jsonify(ret), 200
def query_metadata_by_key(self, key, value, url=None, versioned=None, offset=0,
limit=1000, fields="did,urls,rev", **kwargs):
if kwargs:
raise UserError("Unexpected query parameter(s) {}".format(kwargs.keys()))
versioned = versioned.lower() in ["true", "t", "yes", "y"] if versioned else None
with self.driver.session as session:
query = session.query(IndexRecordUrlMetadata.did,
IndexRecordUrlMetadata.url,
IndexRecord.rev)\
.filter(IndexRecord.did == IndexRecordUrlMetadata.did,
IndexRecordUrlMetadata.key == key, IndexRecordUrlMetadata.value == value)
# filter by version
if versioned is True:
query = query.filter(IndexRecord.version.isnot(None))
elif versioned is False:
query = query.filter(~IndexRecord.version.isnot(None))
# add url filter
if url:
query = query.filter(IndexRecordUrlMetadata.url.like("%{}%".format(url)))
# [('did', 'url', 'rev')]
record_list = query.order_by(IndexRecordUrlMetadata.did.asc()).offset(offset).limit(limit).all()
return self._format_response(fields, record_list)
def post_index_record():
'''
Create a new record.
'''
try: jsonschema.validate(flask.request.json, POST_RECORD_SCHEMA)
except jsonschema.ValidationError as err:
raise UserError(err)
form = flask.request.json['form']
size = flask.request.json['size']
urls = flask.request.json['urls']
hashes = flask.request.json['hashes']
did = flask.request.json.get('did')
did, rev = blueprint.index_driver.add(form, size,
urls=urls,
hashes=hashes,
did=did,
)
ret = {
'did': did,
'rev': rev,
}
return flask.jsonify(ret), 200
def get_alias():
"""
Returns a list of records.
"""
limit = flask.request.args.get("limit")
try:
limit = 100 if limit is None else int(limit)
except ValueError as err:
raise UserError("limit must be an integer")
if limit <= 0 or limit > 1024:
raise UserError("limit must be between 1 and 1024")
size = flask.request.args.get("size")
try:
size = size if size is None else int(size)
except ValueError as err:
raise UserError("size must be an integer")
if size is not None and size < 0:
raise UserError("size must be > 0")
start = flask.request.args.get("start")
hashes = flask.request.args.getlist("hash")
hashes = {h: v for h, v in (x.split(":", 1) for x in hashes)}
# TODO FIXME this needs reworking
validate_hashes(**hashes)
hashes = hashes if hashes else None
if limit < 0 or limit > 1024:
raise UserError("limit must be between 0 and 1024")
aliases = blueprint.alias_driver.aliases(
start=start, limit=limit, size=size, hashes=hashes
)
base = {
"aliases": aliases,
"limit": limit,
"start": start,
"size": size,
"hashes": hashes,
}
return flask.jsonify(base), 200
def get_signed_url(object_id, access_id):
if not access_id:
raise (UserError("Access ID/Protocol is required."))
res = flask.current_app.fence_client.get_signed_url_for_object(
object_id=object_id, access_id=access_id)
if not res:
raise IndexNoRecordFound("No signed url found")
return res, 200
def update_blank_record(self, did, rev, size, hashes, urls):
"""
Update a blank record with size and hashes, raise exception
if the record is non-empty or the revision is not matched
"""
hashes = hashes or {}
urls = urls or []
if not size or not hashes:
raise UserError("No size or hashes provided")
with self.session as session:
query = session.query(IndexRecord).filter(IndexRecord.did == did)
try:
record = query.one()
except NoResultFound:
raise NoRecordFound('no record found')
except MultipleResultsFound:
raise MultipleRecordsFound('multiple records found')
if record.size or record.hashes:
raise UserError("update api is not supported for non-empty record!")
if rev != record.rev:
raise RevisionMismatch('revision mismatch')
record.size = size
record.hashes = [IndexRecordHash(
did=record.did,
hash_type=h,
hash_value=v,
) for h, v in hashes.items()]
record.urls = [IndexRecordUrl(
did=record.did,
url=url,
) for url in urls]
record.rev = str(uuid.uuid4())[:8]
session.add(record)
session.commit()
return record.did, record.rev, record.baseid
def create_urls_metadata(urls_metadata, record, session):
"""
create url metadata record in database
"""
urls = {u.url for u in record.urls}
for url, url_metadata in urls_metadata.items():
if url not in urls:
raise UserError("url {} in urls_metadata does not exist".format(url))
for k, v in url_metadata.items():
session.add(IndexRecordUrlMetadata(url=url, key=k, value=v, did=record.did))
def list_dos_records():
'''
Returns a record from the local ids, alias, or global resolvers.
Returns DOS Schema
'''
start = flask.request.args.get('page_token')
limit = flask.request.args.get('page_size')
try:
limit = 100 if limit is None else int(limit)
except ValueError:
raise UserError('limit must be an integer')
if limit <= 0 or limit > 1024:
raise UserError('limit must be between 1 and 1024')
url = flask.request.args.get('url')
# Support this in the future when we have
# more fully featured aliases?
# alias = flask.request.args.get('alias')
checksum = flask.request.args.get('checksum')
if checksum:
hashes = {checksum['type']: checksum['checksum']}
else:
hashes = None
records = blueprint.index_driver.ids(start=start,
limit=limit,
urls=url,
hashes=hashes)
for record in records:
record['alias'] = blueprint.index_driver.get_aliases_for_did(
record['did'])
ret = {
"data_objects":
[indexd_to_dos(record)['data_object'] for record in records]
}
return flask.jsonify(ret), 200
def delete_index_record(record):
'''
Delete an existing record.
'''
rev = flask.request.args.get('rev')
if rev is None:
raise UserError('no revision specified')
blueprint.index_driver.delete(record, rev)
return '', 200
def delete_alias_record(record):
'''
Delete an alias.
'''
rev = flask.request.args.get('rev')
if rev is None:
raise UserError('no revision specified')
blueprint.alias_driver.delete(record, rev)
return '', 200
def delete_index_record(record):
"""
Delete an existing record.
"""
rev = flask.request.args.get("rev")
if rev is None:
raise UserError("no revision specified")
# authorize done in delete
blueprint.index_driver.delete(record, rev)
return "", 200
def get_urls():
"""
Returns a list of urls.
"""
ids = flask.request.args.get("ids")
if ids:
ids = ids.split(",")
hashes = flask.request.args.getlist("hash")
hashes = {h: v for h, v in (x.split(":", 1) for x in hashes)}
size = flask.request.args.get("size")
if size:
try:
size = int(size)
except TypeError:
raise UserError("size must be an integer")
if size < 0:
raise UserError("size must be >= 0")
try:
start = int(flask.request.args.get("start", 0))
except TypeError:
raise UserError("start must be an integer")
try:
limit = int(flask.request.args.get("limit", 100))
except TypeError:
raise UserError("limit must be an integer")
if start < 0:
raise UserError("start must be >= 0")
if limit < 0:
raise UserError("limit must be >= 0")
if limit > 1024:
raise UserError("limit must be <= 1024")
validate_hashes(**hashes)
urls = blueprint.index_driver.get_urls(size=size,
ids=ids,
hashes=hashes,
start=start,
limit=limit)
ret = {
"urls": urls,
"limit": limit,
"start": start,
"size": size,
"hashes": hashes
}
return flask.jsonify(ret), 200
def get_urls():
'''
Returns a list of urls.
'''
ids = flask.request.args.getlist('ids')
hashes = flask.request.args.getlist('hash')
hashes = {h: v for h, v in map(lambda x: x.split(':', 1), hashes)}
size = flask.request.args.get('size')
if size:
try:
size = int(size)
except TypeError:
raise UserError('size must be an integer')
if size < 0:
raise UserError('size must be >= 0')
try:
start = int(flask.request.args.get('start', 0))
except TypeError:
raise UserError('start must be an integer')
try:
limit = int(flask.request.args.get('limit', 100))
except TypeError:
raise UserError('limit must be an integer')
if start < 0:
raise UserError('start must be >= 0')
if limit < 0:
raise UserError('limit must be >= 0')
if limit > 1024:
raise UserError('limit must be <= 1024')
validate_hashes(**hashes)
urls = blueprint.index_driver.get_urls(
size=size,
ids=ids,
hashes=hashes,
start=start,
limit=limit,
)
ret = {
'urls': urls,
'limit': limit,
'start': start,
'size': size,
'hashes': hashes,
}
return flask.jsonify(ret), 200
def list_dos_records():
"""
Returns a record from the local ids, alias, or global resolvers.
Returns DOS Schema
"""
start = flask.request.args.get("page_token")
limit = flask.request.args.get("page_size")
try:
limit = 100 if limit is None else int(limit)
except ValueError:
raise UserError("limit must be an integer")
if limit <= 0 or limit > 1024:
raise UserError("limit must be between 1 and 1024")
url = flask.request.args.get("url")
# Support this in the future when we have
# more fully featured aliases?
# alias = flask.request.args.get('alias')
checksum = flask.request.args.get("checksum")
if checksum:
hashes = {checksum["type"]: checksum["checksum"]}
else:
hashes = None
records = blueprint.index_driver.ids(
start=start, limit=limit, urls=url, hashes=hashes
)
for record in records:
record["alias"] = blueprint.index_driver.get_aliases_for_did(record["did"])
ret = {"data_objects": [indexd_to_dos(record)["data_object"] for record in records]}
return flask.jsonify(ret), 200
def list_drs_records():
limit = flask.request.args.get("limit")
start = flask.request.args.get("start")
page = flask.request.args.get("page")
form = flask.request.args.get("form")
try:
limit = 100 if limit is None else int(limit)
except ValueError as err:
raise UserError("limit must be an integer")
if limit < 0 or limit > 1024:
raise UserError("limit must be between 0 and 1024")
if page is not None:
try:
page = int(page)
except ValueError as err:
raise UserError("page must be an integer")
if form == "bundle":
records = blueprint.index_driver.get_bundle_list(start=start,
limit=limit,
page=page)
elif form == "object":
records = blueprint.index_driver.ids(start=start,
limit=limit,
page=page)
else:
records = blueprint.index_driver.get_bundle_and_object_list(
start=start, limit=limit, page=page)
ret = {
"drs_objects": [indexd_to_drs(record, True) for record in records],
}
return flask.jsonify(ret), 200
def put_index_record(record):
"""
Update an existing record.
"""
try:
jsonschema.validate(flask.request.json, PUT_RECORD_SCHEMA)
except jsonschema.ValidationError as err:
raise UserError(err)
rev = flask.request.args.get("rev")
# authorize done in update
did, baseid, rev = blueprint.index_driver.update(record, rev, flask.request.json)
ret = {"did": did, "baseid": baseid, "rev": rev}
return flask.jsonify(ret), 200
def replace_aliases_for_did(self, aliases, did):
"""
Replace all aliases for one DID / GUID with new aliases.
"""
with self.session as session:
self.logger.info(
f"Trying to replace aliases for did {did} with new aliases {aliases}..."
)
index_record = get_record_if_exists(did, session)
if index_record is None:
self.logger.warn(f"No record found for did {did}")
raise NoRecordFound(did)
# authorization
try:
resources = [u.resource for u in index_record.authz]
auth.authorize("update", resources)
except AuthError as err:
self.logger.warn(
f"Auth error while replacing aliases for did {did}: User not authorized to update one or more of these resources: {resources}"
)
raise err
try:
# delete this GUID's aliases
session.query(IndexRecordAlias).filter(
IndexRecordAlias.did == did
).delete(synchronize_session="evaluate")
# add new aliases
index_record_aliases = [
IndexRecordAlias(did=did, name=alias) for alias in aliases
]
session.add_all(index_record_aliases)
session.commit()
self.logger.info(
f"Replaced aliases for did {did} with new aliases {aliases}"
)
except IntegrityError:
# One or more aliases in request were non-unique
self.logger.warn(
f"One or more aliases in request already associated with another GUID: {aliases}"
)
raise UserError(
f"One or more aliases in request already associated with another GUID: {aliases}"
)
def post_index_record():
'''
Create a new record.
'''
try:
jsonschema.validate(flask.request.json, POST_RECORD_SCHEMA)
except jsonschema.ValidationError as err:
raise UserError(err)
did = flask.request.json.get('did')
form = flask.request.json['form']
size = flask.request.json['size']
urls = flask.request.json['urls']
acl = flask.request.json.get('acl', [])
hashes = flask.request.json['hashes']
file_name = flask.request.json.get('file_name')
metadata = flask.request.json.get('metadata')
urls_metadata = flask.request.json.get('urls_metadata')
version = flask.request.json.get('version')
baseid = flask.request.json.get('baseid')
uploader = flask.request.json.get('uploader')
did, rev, baseid = blueprint.index_driver.add(
form,
did,
size=size,
file_name=file_name,
metadata=metadata,
urls_metadata=urls_metadata,
version=version,
urls=urls,
acl=acl,
hashes=hashes,
baseid=baseid,
uploader=uploader,
)
ret = {
'did': did,
'rev': rev,
'baseid': baseid,
}
return flask.jsonify(ret), 200
def post_index_record():
"""
Create a new record.
"""
try:
jsonschema.validate(flask.request.json, POST_RECORD_SCHEMA)
except jsonschema.ValidationError as err:
raise UserError(err)
authz = flask.request.json.get("authz", [])
auth.authorize("create", authz)
did = flask.request.json.get("did")
form = flask.request.json["form"]
size = flask.request.json["size"]
urls = flask.request.json["urls"]
acl = flask.request.json.get("acl", [])
hashes = flask.request.json["hashes"]
file_name = flask.request.json.get("file_name")
metadata = flask.request.json.get("metadata")
urls_metadata = flask.request.json.get("urls_metadata")
version = flask.request.json.get("version")
baseid = flask.request.json.get("baseid")
uploader = flask.request.json.get("uploader")
did, rev, baseid = blueprint.index_driver.add(
form,
did,
size=size,
file_name=file_name,
metadata=metadata,
urls_metadata=urls_metadata,
version=version,
urls=urls,
acl=acl,
authz=authz,
hashes=hashes,
baseid=baseid,
uploader=uploader,
)
ret = {"did": did, "rev": rev, "baseid": baseid}
return flask.jsonify(ret), 200
def post_index_blank_record():
"""
Create a blank new record with only uploader and optionally
file_name fields filled
"""
uploader = flask.request.get_json().get("uploader")
file_name = flask.request.get_json().get("file_name")
if not uploader:
raise UserError("no uploader specified")
did, rev, baseid = blueprint.index_driver.add_blank_record(
uploader=uploader, file_name=file_name
)
ret = {"did": did, "rev": rev, "baseid": baseid}
return flask.jsonify(ret), 201
def update_all_index_record_versions(record):
"""
Update metadata for all record versions.
NOTE currently the only fields that can be updated for all versions are
(`authz`, `acl`).
"""
request_json = flask.request.get_json(force=True)
try:
jsonschema.validate(request_json, UPDATE_ALL_VERSIONS_SCHEMA)
except jsonschema.ValidationError as err:
logger.warning(f"Bad request body:\n{err}")
raise UserError(err)
acl = request_json.get("acl")
authz = request_json.get("authz")
# authorization and error handling done in driver
ret = blueprint.index_driver.update_all_versions(record, acl=acl, authz=authz)
return flask.jsonify(ret), 200
def replace_aliases(record):
"""
Replace all aliases associated with this DID / GUID
"""
# we set force=True so that if MIME type of request is not application/JSON,
# get_json will still throw a UserError.
aliases_json = flask.request.get_json(force=True)
try:
jsonschema.validate(aliases_json, RECORD_ALIAS_SCHEMA)
except jsonschema.ValidationError as err:
logger.warning(f"Bad request body:\n{err}")
raise UserError(err)
aliases = [record["value"] for record in aliases_json["aliases"]]
# authorization and error handling done in driver
blueprint.index_driver.replace_aliases_for_did(aliases, record)
aliases_payload = {"aliases": [{"value": alias} for alias in aliases]}
return flask.jsonify(aliases_payload), 200
def add_index_record_version(record):
'''
Add a record version
'''
try:
jsonschema.validate(flask.request.json, POST_RECORD_SCHEMA)
except jsonschema.ValidationError as err:
raise UserError(err)
new_did = flask.request.json.get('did')
form = flask.request.json['form']
size = flask.request.json['size']
urls = flask.request.json['urls']
acl = flask.request.json.get('acl', [])
hashes = flask.request.json['hashes']
file_name = flask.request.json.get('file_name')
metadata = flask.request.json.get('metadata')
urls_metadata = flask.request.json.get('urls_metadata')
version = flask.request.json.get('version')
did, baseid, rev = blueprint.index_driver.add_version(
record,
form,
new_did=new_did,
size=size,
urls=urls,
acl=acl,
file_name=file_name,
metadata=metadata,
urls_metadata=urls_metadata,
version=version,
hashes=hashes,
)
ret = {
'did': did,
'baseid': baseid,
'rev': rev,
}
return flask.jsonify(ret), 200