def get_cdmi_dir_obj(path):
"""Get a directory entry through CDMI.
Get the listing of a directory.
TODO: find a way to stream the listing.
"""
cdmi_filters = []
if request_wants(ContentTypes.cdmi_object):
try:
cdmi_filters = _get_cdmi_filters(request.args)
except MalformedArgumentValueException as e:
return e.msg, 400
try:
dir_gen = storage.ls(path)
except storage.NotFoundException as e:
return e.msg, 404
except storage.NotAuthorizedException as e:
return e.msg, 401
except storage.StorageException as e:
return e.msg, 500
except storage.MalformedPathException as e:
return e.msg, 400
if request_wants(ContentTypes.cdmi_object):
cdmi_json_gen = _get_cdmi_json_dir_generator(path, dir_gen)
if cdmi_filters:
filtered_gen = ((a, b(cdmi_filters[a])) for a, b in cdmi_json_gen
if a in cdmi_filters)
else:
filtered_gen = ((a, b()) for a, b in cdmi_json_gen)
json_stream_wrapper = _wrap_with_json_generator(filtered_gen)
return Response(stream_with_context(json_stream_wrapper))
elif request_wants(ContentTypes.json):
dir_gen_wrapper = _create_dirlist_gen(dir_gen, path)
json_stream_wrapper = _wrap_with_json_generator(dir_gen_wrapper)
return Response(stream_with_context(json_stream_wrapper))
else:
return render_template('dirlisting.html',
dirlist=dir_gen,
path=path,
path_links=create_path_links(path),
parent_path=common.split_path(path)[0])
def get_request(request_id):
"""Poll the status of a request by ID."""
r = RegistrationRequest.query.get(request_id)
#TODO: json error?
if r is None:
return abort(404)
if request_wants(ContentTypes.json):
return flask.jsonify(
{'request': RegistrationRequestSerializer(r).data})
return flask.render_template('singleRequest.html', r=r)
def get_requests():
"""Get a requests list."""
page = int(request.args.get('page', '1'))
reg_requests = RegistrationRequest.query.order_by(
RegistrationRequest.timestamp.desc()).paginate(page,
current_app.config[
'REQUESTS_PER_PAGE'],
False)
if request_wants(ContentTypes.json):
return flask.jsonify(
{"requests": RegistrationRequestSerializer(reg_requests.items,
many=True).data,
"_links": get_hal_links(reg_requests, page)})
return flask.render_template('requests.html', requests=reg_requests)
def get_request(request_id):
"""Poll the status of a request by ID."""
r = RegistrationRequest.query.get(request_id)
#TODO: json error?
if r is None:
return abort(404)
if request_wants(ContentTypes.json):
return flask.jsonify(
{'request': RegistrationRequestSerializer(r).data})
return flask.render_template('singleRequest.html',
scratch=current_app.config.get(
'SCRATCH_SPACE', None),
r=r)
def post_request():
"""Submit a new registration request
Specify in the message body:
src: url of the source file
checksum: the file you expect the file will have.
The function returns a URL to check the status of the request.
The URL includes a request ID.
"""
current_app.logger.debug('Entering post_request()')
if flask.request.headers.get('Content-Type') == 'application/json':
req_body = json.loads(flask.request.data)
else:
req_body = flask.request.form
registration_request = \
RegistrationRequest(src_url=req_body['src_url'],
status_description='Registration request created',
timestamp=datetime.utcnow())
db.session.add(registration_request)
db.session.commit()
context = Context()
context.request_id = registration_request.id
context.auth = HTTPBasicAuth(request.authorization.username,
request.authorization.password)
context.src_url, context.md_url = extract_urls(req_body['src_url'])
current_app.logger.debug('Adding task %s ' % context)
db.session.close()
add_task(context)
if request_wants(ContentTypes.json):
return flask.jsonify(request_id=registration_request.id), 201
else:
return flask.render_template('requestcreated.html',
scratch=current_app.config.get(
'SCRATCH_SPACE', None),
reg=registration_request), 201
def post_request():
"""Submit a new registration request
Specify in the message body:
src: url of the source file
checksum: the file you expect the file will have.
The function returns a URL to check the status of the request.
The URL includes a request ID.
"""
current_app.logger.debug('Entering post_request()')
if flask.request.headers.get('Content-Type') == 'application/json':
req_body = json.loads(flask.request.data)
else:
req_body = flask.request.form
registration_request = \
RegistrationRequest(src_url=req_body['src_url'],
status_description='Registration request created',
timestamp=datetime.utcnow())
db.session.add(registration_request)
db.session.commit()
context = Context()
context.request_id = registration_request.id
context.auth = HTTPBasicAuth(request.authorization.username, request
.authorization.password)
context.src_url, context.md_url = extract_urls(req_body['src_url'])
current_app.logger.debug('Adding task %s ' % context)
db.session.close()
add_task(context)
if request_wants(ContentTypes.json):
return flask.jsonify(request_id=registration_request.id), 201
else:
return flask.render_template(
'requestcreated.html',
scratch=current_app.config.get('SCRATCH_SPACE', None),
reg=registration_request), 201
def get_requests():
"""Get a requests list."""
page = int(request.args.get('page', '1'))
reg_requests = RegistrationRequest.query.order_by(
RegistrationRequest.timestamp.desc()).paginate(
page, current_app.config['REQUESTS_PER_PAGE'], False)
if request_wants(ContentTypes.json):
return flask.jsonify({
"requests":
RegistrationRequestSerializer(reg_requests.items, many=True).data,
"_links":
get_hal_links(reg_requests, page)
})
src = request.args.get('src', '')
return flask.render_template('requests.html',
scratch=current_app.config.get(
'SCRATCH_SPACE', None),
requests=reg_requests,
src=src)
def get_cdmi_file_obj(path):
"""Get a file from storage through CDMI.
We might want to implement 3rd party copy in
pull mode here later. That can make introduce
problems with metadata handling, though.
"""
def parse_range(range_str):
start, end = range_str.split('-')
try:
start = int(start)
except:
start = storage.START
try:
end = int(end)
except:
end = storage.END
return start, end
range_requests = []
cdmi_filters = []
if request_wants(ContentTypes.cdmi_object):
try:
cdmi_filters = _get_cdmi_filters(request.args)
except MalformedArgumentValueException as e:
return e.msg, 400
try:
range_requests = cdmi_filters['value']
except KeyError:
pass
elif request.headers.get('Range'):
ranges = request.headers.get('Range')
range_regex = re.compile('(\d*-\d*)')
matches = range_regex.findall(ranges)
range_requests = map(parse_range, matches)
try:
(stream_gen,
file_size,
content_len,
range_list) = storage.read(path, range_requests)
except storage.IsDirException as e:
params = urlparse(request.url).query
return redirect('%s/?%s' % (path, params))
except storage.NotFoundException as e:
return e.msg, 404
except storage.NotAuthorizedException as e:
return e.msg, 401
except storage.MalformedPathException as e:
return e.msg, 400
response_headers = {'Content-Length': content_len}
# do not send the content-length to enable
# transfer-encoding chunked -- do not use chunked to let it
# work with ROOT, no effect on mem usage anyway
# Do not try to guess the type
#response_headers['Content-Type'] = 'application/octet-stream'
response_status = 200
multipart = False
if file_size != content_len:
response_status = 206
if len(range_list) > 1:
multipart = True
else:
response_headers['Content-Range'] = ('bytes %d-%d/%d'
% (range_list[0][0],
range_list[0][1],
file_size))
multipart_frontier = 'frontier'
if multipart:
del response_headers['Content-Length']
response_headers['Content-Type'] = ('multipart/byteranges; boundary=%s'
% multipart_frontier)
def wrap_multipart_stream_gen(stream_gen, delim, file_size):
multipart = False
for segment_size, segment_start, segment_end, data in stream_gen:
if segment_size:
multipart = True
current_app.logger.debug('started a multipart segment')
#yield '\n--%s\n\n%s' % (delim, data)
yield ('\n--%s\n'
'Content-Length: %d\n'
'Content-Range: bytes %d-%d/%d\n'
'\n%s') % (delim, segment_size,
segment_start, segment_end,
file_size, data)
#% (delim, segment_size, data)
else:
yield data
if multipart:
yield '\n--%s--\n' % delim
# yield 'epilogue'
wrapped_stream_gen = wrap_multipart_stream_gen(stream_gen,
multipart_frontier,
file_size)
if request_wants(ContentTypes.cdmi_object):
if len(range_list) > 1:
pass # throw a terrifying exception here, cdmi must not multipart!
cdmi_json_gen = _get_cdmi_json_file_generator(path,
wrapped_stream_gen,
file_size)
if cdmi_filters:
filtered_gen = ((a, b(cdmi_filters[a])) for a, b in cdmi_json_gen
if a in cdmi_filters)
else:
filtered_gen = ((a, b()) for a, b in cdmi_json_gen)
json_stream_wrapper = _wrap_with_json_generator(filtered_gen)
return Response(stream_with_context(json_stream_wrapper))
return Response(stream_with_context(wrapped_stream_gen),
headers=response_headers,
status=response_status)
