def _get(self, _id, min_role=None, filename=None, perm_only=False, dbc=None, dbc_name=None):
dbc = dbc or self.dbc
dbc_name = dbc_name or self.__class__.__name__
container = dbc.find_one({'_id': _id}, ['permissions'] if perm_only else None)
if not container:
self.abort(404, 'no such ' + dbc_name)
user_perm = util.user_perm(container['permissions'], self.uid, self.source_site)
if self.public_request:
ticket_id = self.request.GET.get('ticket')
if ticket_id:
ticket = self.app.db.downloads.find_one({'_id': ticket_id})
if not ticket: # FIXME need better security
self.abort(404, 'no such ticket')
if ticket['target'] != _id or ticket['filename'] != filename:
self.abort(400, 'ticket not for this resource')
elif not container.get('public', False):
self.abort(403, 'this ' + dbc_name + ' is not public')
del container['permissions']
elif not self.superuser_request:
if not user_perm:
self.abort(403, self.uid + ' does not have permissions on this ' + dbc_name)
if min_role and users.INTEGER_ROLES[user_perm['access']] < users.INTEGER_ROLES[min_role]:
self.abort(403, self.uid + ' does not have at least ' + min_role + ' permissions on this ' + dbc_name)
if user_perm['access'] != 'admin': # if not admin, mask permissions of other users
container['permissions'] = [user_perm]
if self.request.GET.get('paths', '').lower() in ('1', 'true'):
for fileinfo in container['files']:
fileinfo['path'] = str(_id)[-3:] + '/' + str(_id) + '/' + fileinfo['filename']
container['_id'] = str(container['_id'])
container.setdefault('timestamp', datetime.datetime.utcnow())
container['timestamp'], container['timezone'] = util.format_timestamp(container['timestamp'], container.get('timezone')) # TODO json serializer should do this
for note in container.get('notes', []):
note['timestamp'], _ = util.format_timestamp(note['timestamp']) # TODO json serializer should do this
return container, user_perm
def _get(self,
_id,
min_role=None,
filename=None,
perm_only=False,
dbc=None,
dbc_name=None):
dbc = dbc or self.dbc
dbc_name = dbc_name or self.__class__.__name__
container = dbc.find_one({'_id': _id},
['permissions'] if perm_only else None)
if not container:
self.abort(404, 'no such ' + dbc_name)
user_perm = util.user_perm(container['permissions'], self.uid,
self.source_site)
if self.public_request:
ticket_id = self.request.GET.get('ticket')
if ticket_id:
ticket = self.app.db.downloads.find_one({'_id': ticket_id})
if not ticket: # FIXME need better security
self.abort(404, 'no such ticket')
if ticket['target'] != _id or ticket['filename'] != filename:
self.abort(400, 'ticket not for this resource')
elif not container.get('public', False):
self.abort(403, 'this ' + dbc_name + ' is not public')
del container['permissions']
elif not self.superuser_request:
if not user_perm:
self.abort(
403, self.uid + ' does not have permissions on this ' +
dbc_name)
if min_role and users.INTEGER_ROLES[
user_perm['access']] < users.INTEGER_ROLES[min_role]:
self.abort(
403, self.uid + ' does not have at least ' + min_role +
' permissions on this ' + dbc_name)
if user_perm[
'access'] != 'admin': # if not admin, mask permissions of other users
container['permissions'] = [user_perm]
if self.request.GET.get('paths', '').lower() in ('1', 'true'):
for fileinfo in container['files']:
fileinfo['path'] = str(_id)[-3:] + '/' + str(
_id) + '/' + fileinfo['filename']
container['_id'] = str(container['_id'])
container.setdefault('timestamp', datetime.datetime.utcnow())
container['timestamp'], container['timezone'] = util.format_timestamp(
container['timestamp'],
container.get('timezone')) # TODO json serializer should do this
for note in container.get('notes', []):
note['timestamp'], _ = util.format_timestamp(
note['timestamp']) # TODO json serializer should do this
return container, user_perm
def post(self, gid):
"""Create a new Project."""
json_body = self._post()
group = self.app.db.groups.find_one({'_id': gid}, ['roles'])
if not group:
self.abort(400, 'invalid group id')
if not self.superuser_request and util.user_perm(group['roles'], self.uid).get('access') != 'admin':
self.abort(400, 'must be group admin to create project')
json_body['group'] = gid
json_body['permissions'] = group['roles']
json_body['public'] = json_body.get('public', False)
json_body['files'] = []
return {'_id': str(self.dbc.insert(json_body))}
def post(self, gid):
"""Create a new Project."""
json_body = self._post()
group = self.app.db.groups.find_one({'_id': gid}, ['roles'])
if not group:
self.abort(400, 'invalid group id')
if not self.superuser_request and util.user_perm(
group['roles'], self.uid).get('access') != 'admin':
self.abort(400, 'must be group admin to create project')
json_body['group'] = gid
json_body['permissions'] = group['roles']
json_body['public'] = json_body.get('public', False)
json_body['files'] = []
return {'_id': str(self.dbc.insert(json_body))}
def post(self, sid):
"""Create a new Acquisition."""
json_body = self._post()
_id = bson.ObjectId(sid)
session = self.app.db.sessions.find_one({'_id': _id}, ['permissions', 'public'])
if not session:
self.abort(404, 'no such session')
if not self.superuser_request and util.user_perm(session['permissions'], self.uid).get('access') != 'admin':
self.abort(400, 'must be session admin to create acquisition')
json_body['session'] = _id
json_body['permissions'] = session['permissions']
json_body['public'] = session.get('public', False)
json_body['files'] = []
if 'timestamp' in json_body:
json_body['timestamp'] = util.parse_timestamp(json_body['timestamp'])
return {'_id': str(self.dbc.insert(json_body))}
def put(self, _id):
"""Update an existing Group."""
group = self.dbc.find_one({'_id': _id})
if not group:
self.abort(404, 'no such Group')
user_perm = util.user_perm(group.get('roles', []), self.uid)
if not self.superuser_request and not user_perm.get('access') == 'admin':
self.abort(403, 'must be superuser or group admin to update group')
schema = copy.deepcopy(self.json_schema)
del schema['required']
try:
json_body = self.request.json_body
jsonschema.validate(json_body, schema)
except (ValueError, jsonschema.ValidationError) as e:
self.abort(400, str(e))
self.dbc.update({'_id': _id}, {'$set': util.mongo_dict(json_body)})
def post(self, pid):
"""Create a new Session."""
json_body = self._post()
_id = bson.ObjectId(pid)
project = self.app.db.projects.find_one({'_id': _id}, ['group', 'permissions', 'public'])
if not project:
self.abort(404, 'no such project')
if not self.superuser_request and util.user_perm(project['permissions'], self.uid).get('access') != 'admin':
self.abort(400, 'must be project admin to create session')
json_body['project'] = _id
json_body['group'] = project['group']
json_body['permissions'] = project['permissions']
json_body['public'] = project.get('public', False)
json_body['files'] = []
if 'timestamp' in json_body:
json_body['timestamp'] = util.parse_timestamp(json_body['timestamp'])
return {'_id': str(self.dbc.insert(json_body))}
def post(self, sid):
"""Create a new Acquisition."""
json_body = self._post()
_id = bson.ObjectId(sid)
session = self.app.db.sessions.find_one({'_id': _id},
['permissions', 'public'])
if not session:
self.abort(404, 'no such session')
if not self.superuser_request and util.user_perm(
session['permissions'], self.uid).get('access') != 'admin':
self.abort(400, 'must be session admin to create acquisition')
json_body['session'] = _id
json_body['permissions'] = session['permissions']
json_body['public'] = session.get('public', False)
json_body['files'] = []
if 'timestamp' in json_body:
json_body['timestamp'] = util.parse_timestamp(
json_body['timestamp'])
return {'_id': str(self.dbc.insert(json_body))}
def post(self, pid):
"""Create a new Session."""
json_body = self._post()
_id = bson.ObjectId(pid)
project = self.app.db.projects.find_one(
{'_id': _id}, ['group', 'permissions', 'public'])
if not project:
self.abort(404, 'no such project')
if not self.superuser_request and util.user_perm(
project['permissions'], self.uid).get('access') != 'admin':
self.abort(400, 'must be project admin to create session')
json_body['project'] = _id
json_body['group'] = project['group']
json_body['permissions'] = project['permissions']
json_body['public'] = project.get('public', False)
json_body['files'] = []
if 'timestamp' in json_body:
json_body['timestamp'] = util.parse_timestamp(
json_body['timestamp'])
return {'_id': str(self.dbc.insert(json_body))}
def upload(self):
"""
Recieve a multi-file upload.
3 phases:
1 - upload metadata, obtain upload ticket
2 - upload files, one at a time, but in parallel
3 - send a 'complete' message
"""
def store_file(fd, filename, md5, arcpath, arcname):
with tempfile.TemporaryDirectory(prefix='.tmp', dir=self.app.config['upload_path']) as tempdir_path:
filepath = os.path.join(tempdir_path, filename)
success, _, _, _ = util.receive_stream_and_validate(fd, filepath, md5)
if not success:
self.abort(400, 'Content-MD5 mismatch.')
with lockfile.LockFile(arcpath):
with tarfile.open(arcpath, 'a') as archive:
archive.add(filepath, os.path.join(arcname, filename))
if self.public_request:
self.abort(403, 'must be logged in to upload data')
filename = self.request.GET.get('filename')
ticket_id = self.request.GET.get('ticket')
if not ticket_id:
if filename != 'METADATA.json':
self.abort(400, 'first file must be METADATA.json')
try:
json_body = self.request.json_body
jsonschema.validate(json_body, UPLOAD_SCHEMA)
except (ValueError, jsonschema.ValidationError) as e:
self.abort(400, str(e))
filetype = json_body['filetype']
overwrites = json_body['overwrite']
query = {'name': overwrites['project_name'], 'group': overwrites['group_name']}
project = self.app.db.projects.find_one(query) # verify permissions
if not self.superuser_request:
user_perm = util.user_perm(project['permissions'], self.uid)
if not user_perm:
self.abort(403, self.uid + ' does not have permissions on this project')
if users.INTEGER_ROLES[user_perm['access']] < users.INTEGER_ROLES['rw']:
self.abort(403, self.uid + ' does not have at least ' + min_role + ' permissions on this project')
acq_no = overwrites.get('acq_no')
arcname = overwrites['series_uid'] + ('_' + str(acq_no) if acq_no is not None else '') + '_' + filetype
ticket = util.upload_ticket(arcname=arcname) # store arcname for later reference
self.app.db.uploads.insert_one(ticket)
arcpath = os.path.join(self.app.config['upload_path'], ticket['_id'] + '.tar')
store_file(self.request.body_file, filename, self.request.headers['Content-MD5'], arcpath, arcname)
return {'ticket': ticket['_id']}
ticket = self.app.db.uploads.find_one({'_id': ticket_id})
if not ticket:
self.abort(404, 'no such ticket')
arcpath = os.path.join(self.app.config['upload_path'], ticket_id + '.tar')
if self.request.GET.get('complete', '').lower() not in ('1', 'true'):
if 'Content-MD5' not in self.request.headers:
self.app.db.uploads.remove({'_id': ticket_id}) # delete ticket
self.abort(400, 'Request must contain a valid "Content-MD5" header.')
if not filename:
self.app.db.uploads.remove({'_id': ticket_id}) # delete ticket
self.abort(400, 'Request must contain a filename query parameter.')
self.app.db.uploads.update_one({'_id': ticket_id}, {'$set': {'timestamp': datetime.datetime.utcnow()}}) # refresh ticket
store_file(self.request.body_file, filename, self.request.headers['Content-MD5'], arcpath, ticket['arcname'])
else: # complete -> zip, hash, commit
filepath = arcpath[:-2] + 'gz'
with gzip.open(filepath, 'wb', compresslevel=6) as gzfile:
with open(arcpath) as rawfile:
gzfile.writelines(rawfile)
os.remove(arcpath)
sha1 = hashlib.sha1()
with open(filepath, 'rb') as fd:
for chunk in iter(lambda: fd.read(2**20), ''):
sha1.update(chunk)
datainfo = util.parse_file(filepath, sha1.hexdigest())
if datainfo is None:
util.quarantine_file(filepath, self.app.config['quarantine_path'])
self.abort(202, 'Quarantining %s (unparsable)' % filename)
util.commit_file(self.app.db.acquisitions, None, datainfo, filepath, self.app.config['data_path'])