def create_session(mid, input, parser, aids, kwargs):
"""Create a cached upload session for the given model.
Parameters
----------
mid : string
ID of the model that will store data uploaded during the session.
Returns
-------
uid : string
A unique session identifier.
"""
database = slycat.web.server.database.couchdb.connect()
model = database.get("model", mid)
project = database.get("project", model["project"])
slycat.web.server.authentication.require_project_writer(project)
_start_session_cleanup_worker()
client = cherrypy.request.headers.get("x-forwarded-for")
cherrypy.log.error("Creating upload session for %s" % (client))
uid = uuid.uuid4().hex
with session_cache_lock:
session_cache[uid] = Session(uid, client, mid, input, parser, aids, kwargs)
return uid
def create_session(mid, input, parser, aids, kwargs):
"""Create a cached upload session for the given model.
Parameters
----------
mid : string
ID of the model that will store data uploaded during the session.
Returns
-------
uid : string
A unique session identifier.
"""
database = slycat.web.server.database.couchdb.connect()
model = database.get("model", mid)
project = database.get("project", model["project"])
slycat.web.server.authentication.require_project_writer(project)
_start_session_cleanup_worker()
client = cherrypy.request.headers.get("x-forwarded-for")
cherrypy.log.error("Creating upload session for %s" % (client))
uid = uuid.uuid4().hex
with session_cache_lock:
session_cache[uid] = Session(uid, client, mid, input, parser, aids,
kwargs)
return uid
def _parse_uploads(self):
cherrypy.log.error("Upload parsing started.")
database = slycat.web.server.database.couchdb.connect()
model = database.get("model", self._mid)
def numeric_order(x):
"""Files and file parts must be loaded in numeric, not lexicographical, order."""
return int(x.split("-")[-1])
files = []
storage = path(self._uid)
for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")),
key=numeric_order):
cherrypy.log.error("Assembling %s" % file_dir)
file = ""
for file_part in sorted(glob.glob(os.path.join(file_dir,
"part-*")),
key=numeric_order):
cherrypy.log.error(" Loading %s" % file_part)
with open(file_part, "r") as f:
file += f.read()
files.append(file)
try:
slycat.web.server.plugin.manager.parsers[self._parser]["parse"](
database, model, self._input, files, self._aids,
**self._kwargs)
except Exception as e:
cherrypy.log.error("Exception parsing posted files: %s" % e)
import traceback
cherrypy.log.error(traceback.format_exc())
cherrypy.log.error("Upload parsing finished.")
def _parse_uploads(self):
cherrypy.log.error("Upload parsing started.")
database = slycat.web.server.database.couchdb.connect()
model = database.get("model", self._mid)
def numeric_order(x):
"""Files and file parts must be loaded in numeric, not lexicographical, order."""
return int(x.split("-")[-1])
files = []
storage = path(self._uid)
for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")), key=numeric_order):
cherrypy.log.error("Assembling %s" % file_dir)
file = ""
for file_part in sorted(glob.glob(os.path.join(file_dir, "part-*")), key=numeric_order):
cherrypy.log.error(" Loading %s" % file_part)
with open(file_part, "r") as f:
file += f.read()
files.append(file)
try:
slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input, files, self._aids, **self._kwargs)
except Exception as e:
cherrypy.log.error("Exception parsing posted files: %s" % e)
import traceback
cherrypy.log.error(traceback.format_exc())
cherrypy.log.error("Upload parsing finished.")
def cache_object(pid, key, content_type, content):
cherrypy.log.error("cache_object %s %s %s" % (pid, key, content_type))
database = slycat.web.server.database.couchdb.connect()
project = database.get("project", pid)
slycat.web.server.authentication.require_project_reader(project)
lookup = pid + "-" + key
for cache_object in database.scan("slycat/project-key-cache-objects",
startkey=lookup,
endkey=lookup):
database.put_attachment(cache_object,
filename="content",
content_type=content_type,
content=content)
return
cache_object = {
"_id": uuid.uuid4().hex,
"type": "cache-object",
"project": pid,
"key": key,
"created": datetime.datetime.utcnow().isoformat(),
"creator": cherrypy.request.login,
}
database.save(cache_object)
database.put_attachment(cache_object,
filename="content",
content_type=content_type,
content=content)
def _parse_uploads(self):
"""
calls the parse function specified by the registered parser
:return: not used
"""
cherrypy.log.error("Upload parsing started.")
if self._mid not in parsing_locks:
parsing_locks[self._mid] = threading.Lock()
with parsing_locks[self._mid]:
cherrypy.log.error("got lock: %s" % self._mid)
database = slycat.web.server.database.couchdb.connect()
model = database.get("model", self._mid)
def numeric_order(x):
"""Files and file parts must be loaded in numeric, not lexicographical, order."""
return int(x.split("-")[-1])
# we no longer support multi file uploads
# TODO: need to decide if we should remove this or refactor for multi files again
files = []
storage = path(self._uid)
for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")), key=numeric_order):
cherrypy.log.error("Assembling %s" % file_dir)
file = ""
for file_part in sorted(glob.glob(os.path.join(file_dir, "part-*")), key=numeric_order):
cherrypy.log.error(" Loading %s" % file_part)
with open(file_part, "r") as f:
file += f.read()
files.append(file)
try:
# adding this check for backwards compatibility
# new way self._aids[0] is the file name being added to the model and hdf5
# self._aids[1] is the name of the file being pushed to the project_data data object
if isinstance(self._aids[0], list):
slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
files, self._aids[0], **self._kwargs)
else:
slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
files, self._aids, **self._kwargs)
slycat.web.server.handlers.create_project_data(self._mid, self._aids, files)
except Exception as e:
cherrypy.log.error("Exception parsing posted files: %s" % e)
import traceback
cherrypy.log.error(traceback.format_exc())
cherrypy.log.error("Upload parsing finished.")
def cache_object(pid, key, content_type, content):
cherrypy.log.error("cache_object %s %s %s" % (pid, key, content_type))
database = slycat.web.server.database.couchdb.connect()
project = database.get("project", pid)
slycat.web.server.authentication.require_project_reader(project)
lookup = pid + "-" + key
for cache_object in database.scan("slycat/project-key-cache-objects", startkey=lookup, endkey=lookup):
database.put_attachment(cache_object, filename="content", content_type=content_type, content=content)
return
cache_object = {
"_id": uuid.uuid4().hex,
"type": "cache-object",
"project": pid,
"key": key,
"created": datetime.datetime.utcnow().isoformat(),
"creator": cherrypy.request.login,
}
database.save(cache_object)
database.put_attachment(cache_object, filename="content", content_type=content_type, content=content)
def _parse_uploads(self):
"""
calls the parse function specified by the registered parser
:return: not used
"""
cherrypy.log.error("Upload parsing started.")
if self._mid not in parsing_locks:
parsing_locks[self._mid] = threading.Lock()
with parsing_locks[self._mid]:
#cherrypy.log.error("got lock: %s" % self._mid)
database = slycat.web.server.database.couchdb.connect()
model = database.get("model", self._mid)
def numeric_order(x):
"""Files and file parts must be loaded in numeric, not lexicographical, order."""
return int(x.split("-")[-1])
# we no longer support multi file uploads
# TODO: need to decide if we should remove this or refactor for multi files again
files = []
storage = path(self._uid)
for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")),
key=numeric_order):
# cherrypy.log.error("Assembling %s" % file_dir)
is_bin_file = False
file_parts = []
for file_part in sorted(glob.glob(
os.path.join(file_dir, "part-*")),
key=numeric_order):
# cherrypy.log.error(" Loading %s" % file_part)
# is this a text file?
if is_bin_file == False:
# try to open as text file
try:
with open(file_part, "r") as f:
file_parts.append(f.read())
# not a text file, open as binary
except UnicodeDecodeError:
is_bin_file = True
# is it a binary file?
if is_bin_file == True:
with open(file_part, "rb") as f:
file_parts.append(f.read())
# join file parts to make full file
if is_bin_file == False:
file = ''.join(file_parts)
else:
file = b''.join(file_parts)
files.append(file)
try:
# adding this check for backwards compatibility
# new way self._aids[0] is the file name being added to the model and hdf5
# self._aids[1] is the name of the file being pushed to the project_data data object
if isinstance(self._aids[0], list):
slycat.web.server.plugin.manager.parsers[
self._parser]["parse"](database, model, self._input,
files, self._aids[0],
**self._kwargs)
else:
slycat.web.server.plugin.manager.parsers[
self._parser]["parse"](database, model, self._input,
files, self._aids,
**self._kwargs)
if model["model-type"] == "parameter-image":
slycat.web.server.handlers.create_project_data(
self._mid, self._aids, files)
except Exception as e:
cherrypy.log.error("Exception parsing posted files: %s" % e)
import traceback
cherrypy.log.error(traceback.format_exc())
cherrypy.log.error("Upload parsing finished.")
