def create_session(mid, input, parser, aids, kwargs): """Create a cached upload session for the given model. Parameters ---------- mid : string ID of the model that will store data uploaded during the session. Returns ------- uid : string A unique session identifier. """ database = slycat.web.server.database.couchdb.connect() model = database.get("model", mid) project = database.get("project", model["project"]) slycat.web.server.authentication.require_project_writer(project) _start_session_cleanup_worker() client = cherrypy.request.headers.get("x-forwarded-for") cherrypy.log.error("Creating upload session for %s" % (client)) uid = uuid.uuid4().hex with session_cache_lock: session_cache[uid] = Session(uid, client, mid, input, parser, aids, kwargs) return uid
def _parse_uploads(self): cherrypy.log.error("Upload parsing started.") database = slycat.web.server.database.couchdb.connect() model = database.get("model", self._mid) def numeric_order(x): """Files and file parts must be loaded in numeric, not lexicographical, order.""" return int(x.split("-")[-1]) files = [] storage = path(self._uid) for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")), key=numeric_order): cherrypy.log.error("Assembling %s" % file_dir) file = "" for file_part in sorted(glob.glob(os.path.join(file_dir, "part-*")), key=numeric_order): cherrypy.log.error(" Loading %s" % file_part) with open(file_part, "r") as f: file += f.read() files.append(file) try: slycat.web.server.plugin.manager.parsers[self._parser]["parse"]( database, model, self._input, files, self._aids, **self._kwargs) except Exception as e: cherrypy.log.error("Exception parsing posted files: %s" % e) import traceback cherrypy.log.error(traceback.format_exc()) cherrypy.log.error("Upload parsing finished.")
def _parse_uploads(self): cherrypy.log.error("Upload parsing started.") database = slycat.web.server.database.couchdb.connect() model = database.get("model", self._mid) def numeric_order(x): """Files and file parts must be loaded in numeric, not lexicographical, order.""" return int(x.split("-")[-1]) files = [] storage = path(self._uid) for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")), key=numeric_order): cherrypy.log.error("Assembling %s" % file_dir) file = "" for file_part in sorted(glob.glob(os.path.join(file_dir, "part-*")), key=numeric_order): cherrypy.log.error(" Loading %s" % file_part) with open(file_part, "r") as f: file += f.read() files.append(file) try: slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input, files, self._aids, **self._kwargs) except Exception as e: cherrypy.log.error("Exception parsing posted files: %s" % e) import traceback cherrypy.log.error(traceback.format_exc()) cherrypy.log.error("Upload parsing finished.")
def cache_object(pid, key, content_type, content): cherrypy.log.error("cache_object %s %s %s" % (pid, key, content_type)) database = slycat.web.server.database.couchdb.connect() project = database.get("project", pid) slycat.web.server.authentication.require_project_reader(project) lookup = pid + "-" + key for cache_object in database.scan("slycat/project-key-cache-objects", startkey=lookup, endkey=lookup): database.put_attachment(cache_object, filename="content", content_type=content_type, content=content) return cache_object = { "_id": uuid.uuid4().hex, "type": "cache-object", "project": pid, "key": key, "created": datetime.datetime.utcnow().isoformat(), "creator": cherrypy.request.login, } database.save(cache_object) database.put_attachment(cache_object, filename="content", content_type=content_type, content=content)
def _parse_uploads(self): """ calls the parse function specified by the registered parser :return: not used """ cherrypy.log.error("Upload parsing started.") if self._mid not in parsing_locks: parsing_locks[self._mid] = threading.Lock() with parsing_locks[self._mid]: cherrypy.log.error("got lock: %s" % self._mid) database = slycat.web.server.database.couchdb.connect() model = database.get("model", self._mid) def numeric_order(x): """Files and file parts must be loaded in numeric, not lexicographical, order.""" return int(x.split("-")[-1]) # we no longer support multi file uploads # TODO: need to decide if we should remove this or refactor for multi files again files = [] storage = path(self._uid) for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")), key=numeric_order): cherrypy.log.error("Assembling %s" % file_dir) file = "" for file_part in sorted(glob.glob(os.path.join(file_dir, "part-*")), key=numeric_order): cherrypy.log.error(" Loading %s" % file_part) with open(file_part, "r") as f: file += f.read() files.append(file) try: # adding this check for backwards compatibility # new way self._aids[0] is the file name being added to the model and hdf5 # self._aids[1] is the name of the file being pushed to the project_data data object if isinstance(self._aids[0], list): slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input, files, self._aids[0], **self._kwargs) else: slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input, files, self._aids, **self._kwargs) slycat.web.server.handlers.create_project_data(self._mid, self._aids, files) except Exception as e: cherrypy.log.error("Exception parsing posted files: %s" % e) import traceback cherrypy.log.error(traceback.format_exc()) cherrypy.log.error("Upload parsing finished.")
def _parse_uploads(self): """ calls the parse function specified by the registered parser :return: not used """ cherrypy.log.error("Upload parsing started.") if self._mid not in parsing_locks: parsing_locks[self._mid] = threading.Lock() with parsing_locks[self._mid]: #cherrypy.log.error("got lock: %s" % self._mid) database = slycat.web.server.database.couchdb.connect() model = database.get("model", self._mid) def numeric_order(x): """Files and file parts must be loaded in numeric, not lexicographical, order.""" return int(x.split("-")[-1]) # we no longer support multi file uploads # TODO: need to decide if we should remove this or refactor for multi files again files = [] storage = path(self._uid) for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")), key=numeric_order): # cherrypy.log.error("Assembling %s" % file_dir) is_bin_file = False file_parts = [] for file_part in sorted(glob.glob( os.path.join(file_dir, "part-*")), key=numeric_order): # cherrypy.log.error(" Loading %s" % file_part) # is this a text file? if is_bin_file == False: # try to open as text file try: with open(file_part, "r") as f: file_parts.append(f.read()) # not a text file, open as binary except UnicodeDecodeError: is_bin_file = True # is it a binary file? if is_bin_file == True: with open(file_part, "rb") as f: file_parts.append(f.read()) # join file parts to make full file if is_bin_file == False: file = ''.join(file_parts) else: file = b''.join(file_parts) files.append(file) try: # adding this check for backwards compatibility # new way self._aids[0] is the file name being added to the model and hdf5 # self._aids[1] is the name of the file being pushed to the project_data data object if isinstance(self._aids[0], list): slycat.web.server.plugin.manager.parsers[ self._parser]["parse"](database, model, self._input, files, self._aids[0], **self._kwargs) else: slycat.web.server.plugin.manager.parsers[ self._parser]["parse"](database, model, self._input, files, self._aids, **self._kwargs) if model["model-type"] == "parameter-image": slycat.web.server.handlers.create_project_data( self._mid, self._aids, files) except Exception as e: cherrypy.log.error("Exception parsing posted files: %s" % e) import traceback cherrypy.log.error(traceback.format_exc()) cherrypy.log.error("Upload parsing finished.")