Exemplo n.º 1
0
def create_session(mid, input, parser, aids, kwargs):
  """Create a cached upload session for the given model.

  Parameters
  ----------
  mid : string
    ID of the model that will store data uploaded during the session.

  Returns
  -------
  uid : string
    A unique session identifier.
  """
  database = slycat.web.server.database.couchdb.connect()
  model = database.get("model", mid)
  project = database.get("project", model["project"])
  slycat.web.server.authentication.require_project_writer(project)

  _start_session_cleanup_worker()

  client = cherrypy.request.headers.get("x-forwarded-for")
  cherrypy.log.error("Creating upload session for %s" % (client))

  uid = uuid.uuid4().hex
  with session_cache_lock:
    session_cache[uid] = Session(uid, client, mid, input, parser, aids, kwargs)
  return uid
Exemplo n.º 2
0
def create_session(mid, input, parser, aids, kwargs):
    """Create a cached upload session for the given model.

  Parameters
  ----------
  mid : string
    ID of the model that will store data uploaded during the session.

  Returns
  -------
  uid : string
    A unique session identifier.
  """
    database = slycat.web.server.database.couchdb.connect()
    model = database.get("model", mid)
    project = database.get("project", model["project"])
    slycat.web.server.authentication.require_project_writer(project)

    _start_session_cleanup_worker()

    client = cherrypy.request.headers.get("x-forwarded-for")
    cherrypy.log.error("Creating upload session for %s" % (client))

    uid = uuid.uuid4().hex
    with session_cache_lock:
        session_cache[uid] = Session(uid, client, mid, input, parser, aids,
                                     kwargs)
    return uid
Exemplo n.º 3
0
    def _parse_uploads(self):
        cherrypy.log.error("Upload parsing started.")

        database = slycat.web.server.database.couchdb.connect()
        model = database.get("model", self._mid)

        def numeric_order(x):
            """Files and file parts must be loaded in numeric, not lexicographical, order."""
            return int(x.split("-")[-1])

        files = []
        storage = path(self._uid)
        for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")),
                               key=numeric_order):
            cherrypy.log.error("Assembling %s" % file_dir)
            file = ""
            for file_part in sorted(glob.glob(os.path.join(file_dir,
                                                           "part-*")),
                                    key=numeric_order):
                cherrypy.log.error(" Loading %s" % file_part)
                with open(file_part, "r") as f:
                    file += f.read()
            files.append(file)

        try:
            slycat.web.server.plugin.manager.parsers[self._parser]["parse"](
                database, model, self._input, files, self._aids,
                **self._kwargs)
        except Exception as e:
            cherrypy.log.error("Exception parsing posted files: %s" % e)
            import traceback
            cherrypy.log.error(traceback.format_exc())

        cherrypy.log.error("Upload parsing finished.")
Exemplo n.º 4
0
  def _parse_uploads(self):
    cherrypy.log.error("Upload parsing started.")

    database = slycat.web.server.database.couchdb.connect()
    model = database.get("model", self._mid)

    def numeric_order(x):
      """Files and file parts must be loaded in numeric, not lexicographical, order."""
      return int(x.split("-")[-1])

    files = []
    storage = path(self._uid)
    for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")), key=numeric_order):
      cherrypy.log.error("Assembling %s" % file_dir)
      file = ""
      for file_part in sorted(glob.glob(os.path.join(file_dir, "part-*")), key=numeric_order):
        cherrypy.log.error(" Loading %s" % file_part)
        with open(file_part, "r") as f:
          file += f.read()
      files.append(file)

    try:
      slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input, files, self._aids, **self._kwargs)
    except Exception as e:
      cherrypy.log.error("Exception parsing posted files: %s" % e)
      import traceback
      cherrypy.log.error(traceback.format_exc())

    cherrypy.log.error("Upload parsing finished.")
Exemplo n.º 5
0
def cache_object(pid, key, content_type, content):
    cherrypy.log.error("cache_object %s %s %s" % (pid, key, content_type))
    database = slycat.web.server.database.couchdb.connect()
    project = database.get("project", pid)
    slycat.web.server.authentication.require_project_reader(project)

    lookup = pid + "-" + key
    for cache_object in database.scan("slycat/project-key-cache-objects",
                                      startkey=lookup,
                                      endkey=lookup):
        database.put_attachment(cache_object,
                                filename="content",
                                content_type=content_type,
                                content=content)
        return

    cache_object = {
        "_id": uuid.uuid4().hex,
        "type": "cache-object",
        "project": pid,
        "key": key,
        "created": datetime.datetime.utcnow().isoformat(),
        "creator": cherrypy.request.login,
    }
    database.save(cache_object)
    database.put_attachment(cache_object,
                            filename="content",
                            content_type=content_type,
                            content=content)
Exemplo n.º 6
0
  def _parse_uploads(self):
    """
    calls the parse function specified by the registered parser
    :return: not used
    """
    cherrypy.log.error("Upload parsing started.")

    if self._mid not in parsing_locks:
      parsing_locks[self._mid] = threading.Lock()

    with parsing_locks[self._mid]:
      cherrypy.log.error("got lock: %s" % self._mid)
      database = slycat.web.server.database.couchdb.connect()
      model = database.get("model", self._mid)

      def numeric_order(x):
        """Files and file parts must be loaded in numeric, not lexicographical, order."""
        return int(x.split("-")[-1])

      # we no longer support multi file uploads 
      # TODO: need to decide if we should remove this or refactor for multi files again
      files = []
      storage = path(self._uid)
      for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")), key=numeric_order):
        cherrypy.log.error("Assembling %s" % file_dir)
        file = ""
        for file_part in sorted(glob.glob(os.path.join(file_dir, "part-*")), key=numeric_order):
          cherrypy.log.error(" Loading %s" % file_part)
          with open(file_part, "r") as f:
            file += f.read()
        files.append(file)

      try:
      # adding this check for backwards compatibility 
      # new way self._aids[0] is the file name being added to the model and hdf5
      # self._aids[1] is the name of the file being pushed to the project_data data object
        if isinstance(self._aids[0], list):
          slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
                                                                          files, self._aids[0], **self._kwargs)
        else:
          slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
                                                                          files, self._aids, **self._kwargs)
        slycat.web.server.handlers.create_project_data(self._mid, self._aids, files)
      except Exception as e:
        cherrypy.log.error("Exception parsing posted files: %s" % e)
        import traceback
        cherrypy.log.error(traceback.format_exc())
      cherrypy.log.error("Upload parsing finished.")
Exemplo n.º 7
0
def cache_object(pid, key, content_type, content):
  cherrypy.log.error("cache_object %s %s %s" % (pid, key, content_type))
  database = slycat.web.server.database.couchdb.connect()
  project = database.get("project", pid)
  slycat.web.server.authentication.require_project_reader(project)

  lookup = pid + "-" + key
  for cache_object in database.scan("slycat/project-key-cache-objects", startkey=lookup, endkey=lookup):
    database.put_attachment(cache_object, filename="content", content_type=content_type, content=content)
    return

  cache_object = {
    "_id": uuid.uuid4().hex,
    "type": "cache-object",
    "project": pid,
    "key": key,
    "created": datetime.datetime.utcnow().isoformat(),
    "creator": cherrypy.request.login,
  }
  database.save(cache_object)
  database.put_attachment(cache_object, filename="content", content_type=content_type, content=content)
Exemplo n.º 8
0
    def _parse_uploads(self):
        """
    calls the parse function specified by the registered parser
    :return: not used
    """
        cherrypy.log.error("Upload parsing started.")

        if self._mid not in parsing_locks:
            parsing_locks[self._mid] = threading.Lock()

        with parsing_locks[self._mid]:
            #cherrypy.log.error("got lock: %s" % self._mid)
            database = slycat.web.server.database.couchdb.connect()
            model = database.get("model", self._mid)

            def numeric_order(x):
                """Files and file parts must be loaded in numeric, not lexicographical, order."""
                return int(x.split("-")[-1])

            # we no longer support multi file uploads
            # TODO: need to decide if we should remove this or refactor for multi files again
            files = []
            storage = path(self._uid)
            for file_dir in sorted(glob.glob(os.path.join(storage, "file-*")),
                                   key=numeric_order):
                # cherrypy.log.error("Assembling %s" % file_dir)

                is_bin_file = False
                file_parts = []

                for file_part in sorted(glob.glob(
                        os.path.join(file_dir, "part-*")),
                                        key=numeric_order):
                    # cherrypy.log.error(" Loading %s" % file_part)

                    # is this a text file?
                    if is_bin_file == False:

                        # try to open as text file
                        try:
                            with open(file_part, "r") as f:
                                file_parts.append(f.read())

                        # not a text file, open as binary
                        except UnicodeDecodeError:
                            is_bin_file = True

                    # is it a binary file?
                    if is_bin_file == True:
                        with open(file_part, "rb") as f:
                            file_parts.append(f.read())

                # join file parts to make full file
                if is_bin_file == False:
                    file = ''.join(file_parts)
                else:
                    file = b''.join(file_parts)

                files.append(file)

            try:
                # adding this check for backwards compatibility
                # new way self._aids[0] is the file name being added to the model and hdf5
                # self._aids[1] is the name of the file being pushed to the project_data data object
                if isinstance(self._aids[0], list):
                    slycat.web.server.plugin.manager.parsers[
                        self._parser]["parse"](database, model, self._input,
                                               files, self._aids[0],
                                               **self._kwargs)
                else:
                    slycat.web.server.plugin.manager.parsers[
                        self._parser]["parse"](database, model, self._input,
                                               files, self._aids,
                                               **self._kwargs)
                if model["model-type"] == "parameter-image":
                    slycat.web.server.handlers.create_project_data(
                        self._mid, self._aids, files)
            except Exception as e:
                cherrypy.log.error("Exception parsing posted files: %s" % e)
                import traceback
                cherrypy.log.error(traceback.format_exc())
            cherrypy.log.error("Upload parsing finished.")