Exemple #1
0
 def upload(self, file, parser):
     filepath = os.path.join(app.dropbox_path, file.filename)
     _id = save_file_metadata(filepath, status="uploading", parser=parser)
     filepath = add_id(_id, filepath)
     file.save(filepath)
     log.info("uploaded {} as {}".format(file.filename, filepath))
     _id = save_file_metadata(filepath, status="uploaded")
     self.event.set()
     return _id
Exemple #2
0
 def upload(self, file, parser):
     filepath = os.path.join(app.dropbox_path, file.filename)
     _id = save_file_metadata(filepath, status='uploading', parser=parser)
     filepath = add_id(_id, filepath)
     file.save(filepath)
     log.info('uploaded {} as {}'.format(file.filename, filepath))
     _id = save_file_metadata(filepath, status='uploaded')
     self.event.set()
     return _id
Exemple #3
0
    def process(self, filepath):
        # ignore hidden files (e.g., .gitignore)
        if not os.path.basename(filepath)[0] == '.':
            try:
                processed = self.files[filepath]
            except KeyError:
                log.info(
                    'new file in dropbox %s created %s', filepath,
                    datetime.datetime.fromtimestamp(
                        os.path.getctime(filepath)))
                processed = False

            if not processed and not os.path.basename(
                    filepath) == '.gitignore':
                try:
                    _id = ingest.ingest(filepath)

                    # move file to ingested directory
                    dest = add_id(
                        _id,
                        os.path.join(app.ingested_path,
                                     os.path.basename(filepath)))
                    log.info('moving ingested file from %s to %s', filepath,
                             dest)
                    shutil.move(filepath, dest)

                except Exception as e:
                    log.warn('failed to ingest %s', filepath)
                    log.warn(e)

                    _id = save_file_metadata(filepath, status='error')

                    # move file to failed directory
                    dest = add_id(
                        _id,
                        os.path.join(app.failed_path,
                                     os.path.basename(filepath)))
                    log.info('moving failed file from %s to %s', filepath,
                             dest)
                    shutil.move(filepath, dest)

                processed = True

                self.files[filepath] = processed
Exemple #4
0
    def process(self, filepath):
        # ignore hidden files (e.g., .gitignore)
        if not os.path.basename(filepath)[0] == ".":
            try:
                processed = self.files[filepath]
            except KeyError:
                log.info(
                    "new file in dropbox %s created %s",
                    filepath,
                    datetime.datetime.fromtimestamp(os.path.getctime(filepath)),
                )
                processed = False

            if not processed and not os.path.basename(filepath) == ".gitignore":
                try:
                    _id = ingest.ingest(filepath)

                    # move file to ingested directory
                    dest = add_id(_id, os.path.join(app.ingested_path, os.path.basename(filepath)))
                    log.info("moving ingested file from %s to %s", filepath, dest)
                    shutil.move(filepath, dest)

                except Exception as e:
                    log.warn("failed to ingest %s", filepath)
                    log.warn(e)

                    _id = save_file_metadata(filepath, status="error")

                    # move file to failed directory
                    dest = add_id(_id, os.path.join(app.failed_path, os.path.basename(filepath)))
                    log.info("moving failed file from %s to %s", filepath, dest)
                    shutil.move(filepath, dest)

                processed = True

                self.files[filepath] = processed
Exemple #5
0
def save_object(in_json, logged_email):
    # validating json data with schema

    try:
        mainschema_file = config_json["pathinfo"][
            "schemapath"] + "biocomputeobject.json"
        schema_file = open(mainschema_file, "r")
        base_uri = 'file://{}/'.format(os.path.dirname(schema_file.name))
        schema = jsonref.load(schema_file, base_uri=base_uri, jsonschema=True)
    except Exception as e:
        out_json = util.log_error(mongo_cl_log, traceback.format_exc())
        return out_json

    out_json = {}
    util.log_error(mongo_cl_log, 'bco_id', 'msg')
    real_id = bco_id = in_json["bco_id"]

    # pdb.set_trace()
    try:
        data = bio_hash.hashed_object(in_json)
        print data['checksum']
        doc = mongo_cl_bco.find_one(
            {"bco_id": "http://biocomputeobject.org/BCO_000012"})
        print doc['checksum']
        return ''
        if bco_id != "-1":
            doc = mongo_cl_bco.find_one({"bco_id": bco_id})
            creator_list = []
            for o in doc["provenance_domain"]["contributors"]:
                if "createdBy" in o["contribution"]:
                    creator_list.append(o["email"])
            if logged_email not in creator_list:
                return {
                    "taskstatus": 0,
                    "errormsg":
                    "You do not have privilege to change this object!"
                }
            else:
                in_json["provenance_domain"][
                    "modified"] = datetime.datetime.now().isoformat()
                data = bio_hash.hashed_object(in_json)

                #in_json["provenance_domain"]["modified"] = datetime.datetime.now().strftime("%b %d, %Y %H:%M:%S")
                validate(in_json, schema)

                data.pop("bco_id")
                result = mongo_cl_bco.update_one({"bco_id": bco_id},
                                                 {'$set': data},
                                                 upsert=False)
                return {"taskstatus": 1, "bcoid": bco_id}
        else:

            doc = mongo_cl_bco.find_one({"checksum": data["checksum"]})
            if doc != None:
                return {
                    "taskstatus":
                    0,
                    "errormsg":
                    "Error. We found an exact copy of your BCO in our database. Duplicate BCOs cannot be submitted!"
                }
            else:
                bco_id = util.get_next_sequence_value(mongo_cl_counters,
                                                      "bcoid")
                bco_str_id = "000000"[0:6 - len(str(bco_id))] + str(bco_id)
                bco_url = config_json["pathinfo"]["bcoprefix"] % (bco_str_id)
                data["bco_id"] = bco_url
                data["provenance_domain"]["created"] = datetime.datetime.now(
                ).isoformat()
                data["provenance_domain"]["contributors"] = [{
                    "name":
                    "",
                    "affiliation":
                    "",
                    "email":
                    logged_email,
                    "orcid":
                    "",
                    "contribution": ["createdBy"]
                }]
                validate(data, schema)
                data = bio_hash.hashed_object(data)
                result = mongo_cl_bco.insert_one(data)
                out_json = {"bcoid": bco_url, "taskstatus": 1}
    except Exception as e:
        print "*" * 100
        print e
        util.add_id(mongo_cl_invalid, real_id)
        out_json = util.log_error(mongo_cl_log, traceback.format_exc(), str(e))

    return out_json