def upload(self, file, parser):
filepath = os.path.join(app.dropbox_path, file.filename)
_id = save_file_metadata(filepath, status="uploading", parser=parser)
filepath = add_id(_id, filepath)
file.save(filepath)
log.info("uploaded {} as {}".format(file.filename, filepath))
_id = save_file_metadata(filepath, status="uploaded")
self.event.set()
return _id
def upload(self, file, parser):
filepath = os.path.join(app.dropbox_path, file.filename)
_id = save_file_metadata(filepath, status='uploading', parser=parser)
filepath = add_id(_id, filepath)
file.save(filepath)
log.info('uploaded {} as {}'.format(file.filename, filepath))
_id = save_file_metadata(filepath, status='uploaded')
self.event.set()
return _id
def process(self, filepath):
# ignore hidden files (e.g., .gitignore)
if not os.path.basename(filepath)[0] == '.':
try:
processed = self.files[filepath]
except KeyError:
log.info(
'new file in dropbox %s created %s', filepath,
datetime.datetime.fromtimestamp(
os.path.getctime(filepath)))
processed = False
if not processed and not os.path.basename(
filepath) == '.gitignore':
try:
_id = ingest.ingest(filepath)
# move file to ingested directory
dest = add_id(
_id,
os.path.join(app.ingested_path,
os.path.basename(filepath)))
log.info('moving ingested file from %s to %s', filepath,
dest)
shutil.move(filepath, dest)
except Exception as e:
log.warn('failed to ingest %s', filepath)
log.warn(e)
_id = save_file_metadata(filepath, status='error')
# move file to failed directory
dest = add_id(
_id,
os.path.join(app.failed_path,
os.path.basename(filepath)))
log.info('moving failed file from %s to %s', filepath,
dest)
shutil.move(filepath, dest)
processed = True
self.files[filepath] = processed
def process(self, filepath):
# ignore hidden files (e.g., .gitignore)
if not os.path.basename(filepath)[0] == ".":
try:
processed = self.files[filepath]
except KeyError:
log.info(
"new file in dropbox %s created %s",
filepath,
datetime.datetime.fromtimestamp(os.path.getctime(filepath)),
)
processed = False
if not processed and not os.path.basename(filepath) == ".gitignore":
try:
_id = ingest.ingest(filepath)
# move file to ingested directory
dest = add_id(_id, os.path.join(app.ingested_path, os.path.basename(filepath)))
log.info("moving ingested file from %s to %s", filepath, dest)
shutil.move(filepath, dest)
except Exception as e:
log.warn("failed to ingest %s", filepath)
log.warn(e)
_id = save_file_metadata(filepath, status="error")
# move file to failed directory
dest = add_id(_id, os.path.join(app.failed_path, os.path.basename(filepath)))
log.info("moving failed file from %s to %s", filepath, dest)
shutil.move(filepath, dest)
processed = True
self.files[filepath] = processed
def save_object(in_json, logged_email):
# validating json data with schema
try:
mainschema_file = config_json["pathinfo"][
"schemapath"] + "biocomputeobject.json"
schema_file = open(mainschema_file, "r")
base_uri = 'file://{}/'.format(os.path.dirname(schema_file.name))
schema = jsonref.load(schema_file, base_uri=base_uri, jsonschema=True)
except Exception as e:
out_json = util.log_error(mongo_cl_log, traceback.format_exc())
return out_json
out_json = {}
util.log_error(mongo_cl_log, 'bco_id', 'msg')
real_id = bco_id = in_json["bco_id"]
# pdb.set_trace()
try:
data = bio_hash.hashed_object(in_json)
print data['checksum']
doc = mongo_cl_bco.find_one(
{"bco_id": "http://biocomputeobject.org/BCO_000012"})
print doc['checksum']
return ''
if bco_id != "-1":
doc = mongo_cl_bco.find_one({"bco_id": bco_id})
creator_list = []
for o in doc["provenance_domain"]["contributors"]:
if "createdBy" in o["contribution"]:
creator_list.append(o["email"])
if logged_email not in creator_list:
return {
"taskstatus": 0,
"errormsg":
"You do not have privilege to change this object!"
}
else:
in_json["provenance_domain"][
"modified"] = datetime.datetime.now().isoformat()
data = bio_hash.hashed_object(in_json)
#in_json["provenance_domain"]["modified"] = datetime.datetime.now().strftime("%b %d, %Y %H:%M:%S")
validate(in_json, schema)
data.pop("bco_id")
result = mongo_cl_bco.update_one({"bco_id": bco_id},
{'$set': data},
upsert=False)
return {"taskstatus": 1, "bcoid": bco_id}
else:
doc = mongo_cl_bco.find_one({"checksum": data["checksum"]})
if doc != None:
return {
"taskstatus":
0,
"errormsg":
"Error. We found an exact copy of your BCO in our database. Duplicate BCOs cannot be submitted!"
}
else:
bco_id = util.get_next_sequence_value(mongo_cl_counters,
"bcoid")
bco_str_id = "000000"[0:6 - len(str(bco_id))] + str(bco_id)
bco_url = config_json["pathinfo"]["bcoprefix"] % (bco_str_id)
data["bco_id"] = bco_url
data["provenance_domain"]["created"] = datetime.datetime.now(
).isoformat()
data["provenance_domain"]["contributors"] = [{
"name":
"",
"affiliation":
"",
"email":
logged_email,
"orcid":
"",
"contribution": ["createdBy"]
}]
validate(data, schema)
data = bio_hash.hashed_object(data)
result = mongo_cl_bco.insert_one(data)
out_json = {"bcoid": bco_url, "taskstatus": 1}
except Exception as e:
print "*" * 100
print e
util.add_id(mongo_cl_invalid, real_id)
out_json = util.log_error(mongo_cl_log, traceback.format_exc(), str(e))
return out_json