def __init__(self,
uuid=None,
accession=None,
file_format='',
lab='4dn-dcic-lab',
extra_files=None,
source_experiments=None,
award='1U01CA200059-01',
status='to be uploaded by workflow',
md5sum=None,
file_size=None,
other_fields=None,
**kwargs):
self.uuid = uuid if uuid else str(uuid4())
self.accession = accession if accession else generate_rand_accession()
self.status = status
self.lab = lab
self.award = award
self.file_format = parse_formatstr(file_format)
if extra_files:
self.extra_files = extra_files
if source_experiments:
self.source_experiments = source_experiments
if md5sum:
self.md5sum = md5sum
if file_size:
self.file_size = file_size
if other_fields:
for field in other_fields:
setattr(self, field, other_fields[field])
def post_random_file(bucket, keypairs_file):
"""Generates a fake pairs.gz file with random uuid and accession
and posts it to fourfront. The content is unique since it contains
its own uuid. The file metadata does not contain md5sum or
content_md5sum.
"""
uuid = str(uuid4())
accession = generate_rand_accession()
newfile = {
"accession": accession,
"file_format": "pairs",
"award": "b0b9c607-f8b4-4f02-93f4-9895b461334b",
"lab": "828cd4fe-ebb0-4b36-a94a-d2e3a36cc989",
"uuid": uuid
}
upload_key = uuid + '/' + accession + '.pairs.gz'
tmpfilename = 'alsjekvjf.gz'
with gzip.open(tmpfilename, 'wb') as f:
f.write(uuid)
key = fdnDCIC.FDN_Key(keypairs_file, "default")
connection = fdnDCIC.FDN_Connection(key)
response = fdnDCIC.new_FDN(connection, 'FileProcessed', newfile)
print(response)
s3 = boto3.resource('s3')
s3.meta.client.upload_file(tmpfilename, bucket, upload_key)
return newfile
def post_random_file(bucket,
ff_key,
file_format='pairs',
extra_file_format='pairs_px2',
file_extension='pairs.gz',
extra_file_extension='pairs.gz.px2',
schema='file_processed',
extra_status=None):
"""Generates a fake file with random uuid and accession
and posts it to fourfront. The content is unique since it contains
its own uuid. The file metadata does not contain md5sum or
content_md5sum.
Uses the given fourfront keys
"""
uuid = str(uuid4())
accession = generate_rand_accession()
newfile = {
"accession": accession,
"file_format": file_format,
"award": "b0b9c607-f8b4-4f02-93f4-9895b461334b",
"lab": "828cd4fe-ebb0-4b36-a94a-d2e3a36cc989",
"uuid": uuid
}
upload_key = uuid + '/' + accession + '.' + file_extension
tmpfilename = 'alsjekvjf'
with gzip.open(tmpfilename, 'wb') as f:
f.write(uuid)
s3 = boto3.resource('s3')
s3.meta.client.upload_file(tmpfilename, bucket, upload_key)
# extra file
if extra_file_format:
newfile["extra_files"] = [{
"file_format": extra_file_format,
"accession": accession,
"uuid": uuid
}]
if extra_status:
newfile["extra_files"][0]['status'] = extra_status
extra_upload_key = uuid + '/' + accession + '.' + extra_file_extension
extra_tmpfilename = 'alsjekvjf-extra'
with open(extra_tmpfilename, 'w') as f:
f.write(uuid + extra_file_extension)
s3.meta.client.upload_file(extra_tmpfilename, bucket, extra_upload_key)
response = post_metadata(newfile, schema, key=ff_key)
print(response)
return newfile
def post_random_file(bucket,
ff_key,
file_format='pairs',
extra_file_format='pairs_px2',
file_extension='pairs.gz',
extra_file_extension='pairs.gz.px2',
schema='file_processed',
extra_status=None):
"""Generates a fake file with random uuid and accession
and posts it to fourfront. The content is unique since it contains
its own uuid. The file metadata does not contain md5sum or
content_md5sum.
Uses the given fourfront keys
"""
uuid = str(uuid4())
accession = generate_rand_accession(ACCESSION_PREFIX, 'FI')
newfile = {
"accession": accession,
"file_format": file_format,
"institution": DEFAULT_INSTITUTION,
"project": DEFAULT_PROJECT,
"uuid": uuid
}
upload_key = uuid + '/' + accession + '.' + file_extension
tmpfilename = 'alsjekvjf'
with gzip.open(tmpfilename, 'wb') as f:
f.write(uuid.encode('utf-8'))
s3 = boto3.resource('s3')
s3.meta.client.upload_file(tmpfilename, bucket, upload_key)
# extra file
if extra_file_format:
newfile["extra_files"] = [{
"file_format": extra_file_format,
"accession": accession,
"uuid": uuid
}]
if extra_status:
newfile["extra_files"][0]['status'] = extra_status
extra_upload_key = uuid + '/' + accession + '.' + extra_file_extension
extra_tmpfilename = 'alsjekvjf-extra'
with open(extra_tmpfilename, 'w') as f:
f.write(uuid + extra_file_extension)
s3.meta.client.upload_file(extra_tmpfilename, bucket, extra_upload_key)
response = post_metadata(newfile, schema, key=ff_key)
print(response)
return newfile
def post_random_file(bucket, ff_key):
"""Generates a fake pairs.gz file with random uuid and accession
and posts it to fourfront. The content is unique since it contains
its own uuid. The file metadata does not contain md5sum or
content_md5sum.
Uses the given fourfront keys
"""
uuid = str(uuid4())
accession = generate_rand_accession()
newfile = {
"accession":
accession,
"file_format":
"pairs",
"award":
"b0b9c607-f8b4-4f02-93f4-9895b461334b",
"lab":
"828cd4fe-ebb0-4b36-a94a-d2e3a36cc989",
"uuid":
uuid,
"extra_files": [{
"file_format": "pairs_px2",
"accession": accession,
"uuid": uuid
}]
}
upload_key = uuid + '/' + accession + '.pairs.gz'
tmpfilename = 'alsjekvjf.gz'
with gzip.open(tmpfilename, 'wb') as f:
f.write(uuid)
extra_upload_key = uuid + '/' + accession + '.pairs.gz.px2'
extra_tmpfilename = 'alsjekvjf-extra.gz'
with gzip.open(extra_tmpfilename, 'wb') as f:
f.write(uuid + '.px2')
response = post_metadata(newfile, 'file_processed', key=ff_key)
print(response)
s3 = boto3.resource('s3')
s3.meta.client.upload_file(tmpfilename, bucket, upload_key)
s3.meta.client.upload_file(extra_tmpfilename, bucket, extra_upload_key)
return newfile