def mark_obsolete(self, accession):
"""
Mark Genestack file as obsolete one by adding corresponding key to metainfo.
:param accession: accession of file
:return: None
"""
metainfo = Metainfo()
metainfo.add_boolean('genestack:obsolete', True)
self.add_metainfo_values(accession, metainfo)
def mark_obsolete(self, accession):
"""
Mark Genestack file as obsolete one by adding corresponding key to metainfo.
:param accession: accession of file
:return: None
"""
metainfo = Metainfo()
metainfo.add_boolean('genestack:obsolete', True)
self.add_metainfo_values(accession, metainfo)
def mark_for_tests(self, app_file):
"""
Mark Genestack file as test one by adding corresponding key to metainfo.
Test file will calculate md5 checksums of its encapsulated physical files
during initialization.
:param app_file: accession of file
:return: None
"""
metainfo = Metainfo()
metainfo.add_boolean(CALCULATE_CHECKSUMS_KEY, True)
self.add_metainfo_values(app_file, metainfo)
def mark_for_tests(self, app_file):
"""
Mark Genestack file as test one by adding corresponding key to metainfo.
Test file will calculate md5 checksums of its encapsulated physical files
during initialization.
:param app_file: accession of file
:return: None
"""
metainfo = Metainfo()
metainfo.add_boolean(CALCULATE_CHECKSUMS_KEY, True)
self.add_metainfo_values(app_file, metainfo)
def _fill_dataset_metainfo(dataset_metainfo, name):
dataset_metainfo = dataset_metainfo or Metainfo()
if Metainfo.NAME in dataset_metainfo:
raise GenestackException(
'Provided metainfo must not have "%s" field set' %
Metainfo.NAME)
dataset_metainfo.add_string(Metainfo.NAME, name)
return dataset_metainfo
def add_checksums(self, app_file, expected_checksums):
"""
Add expected MD5 checksum to the metainfo of a CLA file.
Expected checksums are calculated in the following way:
- The number of checksums equals number of entries in storage.
For instance, a Reference Genome file has 2 entries (annotation and sequence files).
- If there are multiple files in one entry, they will be concatenated in the same order
as they were ``PUT`` to storage by the initialization script.
- If a file is marked for testing, then after initialization its metainfo
will contain both expected and actual checksum values.
:param app_file: accession of application file
:param expected_checksums: collection of MD5 checksums
:return: None
"""
metainfo = Metainfo()
for key, value in expected_checksums.items():
metainfo.add_string('%s%s' % (EXPECTED_CHECKSUM_PREFIX, key), value)
self.add_metainfo_values(app_file, metainfo)
def add_checksums(self, app_file, expected_checksums):
"""
Add expected MD5 checksum to the metainfo of a CLA file.
Expected checksums are calculated in the following way:
- The number of checksums equals number of entries in storage.
For instance, a Reference Genome file has 2 entries (annotation and sequence files).
- If there are multiple files in one entry, they will be concatenated in the same order
as they were ``PUT`` to storage by the initialization script.
- If a file is marked for testing, then after initialization its metainfo
will contain both expected and actual checksum values.
:param app_file: accession of application file
:param expected_checksums: collection of MD5 checksums
:return: None
"""
metainfo = Metainfo()
for key, value in expected_checksums.items():
metainfo.add_string('%s%s' % (EXPECTED_CHECKSUM_PREFIX, key),
value)
self.add_metainfo_values(app_file, metainfo)
def collect_metainfos(self, accessions):
"""
Get complete metainfo of a list of files.
:param accessions: list of accessions
:type accessions: list[str]
:return: list of metainfo objects
:rtype: list[Metainfo]
"""
return [
Metainfo.parse_metainfo_from_dict(mi)
for mi in self.invoke('getMetainfo', accessions)
]
def create_folder(self,
name,
parent=None,
description=None,
metainfo=None):
"""
Create a folder.
:param name: name of the folder
:type name: str
:param parent: if not specified, create folder in the user's private folder
:type parent: str
:param description: description of the folder (goes into the metainfo)
:type description: str
:param metainfo: additional Metainfo. Description and accession should be specified either via arguments or in a metainfo object (but not in both).
:type metainfo: Metainfo
:return: accession of created folder
"""
metainfo = metainfo or Metainfo()
metainfo.add_string(Metainfo.NAME, name)
if description is not None:
metainfo.add_string(Metainfo.DESCRIPTION, description)
return self.invoke('createFolder', parent, metainfo)
def _copy_metainfo(metainfo):
return Metainfo(metainfo) if metainfo else Metainfo()
created_pairs = set()
# parse the CSV file
with open(tsv_input, 'r') as the_file:
reader = csv.DictReader(the_file, dialect='excel_tab')
field_names = reader.fieldnames
for file_data in reader:
# skip the entry if the file was already included in a previously created paired-end assay
if file_data[FILE_ACCESSION] in created_pairs:
continue
# for each entry, prepare a Metainfo object
metainfo = Metainfo()
for key in VALID_FIELDS.keys():
metainfo.add_string(VALID_FIELDS.get(key) or key, file_data[key])
metainfo.add_external_link(BioMetaKeys.READS_LINK, ENCODE_URL_PATTERN.format(file_data[FILE_ACCESSION]))
if file_data.get(PAIRED_ACCESSION):
# add URL of second mate if the reads are paired-end
metainfo.add_string(FILE_ACCESSION, PAIRED_ACCESSION)
metainfo.add_external_link(BioMetaKeys.READS_LINK, ENCODE_URL_PATTERN.format(file_data[PAIRED_ACCESSION]))
created_pairs.add(file_data[PAIRED_ACCESSION])
# create the sequencing assay on Genestack
created_file = importer.create_sequencing_assay(experiment, metainfo=metainfo)
print('Created file "%s" (%s)' % (file_data[FILE_ACCESSION], created_file))
# parse the CSV file
with open(csv_input, 'r') as the_file:
reader = csv.DictReader(the_file, delimiter=",")
field_names = reader.fieldnames
# check if mandatory keys are in the CSV file
for mandatory_key in MANDATORY_KEYS:
if mandatory_key not in field_names:
raise GenestackException("The key '%s' must be supplied in the CSV file" % mandatory_key)
for file_data in reader:
# for each entry, prepare a Metainfo object
metainfo = Metainfo()
for key in field_names:
# 'link' and 'organism' are treated separately, as they are added to the metainfo using specific methods
if key == "link":
url = file_data[key]
metainfo.add_external_link(key=BioMetaKeys.READS_LINK, text="link", url=url, fmt=file_format)
elif key == "organism":
metainfo.add_string(BioMetaKeys.ORGANISM, file_data[key])
# all the other keys are added as strings
else:
metainfo_key = SPECIAL_KEYS.get(key.lower(), key)
metainfo.add_string(metainfo_key, file_data[key])
# create the sequencing assay on Genestack
created_file = importer.create_sequencing_assay(experiment, metainfo=metainfo)
if args.local_key not in field_names:
raise GenestackException(
"Error: the local key %s is not present in the supplied CSV file"
% args.local_key)
for file_data in reader:
# find the corresponding file
local_identifier = file_data[local_key]
remote_file = identifier_map.get(local_identifier)
if not remote_file:
print('Warning: no match found for file name "%s"' %
local_identifier)
continue
# prepare a Metainfo object
metainfo = Metainfo()
for key in field_names:
# key parsing logic
value = file_data[key]
if value == "" or value is None:
continue
if key == args.local_key:
continue
if key == "organism":
metainfo.add_string(BioMetaKeys.ORGANISM, value)
else:
metainfo_key = SPECIAL_KEYS.get(key.lower(), key)
if parse_as_boolean(value) is not None:
metainfo.add_boolean(metainfo_key,
parse_as_boolean(value))
else:
def test_metainfo_io(conn):
data_importer = DataImporter(conn)
fu = FilesUtil(conn)
created = fu.get_special_folder(SpecialFolders.CREATED)
info = Metainfo()
info.add_boolean("a", True)
info.add_file_reference("b", created)
info.add_date_time("c", "2015-12-13")
info.add_integer("d", 239)
info.add_decimal("e", 238.583)
info.add_decimal("e", -13.4)
info.add_string("f", "hello")
info.add_memory_size("g", 2847633)
info.add_person("i", "Rosalind Franklin", "+1-202-555-0123",
"*****@*****.**")
info.add_publication("j",
"My Publication",
"Myself",
"Journal of Me",
"23/12/2014",
pages="12-23")
info.add_value(Metainfo.NAME, StringValue("Test report file"))
report_file = None
try:
report_file = data_importer.create_report_file(metainfo=info,
urls=[TEST_URL],
parent=created)
metainfo = next(iter(fu.collect_metainfos([report_file])))
assert metainfo.get('a')[0].get_boolean()
assert isinstance(metainfo.get('b')[0].get_accession(), str)
assert metainfo.get('c')[0].get_date() == _strptime_local(
'2015-12-13', '%Y-%m-%d')
assert metainfo.get('d')[0].get_int() == 239
assert metainfo.get('e')[0].get_decimal() == 238.583
assert metainfo.get('e')[1].get_decimal() == -13.4
assert metainfo.get('f')[0].get_string() == "hello"
assert metainfo.get('g')[0].get_int() == 2847633
assert metainfo.get('i')[0].get_person() == {
'name': 'Rosalind Franklin',
'phone': '+1-202-555-0123',
'email': '*****@*****.**'
}
assert metainfo.get('j')[0].get_publication() == {
'title': 'My Publication',
'authors': 'Myself',
'journalName': 'Journal of Me',
'issueDate': '23/12/2014',
'pages': '12-23',
'issueNumber': None,
'identifiers': {}
}
assert metainfo.get(
Metainfo.NAME)[0].get_string() == "Test report file"
assert metainfo.get(BioMetaKeys.DATA_LINK)[0].get_url() == TEST_URL
finally:
if report_file is not None:
fu.unlink_file(report_file, created)
def test_metainfo_io(conn):
data_importer = DataImporter(conn)
fu = FilesUtil(conn)
created = fu.get_special_folder(SpecialFolders.CREATED)
info = Metainfo()
info.add_boolean("a", True)
info.add_file_reference("b", created)
info.add_date_time("c", "2015-12-13")
info.add_integer("d", 239)
info.add_decimal("e", 238.583)
info.add_decimal("e", -13.4)
info.add_string("f", "hello")
info.add_memory_size("g", 2847633)
info.add_person("i", "Rosalind Franklin", "+1-202-555-0123", "*****@*****.**")
info.add_publication("j", "My Publication", "Myself", "Journal of Me", "23/12/2014", pages="12-23")
info.add_value(Metainfo.NAME, StringValue("Test report file"))
report_file = None
try:
report_file = data_importer.create_report_file(metainfo=info, urls=[TEST_URL], parent=created)
metainfo = next(iter(fu.collect_metainfos([report_file])))
assert metainfo.get('a')[0].get_boolean()
assert isinstance(metainfo.get('b')[0].get_accession(), str)
assert metainfo.get('c')[0].get_date() == _strptime_local('2015-12-13', '%Y-%m-%d')
assert metainfo.get('d')[0].get_int() == 239
assert metainfo.get('e')[0].get_decimal() == 238.583
assert metainfo.get('e')[1].get_decimal() == -13.4
assert metainfo.get('f')[0].get_string() == "hello"
assert metainfo.get('g')[0].get_int() == 2847633
assert metainfo.get('i')[0].get_person() == {'name': 'Rosalind Franklin', 'phone': '+1-202-555-0123',
'email': '*****@*****.**'}
assert metainfo.get('j')[0].get_publication() == {'title': 'My Publication', 'authors': 'Myself',
'journalName': 'Journal of Me', 'issueDate': '23/12/2014',
'pages': '12-23', 'issueNumber': None, 'identifiers': {}}
assert metainfo.get(Metainfo.NAME)[0].get_string() == "Test report file"
assert metainfo.get(BioMetaKeys.DATA_LINK)[0].get_url() == TEST_URL
finally:
if report_file is not None:
fu.unlink_file(report_file, created)
def metainfo():
return Metainfo()
reader = csv.DictReader(the_file, delimiter=",")
field_names = reader.fieldnames
if args.local_key not in field_names:
raise GenestackException("Error: the local key %s is not present in the supplied CSV file" % args.local_key)
for file_data in reader:
# find the corresponding file
local_identifier = file_data[local_key]
remote_file = identifier_map.get(local_identifier)
if not remote_file:
print('Warning: no match found for file name "%s"' % local_identifier)
continue
# prepare a Metainfo object
metainfo = Metainfo()
for key in field_names:
# key parsing logic
value = file_data[key]
if value == "" or value is None:
continue
if key == args.local_key:
continue
if key == "organism":
metainfo.add_string(BioMetaKeys.ORGANISM, value)
else:
metainfo_key = SPECIAL_KEYS.get(key.lower(), key)
if parse_as_boolean(value) is not None:
metainfo.add_boolean(metainfo_key, parse_as_boolean(value))
else:
metainfo.add_string(metainfo_key, value)
# parse the CSV file
with open(csv_input, 'r') as the_file:
reader = csv.DictReader(the_file, delimiter=",")
field_names = reader.fieldnames
# check if mandatory keys are in the CSV file
for mandatory_key in MANDATORY_KEYS:
if mandatory_key not in field_names:
raise GenestackException("The key '%s' must be supplied in the CSV file" % mandatory_key)
for file_data in reader:
# for each entry, prepare a Metainfo object
metainfo = Metainfo()
for key in field_names:
# 'link' and 'organism' are treated separately, as they are added to the metainfo using specific methods
if key == "link":
url = file_data[key]
metainfo.add_external_link(key=BioMetaKeys.READS_LINK, text="link", url=url, fmt=file_format)
elif key == "organism":
metainfo.add_string(BioMetaKeys.ORGANISM, file_data[key])
# all the other keys are added as strings
else:
metainfo_key = SPECIAL_KEYS.get(key.lower(), key)
metainfo.add_string(metainfo_key, file_data[key])
# create the sequencing assay on Genestack
created_file = importer.create_sequencing_assay(experiment, metainfo=metainfo)