def __init__(self, host, catalog):
# Don't authenticate - get only publicly-readable data
server = DerivaServer("https", host)
catalog = server.connect_ermrest(catalog)
catalog.dcctx['cid'] = "oneoff/load_tables.py"
self.pb = catalog.getPathBuilder()
self.schema_map = {"Vocab": "Vocabulary"}
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = "oneoff/model"
model = catalog.getCatalogModel()
#refactor_fkeys(model, 2, combo1_included=True, combo2_included=False, primary_types=("COMBO1"))
#refactor_fkeys(model, 2, combo1_included=True, combo2_included=True, primary_types=())
fw = open('rename.log', 'w')
fw.close()
fw = open('new.log', 'w')
fw.close()
refactor_fkeys(model, 2, combo1_included=True, combo2_included=True)
refactor_fkeys(model, 4, combo1_included=True, combo2_included=True)
"""
Check that the new columns are not among the renamed columns
"""
for new_column_table, new_column_values in new_columns.items():
if new_column_table in renamed_columns.keys():
renamed_values = renamed_columns[new_column_table]
for new_column in new_column_values:
for old_column, renamed_column in renamed_values.items():
if new_column == renamed_column:
print(
'WARNING: New column {}:{} is also a renamed column'
.format(new_column_table, new_column))
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = "oneoff/model"
model = catalog.getCatalogModel()
"""
Create the new vocabulary tables
"""
utils.create_table_if_not_exist(
model, 'Vocab',
utils.define_Vocab_table('Data_Dictionary_Name', 'Dictionary names'))
utils.create_table_if_not_exist(
model, 'Vocab',
utils.define_Vocab_table('Data_Dictionary_Category',
'Dictionary categories'))
"""
Load data into the new vocabulary tables
"""
utils.add_rows_to_vocab_table(catalog, 'Data_Dictionary_Name',
Data_Dictionary_Name_rows)
utils.add_rows_to_vocab_table(catalog, 'Data_Dictionary_Category',
Data_Dictionary_Category_rows)
"""
Create the acls bindings
"""
utils.set_table_acl_bindings(catalog, 'Vocab', 'Data_Dictionary_Name',
acl_bindings)
utils.set_table_acl_bindings(catalog, 'Vocab', 'Data_Dictionary_Category',
acl_bindings)
def __init__(self, host, credentials, catalog_number=None):
self.server = DerivaServer("https", host, credentials=credentials)
if catalog_number is None:
self.catalog = self.server.create_ermrest_catalog()
else:
self.catalog = self.server.connect_ermrest(catalog_number)
self.model = self.catalog.getCatalogModel()
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = "oneoff/data"
model = catalog.getCatalogModel()
#load_table_RNASeq_Reference_Genome(catalog)
load_table_RNASeq_Workflow(catalog)
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
utils.set_nullok_column_if_exists(model, 'Vocab', 'File_Type',
'Table_Name', False)
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
"""
Update vocabularies
"""
update_Data_Dictionary_Name(catalog, model)
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = "oneoff/model"
model = catalog.getCatalogModel()
"""
Create the acls
"""
utils.set_table_acls(catalog, 'PDB', 'Data_Dictionary', acls)
utils.set_table_acls(catalog, 'PDB', 'Supported_Dictionary', acls)
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = "oneoff/model"
model = catalog.getCatalogModel()
set_comments(model)
# let's the library deals with applying the difference
model.apply()
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
"""
Create table
"""
utils.create_table_if_not_exist(model, 'PDB', define_tdoc_ihm_cross_link_pseudo_site()) #Requires ihm_cross_link_restraint, ihm_model_list, ihm_pseudo_site, cross_link_partner
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
"""
Create table
"""
utils.create_table_if_not_exist(model, 'PDB',
define_tdoc_Supported_Dictionary())
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
"""
Create table
"""
load_dictionaries(catalog)
def suite_setup(self):
# create catalog
server = DerivaServer('https',
self._hostname,
credentials=get_credential(self._hostname))
if self._reuse_catalog_id:
self._ermrest_catalog = server.connect_ermrest(
self._reuse_catalog_id)
self.unit_teardown(
) # in the event that the last run terminated abruptly and didn't properly teardown
else:
self._ermrest_catalog = server.create_ermrest_catalog()
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
"""
Create table
"""
utils.create_table_if_not_exist(model, 'PDB',
define_tdoc_ihm_data_transformation())
"""
Update existing model
"""
update_PDB_ihm_related_datasets(model) #Requires ihm_data_transformation
def setUpClass(cls):
logger.debug("setupUpClass begin")
credential = os.getenv("DERIVA_PY_TEST_CREDENTIAL") or get_credential(
hostname)
server = DerivaServer('https', hostname, credential)
cls.catalog = server.create_ermrest_catalog()
try:
define_test_schema(cls.catalog)
populate_test_catalog(cls.catalog)
except Exception:
# on failure, delete catalog and re-raise exception
cls.catalog.delete_ermrest_catalog(really=True)
raise
logger.debug("setupUpClass done")
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
"""
Create table
"""
utils.create_table_if_not_exist(
model, 'PDB', define_tdoc_ihm_hdx_restraint(
)) #Requires ihm_feature_list, ihm_dataset_list
"""
Update existing model
"""
update_PDB_ihm_derived_distance_restraint(model)
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = "oneoff/model"
rename_pk(catalog, 'PDB', 'entry', 'entry_id_unique_key',
'entry_id_primary_key')
create_pk(catalog, 'PDB', 'struct', ['entry_id'], 'struct_primary_key')
drop_fk(catalog, 'PDB', 'pdbx_entry_details',
'pdbx_entry_details_structure_id_fkey')
drop_column(catalog, 'PDB', 'pdbx_entry_details', 'structure_id')
create_pk(catalog, 'PDB', 'pdbx_entry_details', ['entry_id'],
'pdbx_entry_details_primary_key')
set_acl_binding(catalog, 'PDB', 'pdbx_entry_details',
pdbx_entry_details_acl_binding)
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
utils.alter_on_update_fkey_if_exist(
model, 'PDB', 'ihm_poly_probe_position',
'ihm_poly_probe_position_mutation_flag_fkey', 'CASCADE')
utils.alter_on_update_fkey_if_exist(
model, 'PDB', 'ihm_poly_probe_position',
'ihm_poly_probe_position_modification_flag_fkey', 'CASCADE')
"""
Update vocabularies
"""
update_vocabularies(catalog)
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
"""
Update existing model
"""
update_PDB_ihm_ensemble_info(
model) #Requires sub_sample_flag and sub_sampling_type
"""
Create table
"""
utils.create_table_if_not_exist(
model, 'PDB', define_tdoc_ihm_ensemble_sub_sample(
)) #Requires ihm_ensemble_info, ihm_model_group, ihm_external_files
def __init__(self, values, connect_to_ermrest=True):
self.attrs = [
"database", "hatrac_parent", "species_schema", "species_table",
"chromosome_schema", "chromosome_table", "gene_type_schema",
"gene_type_table", "gene_schema", "gene_table", "dbxref_schema",
"dbxref_table", "catalog_id", "host", "curie_prefix",
"source_file_schema", "source_file_table", "ontology_schema",
"ontology_table", "scratch_db", "scratch_directory"
]
config_file = None
if isinstance(values, argparse.Namespace):
if hasattr(values, 'config_file'):
config_file = values.config_file
for attr in self.attrs:
if hasattr(values, attr):
setattr(self, attr, getattr(values, attr))
else:
setattr(self, attr, None)
else:
config_file = values.get('config_file')
for attr in self.attrs:
setattr(self, attr, values.get(attr))
if (config_file):
file = open(config_file, "r")
defaults = json.load(file)
file.close()
for attr in self.attrs:
if defaults.get(attr) and not getattr(self, attr):
setattr(self, attr, defaults[attr])
shell_attrs = []
for attr in self.attrs:
val = getattr(self, attr)
if val:
shell_attrs.append('export {attr}="{val}"'.format(attr=attr,
val=val))
self.shell_attr_string = ';'.join(shell_attrs)
self.pb = None
if connect_to_ermrest:
self.credential = get_credential(self.host)
server = DerivaServer('https', self.host, self.credential)
catalog = server.connect_ermrest(self.catalog_id)
self.model = catalog.getCatalogModel()
self.pb = catalog.getPathBuilder()
def create_catalog(server='demo.derivacloud.org', catalog_name='test'):
global catalog_id
credentials = get_credential(server)
catalog = DerivaServer('https', server,
credentials=credentials).create_ermrest_catalog()
catalog_id = catalog.catalog_id
logger.info('Catalog_id is {}'.format(catalog_id))
logger.info('Configuring catalog....')
catalog = DerivaCatalogConfigure(host, catalog_id=catalog_id)
catalog.configure_baseline_catalog(catalog_name=catalog_name,
admin='DERIVA Demo Admin',
curator='DERIVA Demo Curator',
writer='DERIVA Demo Writer',
reader='*')
return catalog
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = "oneoff/model"
model = catalog.getCatalogModel()
#-- clean up
if False:
drop_table(catalog, 'PDB', 'ihm_derived_angle_restraint')
# new changes
if True:
model = catalog.getCatalogModel()
#utils.create_table_if_not_exist(model, "Vocab", utils.define_Vocab_table('ihm_derived_angle_restraint_group_conditionality', 'Conditionality of a group of angles restrained together'))
#utils.create_table_if_not_exist(model, "Vocab", utils.define_Vocab_table('ihm_derived_angle_restraint_restraint_type', 'The type of angle restraint'))
#add_rows_to_Vocab_ihm_derived_angle_restraint_group_conditionality(catalog)
#add_rows_to_Vocab_ihm_derived_angle_restraint_restraint_type(catalog)
utils.create_table_if_not_exist(
model, "PDB", define_tdoc_ihm_derived_angle_restraint())
def __init__(self, protocol, host, catalog_id, license_url=None):
""" Creates the sitemap object
:param protocl: protocol (e.g., https)
:param host: the host name
:param catalog_id: the catalog id
:param license_url: the url to the license used for images in this catalog
"""
if sys.version_info[0] < 3:
raise NotImplementedError(
"SitemapBuilder is only supported for Python 3.x")
self.protocol = protocol
self.host = host
self.catalog_id = catalog_id
self.license_url = license_url
self._create_xml_tree()
# Don't authenticate - we want only publicly-available content in the index
self.server = DerivaServer(protocol, host)
self.catalog = self.server.connect_ermrest(catalog_id)
self.model = self.catalog.getCatalogModel()
self.pb = self.catalog.getPathBuilder()
self.table_specs = []
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
"""
Create table
"""
utils.create_table_if_not_exist(model, 'PDB',
define_tdoc_ihm_pseudo_site())
"""
Create combo1/combo2 keys in existing tables
"""
utils.create_key_if_not_exists(model, 'PDB', 'ihm_model_list',
['RID', 'model_id'],
'ihm_model_list_combo2_key')
utils.create_key_if_not_exists(model, 'PDB', 'ihm_feature_list',
['RID', 'structure_id', 'feature_id'],
'ihm_feature_list_combo1_key')
utils.create_key_if_not_exists(model, 'PDB', 'ihm_dataset_list',
['RID', 'structure_id', 'id'],
'ihm_dataset_list_combo1_key')
"""
Update existing model
"""
update_PDB_ihm_cross_link_restraint(model) #Requires pseudo_site_flag
update_PDB_ihm_pseudo_site_feature(
model) #Requires ihm_pseudo_site and ihm_feature_list_combo1_key
"""
Rename primary keys
"""
utils.rename_key_if_exist(model, 'PDB', 'ihm_model_group',
'ihm_model_group_RID_id_key',
'ihm_model_group_combo2_key')
utils.rename_key_if_exist(model, 'PDB', 'ihm_external_files',
'ihm_external_files_id_RID_key',
'ihm_external_files_combo2_key')
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = "oneoff/model"
model = catalog.getCatalogModel()
#-- clean up
if False:
drop_table(catalog, 'PDB', 'struct_ref')
drop_table(catalog, 'PDB', 'struct_ref_seq')
drop_table(catalog, 'PDB', 'struct_ref_seq_dif')
# new changes
if True:
model = catalog.getCatalogModel()
#utils.create_table_if_not_exist(model, "Vocab", utils.define_Vocab_table('struct_ref_db_name', 'The name of the database containing reference information'))
#utils.create_table_if_not_exist(model, "Vocab", utils.define_Vocab_table('struct_ref_seq_dif_details', 'Details about the special aspects of point differences in the alignment of the macromolecular sequence in the integrative model and the sequence in the reference database'))
#add_rows_to_Vocab_struct_ref_db_name(catalog)
#add_rows_to_Vocab_struct_ref_seq_dif_details(catalog)
utils.create_key_if_not_exists(model, 'PDB', 'entity', ['RID', 'structure_id', 'id'], 'entity_combo1_key')
utils.create_table_if_not_exist(model, "PDB", define_tdoc_struct_ref())
utils.create_table_if_not_exist(model, "PDB", define_tdoc_struct_ref_seq())
utils.create_table_if_not_exist(model, "PDB", define_tdoc_struct_ref_seq_dif())
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = 'oneoff/model'
model = catalog.getCatalogModel()
schema = model.schemas['Vocab']
for table_name in tables:
table = schema.tables[table_name]
referenced_by = table.referenced_by
if len(referenced_by) != 1:
print('Table {} has more than 1 FK'.format(table_name))
else:
referenced_by = referenced_by[0]
schema_name = referenced_by.constraint_schema.name
tname = referenced_by.table.name
fk_name = referenced_by.constraint_name
utils.alter_on_update_fkey_if_exist(model, schema_name, tname,
fk_name, 'CASCADE')
"""
Update vocabularies
"""
update_vocabularies(catalog)
print('Updated the vocabulary tables.')
def deriva_modify(servername, catalog_id, acls=None):
"""Modify a DERIVA catalog's options using the CfdeDataPackage.
Currently limited to ACL changes only.
Arguments:
servername (str): The name of the DERIVA server
catalog_id (str or int): The ID of the catalog to change. The catalog must exist.
acls (dict): The Access Control Lists to set.
Returns:
dict: The results of the update.
success (bool): True if the ACLs were successfully changed.
"""
catalog_id = str(int(catalog_id))
# Format credentials in DerivaServer-expected format
creds = {
"bearer-token": get_deriva_token()
}
# Get the catalog model object to modify
server = DerivaServer("https", servername, creds)
catalog = server.connect_ermrest(catalog_id)
cat_model = catalog.getCatalogModel()
# If modifying ACL, set ACL
if acls:
# Ensure catalog owner still in ACLs - DERIVA forbids otherwise
acls['owner'] = list(set(acls['owner']).union(cat_model.acls['owner']))
# Apply acls
cat_model.acls.update(acls)
# Submit changes to server
cat_model.apply()
return {
"success": True
}
def main(server_name, catalog_id, credentials):
server = DerivaServer('https', server_name, credentials)
catalog = server.connect_ermrest(catalog_id)
catalog.dcctx['cid'] = "oneoff/model"
"""
set the table acls
"""
utils.set_table_acls(catalog, 'PDB', 'ihm_pseudo_site', table_acls)
utils.set_table_acls(catalog, 'PDB', 'ihm_cross_link_pseudo_site',
table_acls)
utils.set_table_acls(catalog, 'PDB', 'ihm_ensemble_sub_sample', table_acls)
utils.set_table_acls(catalog, 'PDB', 'ihm_data_transformation', table_acls)
utils.set_table_acls(catalog, 'PDB', 'ihm_derived_angle_restraint',
table_acls)
utils.set_table_acls(catalog, 'PDB', 'ihm_hdx_restraint', table_acls)
utils.set_table_acls(catalog, 'PDB', 'ihm_derived_dihedral_restraint',
table_acls)
utils.set_table_acls(catalog, 'PDB', 'struct_ref', table_acls)
utils.set_table_acls(catalog, 'PDB', 'struct_ref_seq', table_acls)
utils.set_table_acls(catalog, 'PDB', 'struct_ref_seq_dif', table_acls)
"""
set the table acl_bindings
"""
utils.set_table_acl_bindings(
catalog, 'PDB', 'ihm_pseudo_site',
literal_eval(
table_acl_bindings_pattern.replace(
'fkey_constraint_name', 'ihm_pseudo_site_structure_id_fkey')))
utils.set_table_acl_bindings(
catalog, 'PDB', 'ihm_cross_link_pseudo_site',
literal_eval(
table_acl_bindings_pattern.replace(
'fkey_constraint_name',
'ihm_cross_link_pseudo_site_structure_id_fkey')))
utils.set_table_acl_bindings(
catalog, 'PDB', 'ihm_ensemble_sub_sample',
literal_eval(
table_acl_bindings_pattern.replace(
'fkey_constraint_name',
'ihm_ensemble_sub_sample_structure_id_fkey')))
utils.set_table_acl_bindings(
catalog, 'PDB', 'ihm_data_transformation',
literal_eval(
table_acl_bindings_pattern.replace(
'fkey_constraint_name',
'ihm_data_transformation_structure_id_fkey')))
utils.set_table_acl_bindings(
catalog, 'PDB', 'ihm_hdx_restraint',
literal_eval(
table_acl_bindings_pattern.replace(
'fkey_constraint_name',
'ihm_hdx_restraint_structure_id_fkey')))
utils.set_table_acl_bindings(
catalog, 'PDB', 'ihm_derived_angle_restraint',
literal_eval(
table_acl_bindings_pattern.replace(
'fkey_constraint_name',
'ihm_derived_angle_restraint_structure_id_fkey')))
utils.set_table_acl_bindings(
catalog, 'PDB', 'ihm_derived_dihedral_restraint',
literal_eval(
table_acl_bindings_pattern.replace(
'fkey_constraint_name',
'ihm_derived_dihedral_restraint_structure_id_fkey')))
utils.set_table_acl_bindings(
catalog, 'PDB', 'struct_ref',
literal_eval(
table_acl_bindings_pattern.replace(
'fkey_constraint_name', 'struct_ref_structure_id_fkey')))
utils.set_table_acl_bindings(
catalog, 'PDB', 'struct_ref_seq',
literal_eval(
table_acl_bindings_pattern.replace(
'fkey_constraint_name', 'struct_ref_seq_structure_id_fkey')))
utils.set_table_acl_bindings(
catalog, 'PDB', 'struct_ref_seq_dif',
literal_eval(
table_acl_bindings_pattern.replace(
'fkey_constraint_name',
'struct_ref_seq_dif_structure_id_fkey')))
"""
set the foreign key acls
"""
utils.set_foreign_key_acls(catalog, 'PDB', 'ihm_pseudo_site',
'ihm_pseudo_site_structure_id_fkey',
foreign_key_acls)
utils.set_foreign_key_acls(catalog, 'PDB', 'ihm_cross_link_pseudo_site',
'ihm_cross_link_pseudo_site_structure_id_fkey',
foreign_key_acls)
utils.set_foreign_key_acls(catalog, 'PDB', 'ihm_ensemble_sub_sample',
'ihm_ensemble_sub_sample_structure_id_fkey',
foreign_key_acls)
utils.set_foreign_key_acls(catalog, 'PDB', 'ihm_data_transformation',
'ihm_data_transformation_structure_id_fkey',
foreign_key_acls)
utils.set_foreign_key_acls(catalog, 'PDB', 'ihm_hdx_restraint',
'ihm_hdx_restraint_structure_id_fkey',
foreign_key_acls)
utils.set_foreign_key_acls(
catalog, 'PDB', 'ihm_derived_angle_restraint',
'ihm_derived_angle_restraint_structure_id_fkey', foreign_key_acls)
utils.set_foreign_key_acls(
catalog, 'PDB', 'ihm_derived_dihedral_restraint',
'ihm_derived_dihedral_restraint_structure_id_fkey', foreign_key_acls)
utils.set_foreign_key_acls(catalog, 'PDB', 'struct_ref',
'struct_ref_structure_id_fkey',
foreign_key_acls)
utils.set_foreign_key_acls(catalog, 'PDB', 'struct_ref',
'struct_ref_db_name_fkey', foreign_key_acls)
utils.set_foreign_key_acls(catalog, 'PDB', 'struct_ref_seq',
'struct_ref_seq_structure_id_fkey',
foreign_key_acls)
utils.set_foreign_key_acls(catalog, 'PDB', 'struct_ref_seq_dif',
'struct_ref_seq_dif_structure_id_fkey',
foreign_key_acls)
utils.set_foreign_key_acls(catalog, 'PDB', 'struct_ref_seq_dif',
'struct_ref_seq_dif_details_fkey',
foreign_key_acls)
def deriva_ingest(servername, data_json_file, catalog_id=None, acls=None):
"""Perform an ingest to DERIVA into a catalog, using the CfdeDataPackage.
Arguments:
servername (str): The name of the DERIVA server.
data_json_file (str): The path to the JSON file with TableSchema data.
catalog_id (str or int): If updating an existing catalog, the existing catalog ID.
Default None, to create a new catalog.
acls (dict): The ACLs to set on the catalog.
Default None to use default ACLs.
Returns:
dict: The result of the ingest.
success (bool): True when the ingest was successful.
catalog_id (str): The catalog's ID.
"""
# Format credentials in DerivaServer-expected format
creds = {
"bearer-token": get_deriva_token()
}
# Get server object
server = DerivaServer("https", servername, creds)
# If ingesting into existing catalog, don't need to provision with schema
if catalog_id:
catalog_id = str(int(catalog_id))
catalog = server.connect_ermrest(catalog_id)
# Otherwise, we need to fetch the latest model for provisioning
else:
canon_schema = requests.get(CONFIG["DERIVA_SCHEMA_LOCATION"]).json()
with tempfile.TemporaryDirectory() as schema_dir:
schema_path = os.path.join(schema_dir, "model.json")
with open(schema_path, 'w') as f:
json.dump(canon_schema, f)
provisional_datapack = CfdeDataPackage(schema_path, verbose=False)
catalog = server.create_ermrest_catalog()
provisional_datapack.set_catalog(catalog)
provisional_datapack.provision()
# Now we create a datapackage to ingest the actual data
datapack = CfdeDataPackage(data_json_file, verbose=False)
# Catalog was created previously
datapack.set_catalog(catalog)
# Apply custom config (if possible - may fail if non-canon schema)
try:
datapack.apply_custom_config()
# Using non-canon schema is not failure unless Deriva rejects data
except Exception:
logger.info(f"Custom config skipped for {catalog.catalog.id}")
# Apply ACLs - either supplied or CfdeDataPackage default
# Defaults are set in .apply_custom_config(), which can fail
if acls is None:
acls = dict(CfdeDataPackage.catalog_acls)
# Ensure catalog owner still in ACLs - DERIVA forbids otherwise
acls['owner'] = list(set(acls['owner']).union(datapack.cat_model_root.acls['owner']))
# Apply acls
datapack.cat_model_root.acls.update(acls)
# Set ERMrest access
datapack.cat_model_root.table('public', 'ERMrest_Client').acls\
.update(datapack.ermrestclient_acls)
datapack.cat_model_root.table('public', 'ERMrest_Group').acls\
.update(datapack.ermrestclient_acls)
# Submit changes to server
datapack.cat_model_root.apply()
# Load data from files into DERIVA
# This is the step that will fail if the data are incorrect
datapack.load_data_files()
return {
"success": True,
"catalog_id": catalog.catalog_id
}
"""Minimal example of a possible rendering of the ETAS starter model.
Developer: Robert Schuler
Modifications: Philip Maechling
"""
"""Minimal example of a possible rendering of the ETAS starter model.
"""
from deriva.core import DerivaServer, get_credential
from deriva.chisel import Model, Schema, Table, Column, Key, ForeignKey, builtin_types, tag
# Connect to server and catalog ------------------------------------------------------------------#
hostname = 'forecast.derivacloud.org' # this is a dev server for throw-away work (change to 'forecast.derivacloud.org)
catalog_id = '5' # this was a throw-away catalog used to test this script (change to TBD)
model = Model.from_catalog(
DerivaServer('https', hostname, credentials=get_credential(hostname)).connect_ermrest(catalog_id)
)
# Cleanup ----------------------------------------------------------------------------------------#
if 'ETAS' in model.schemas:
# Purge anything so we can "do over" repeatedly at this point
model.schemas['ETAS'].drop(cascade=True)
# ETAS schema ------------------------------------------------------------------------------------#
# Create the "schema" to organize tables in the catalog "model"
model.create_schema(Schema.define('ETAS'))