def __init__(self, property_file_name):
self.props = IngestProps(property_file_name, required_props = ['nexus.token', 'ingest.api.url', 'search.api.url', 'uuid.api.url', 'dataset.uuid.file', 'globus.app.client.id', 'globus.app.client.secret'])
if len(sys.argv) >= 2:
self.id_file = sys.argv[1]
else:
self.id_file = self.props.get('dataset.uuid.file')
if string_helper.isBlank(self.id_file):
raise ErrorMessage("ERROR: A list of dataset uuids must be specified in " + self.prop_file_name + " as as property 'dataset.uuid.file' or as the first argument on the command line")
if not os.path.isfile(self.id_file):
raise ErrorMessage("ERROR: Input file " + self.id_file + " does not exist.")
base_file_name = os.path.splitext(os.path.basename(self.id_file))[0]
dir_path = file_helper.ensureTrailingSlash(os.path.dirname(self.id_file))
#set up log files, first for errors, second to record all actions
cur_time = time.strftime("%d-%m-%Y-%H-%M-%S")
error_log_filename = dir_path + base_file_name + "-errors." + cur_time + ".log"
self.error_logger = logging.getLogger('publish.datasets.err')
self.error_logger.setLevel(logging.INFO)
error_logFH = logging.FileHandler(error_log_filename)
self.error_logger.addHandler(error_logFH)
recording_log_filename = dir_path + base_file_name + "-rcding." + cur_time + ".log"
self.recording_logger = logging.getLogger('publish.datasets.rcd')
self.recording_logger.setLevel(logging.INFO)
recording_logFH = logging.FileHandler(recording_log_filename)
self.recording_logger.addHandler(recording_logFH)
#initialize variables, get required values from property file
self.dataset_info = None
self.dataset_info_tsv_path = None
self.token = self.props.get('nexus.token')
self.search_api_url = file_helper.ensureTrailingSlashURL(self.props.get('search.api.url'))
self.ingest_api_url = file_helper.ensureTrailingSlashURL(self.props.get('ingest.api.url'))
#initialize the auth helper and use it to get the
#user information for the person running the script
auth_helper = AuthHelper.create(self.props.get('globus.app.client.id'), self.props.get('globus.app.client.secret'))
user_info = auth_helper.getUserInfo(self.token, getGroups = True)
if isinstance(user_info, Response):
raise ErrorMessage("error validating auth token: " + user_info.get_data(as_text=True))
id_f = open(self.id_file, 'r')
id_lines = id_f.readlines()
id_f.close()
self.ds_ids = []
for id_line in id_lines:
if not string_helper.isBlank(id_line):
tl = id_line.strip()
if not tl.startswith('#'):
self.ds_ids.append(tl)
self.donors_to_reindex = []
self.set_acl_commands = []
def get(self, prop_name, required=False):
if not prop_name in self.props:
if required:
raise ErrorMessage("Required property " + prop_name +
" not found in " + self.prop_file_name)
else:
return None
val = self.props[prop_name]
if required and string_helper.isBlank(val):
raise ErrorMessage("Required property " + prop_name + " from " +
self.prop_file_name + " is blank")
return self.props[prop_name]
def __init__(self,
property_file_name,
argparse_args=None,
required_props=[]):
self.prop_file_name = property_file_name
if argparse_args is None and not os.path.isfile(property_file_name):
raise ErrorMessage("property file does not exist: " +
property_file_name)
if property_file_name is None:
if argparse_args is None:
raise ErrorMessage(
"property file name or argparse arguments required")
self.props = {}
for arg_key in argparse_args.keys():
val = argparse_args[arg_key]
if not val is None:
if isinstance(val, list):
if len(val) == 1:
self.props[arg_key] = val[0]
elif len(val) > 1:
self.props[arg_key] = val
else:
propMgr = Property()
self.props = propMgr.load_property_files(property_file_name)
not_found = []
for required_prop in required_props:
if required_prop not in self.props or string_helper.isBlank(
self.props[required_prop]):
not_found.append(required_prop)
if len(not_found) > 0:
if len(not_found) == 1:
prop = "property"
else:
prop = "properties"
raise ErrorMessage("Required " + prop + " not found in " +
property_file_name + ":[" +
string_helper.listToCommaSeparated(not_found) +
"]")
def __init__(self, property_file_name):
self.log_filename = "sample-metadata-import" + time.strftime(
"%d-%m-%Y-%H-%M-%S") + ".log"
self.logger = logging.getLogger('entity.service')
self.logger.setLevel(logging.INFO)
logFH = logging.FileHandler(self.log_filename)
self.logger.addHandler(logFH)
self.dataset_info = None
self.dataset_info_tsv_path = None
self.props = IngestProps(property_file_name,
required_props=[
'entity.api.url', 'nexus.token',
'globus.app.client.id',
'globus.app.client.secret', 'uuid.api.url'
])
self.entity_api_url = self.props.get('entity.api.url')
self.token = self.props.get('nexus.token')
if len(sys.argv) >= 2:
self.metadata_file = sys.argv[1]
else:
self.metadata_file = self.props.get('sample.metadata.tsv.file')
if string_helper.isBlank(self.metadata_file):
raise ErrorMessage(
"ERROR: Metadata file must be specified in " +
self.prop_file_name +
" or as the first argument on the command line")
if not os.path.isfile(self.metadata_file):
raise ErrorMessage("ERROR: Input file " + self.metadata_file +
" does not exist.")
metadata_file = TSVFile(self.metadata_file)
if not 'sample_id' in metadata_file.get_header_keys():
raise ErrorMessage(
"ERROR: Required sample_id column not found in " +
self.metadata_file)
self.metadata = metadata_file.get_records()
self.uuid_helper = UUIDHelper(ingest_props=self.props)
self.ingested_ids = {}
def __init__(self, property_file_name, required_props=[]):
self.prop_file_name = property_file_name
if not os.path.isfile(property_file_name):
raise ErrorMessage("property file does not exist: " +
property_file_name)
propMgr = Property()
self.props = propMgr.load_property_files(property_file_name)
not_found = []
for required_prop in required_props:
if required_prop not in self.props or string_helper.isBlank(
self.props[required_prop]):
not_found.append(required_prop)
if len(not_found) > 0:
if len(not_found) == 1:
prop = "property"
else:
prop = "properties"
raise ErrorMessage("Required " + prop + " not found in " +
property_file_name + ":[" +
string_helper.listToCommaSeparated(not_found) +
"]")
def __init__(self, file_name):
if string_helper.isBlank(file_name):
raise Exception("ERROR: TSV Filename Not Found")
if not os.path.isfile(file_name):
raise ErrorMessage("TSV file " + file_name + " not found.")
self.records = []
self.header = []
with open(file_name, newline='') as tsvfile:
reader = csv.DictReader(tsvfile, delimiter='\t')
first = True
for row in reader:
data_row = {}
for key in row.keys():
if first: self.header.append(key)
data_row[key] = row[key]
self.records.append(data_row)
if first: first = False
def __init__(self,
ingest_props=None,
auth_token=None,
uuid_service_url=None,
globus_app_id=None,
globus_app_secret=None):
self.props = ingest_props
self.token = self._get_property("nexus.token", auth_token)
self.uuid_url = self._get_property("uuid.api.url", uuid_service_url)
self.globus_app_id = self._get_property("globus.app.client.id",
globus_app_id).strip()
self.globus_app_secret = self._get_property("globus.app.client.secret",
globus_app_secret).strip()
auth_helper = AuthHelper(self.globus_app_id, self.globus_app_secret)
user_info = auth_helper.getUserInfo(self.token, getGroups=True)
if isinstance(user_info, Response):
raise ErrorMessage("error validating auth token: " +
user_info.get_data(as_text=True))
def new_uuid(self, entity_type, generate_doi=False):
url = file_helper.ensureTrailingSlash(self.uuid_url) + "hmuuid"
headers = {
'Authorization': 'Bearer ' + self.token,
'Content-Type': 'application/json'
}
gen_doi = "false"
if generate_doi: gen_doi = "true"
resp = requests.post(url,
json={
'entityType': entity_type,
'generateDOI': gen_doi
},
headers=headers)
status_code = resp.status_code
if status_code < 200 or status_code >= 300:
raise ErrorMessage("Unable to generate UUID " +
str(resp.status_code) + ":" + str(resp.text))
vals = resp.json()
return vals[0]