def setup(self):
probe = self.mc_globus_acl_interface.set_user_globus_id()
if not probe:
self.log.error("Users globus_user_id is undefined")
raise RequiredAttributeException(
"Globus user id is not set correctly")
self.log.info("Using globus id: {}".format(probe))
self.mc_globus_acl_interface.get_cc_transfer_client()
project = get_project_by_id(self.project_id, self.apikey)
self.log.info("Get file list for project = {} ({})".format(
project.name, project.id))
directory = project.get_top_directory()
self.file_list = []
self.path_list = []
path = ""
self.recursively_add_directory(path, directory)
if not self.file_list:
self.log.info("No files found.")
return False
self.log.info("Found {} files.".format(len(self.file_list)))
self.log.info("Found {} directory paths.".format(len(self.path_list)))
for file in self.file_list:
self.log.debug("File: {} - {}".format(file.name, file.path))
for path in self.path_list:
self.log.debug("Path {}".format(path))
return True
def verify(self, metadata, apikey):
verified = True
project = get_project_by_id(metadata.project_id, apikey=apikey)
if not project:
print("Could not find project: " + metadata.project_id)
verified = False
else:
metadata.project = project
print("Found project: " + project.name + " (" + project.id + ")")
experiment = self.get_experiment(project, metadata.experiment_id)
if not experiment:
print("Could not find experiment: " + metadata.experiment_id)
verified = False
else:
metadata.experiment = experiment
print("Found experiment: " + experiment.name + " (" +
experiment.id + ")")
processes = experiment.get_all_processes()
process_table = self.make_process_table(processes)
missing = []
for process_record in metadata.process_metadata:
if not process_record['id'] in process_table:
missing.append(process_record['id'])
if missing:
verified = False
for process_id in missing:
print("Could not find process: " + process_id)
else:
print("Found all processes (" + str(len(process_table)) + ").")
metadata.process_table = process_table
if verified:
return metadata
return None
def move_file_to_project(self, project_id, external_path):
project = get_project_by_id(project_id, apikey=self.apikey)
top_dir = project.get_top_directory()
file_name = os_path.basename(external_path)
self.log.info("external_path = {}, file_name = {}".format(external_path, file_name))
file = top_dir.add_file(file_name, external_path)
return file
def get_project_excel_files(self, project_id):
self.log.info("get_project_excel_files: project_id = {}".format(project_id))
project = get_project_by_id(project_id, apikey=self.apikey)
self.log.info("got project = {}".format(project.name))
if not project:
raise AttributeError("Project not found for projectId = {}".format(project_id))
directory = project.get_top_directory()
found_files = self.recursively_find_excel_files(directory, [], '/')
self.log.info("found {} excel files".format(len(found_files)))
return found_files
def build(self,
spread_sheet_path,
data_dir_path,
project_id,
exp_name,
exp_description=None):
if not project_id:
raise RequiredAttributeException("Project id not defined")
self.project = get_project_by_id(project_id, apikey=self.apikey)
if not self.project:
raise NoSuchItem(
"Missing unexpected missing project; project_id = {}".format(
project_id))
self.metadata.set_project_id(self.project.id)
self.experiment_name = exp_name
self.experiment_description = exp_description
if not self.experiment_description:
self.experiment_description = "Experiment from ETL: excel file path = {}".format(
spread_sheet_path)
self.log.info("Building Experiment from ETL spreadsheet: ")
self.log.info(" project: name = {}, id = {}".format(
self.project.name, self.project.id))
self.log.info(" spreadsheet = {}".format(spread_sheet_path))
self.log.info(" data = {}".format(data_dir_path))
self._set_etl_source_date_from_path(self.project, spread_sheet_path)
self.metadata.set_input_information(spread_sheet_path, data_dir_path)
if data_dir_path:
self.data_directory = self._get_project_directory_from_path(
data_dir_path)
self.log.debug(" data directory name = {}".format(
self.data_directory.name))
self.suppress_data_upload = not self.data_directory
self._create_base_experiment()
self._set_row_positions()
self._set_col_positions()
# noinspection PyBroadException
try:
self._sweep()
except BaseException:
self.log.exception("Error in sweep")
raise
self._write_metadata()
self.log.info("Context project: " + self.project.name + " (" +
self.project.id + ")")
self.log.info("With Experiment: " + self.experiment.name + " (" +
self.experiment.id + ")")
return {
"status": "ok",
"project_id": self.project.id,
"experiment_id": self.experiment.id
}