def test_set_config_options(self): """ Test writing to config file, make sure writen values are written correctly :return: """ # set up config config.set_config_file( os.path.join(path_to_module, "example_config.conf")) config.setup() # Make sure id is initially set to what we expect self.assertEqual(config.read_config_option('client_id'), 'uploader') # Set and test to a new id config.set_config_options(client_id="new_id") self.assertEqual(config.read_config_option('client_id'), "new_id")
def get_parser_from_config(): """ Uses the 'parser' field in the config file to find and return the parser :return: """ parser_instance = config.read_config_option("parser") return parsers.parser_factory(parser_instance)
def write_directory_status(directory_status, run_id=None): """ Writes a status to the status file: Overwrites anything that is in the file Writes a timestamp to the time of last written :param directory_status: DirectoryStatus object containing status to write to directory :param run_id: optional, when used, the run id will be included in the status file, along with the irida instance the run is uploaded to. :return: None """ if not os.access(directory_status.directory, os.W_OK): # Cannot access upload directory raise exceptions.DirectoryError("Cannot access directory", directory_status.directory) uploader_info_file = os.path.join(directory_status.directory, STATUS_FILE_NAME) if run_id: json_data = {STATUS_FIELD: directory_status.status, MESSAGE_FIELD: directory_status.message, DATE_TIME_FIELD: _get_date_time_field(), RUN_ID_FIELD: run_id, IRIDA_INSTANCE_FIELD: config.read_config_option('base_url')} else: json_data = {STATUS_FIELD: directory_status.status, MESSAGE_FIELD: directory_status.message, DATE_TIME_FIELD: _get_date_time_field()} with open(uploader_info_file, "w") as json_file: json.dump(json_data, json_file, indent=4, sort_keys=True) json_file.write("\n")
def initialize_api_from_config(): """ Loads the api parameters from the config file and initializes the api with them :return: the api instance """ client_id = config.read_config_option("client_id") client_secret = config.read_config_option("client_secret") base_url = config.read_config_option("base_url") username = config.read_config_option("username") password = config.read_config_option("password") return _initialize_api(client_id=client_id, client_secret=client_secret, base_url=base_url, username=username, password=password)
def logging_end_block(): """ Logs an block to the console and file that indicates the end of an upload run. :return: """ logging.info("==================================================") logging.info("----------------ENDING UPLOAD RUN-----------------") logging.info("==================================================") if config.read_config_option("readonly", bool, False) is False: logger.remove_directory_logger()
def directory_has_readonly_conflict(directory): """ Returns True if directory is not writable and readonly is not True Else returns False :param directory: :return: boolean """ if (config.read_config_option("readonly", bool, False) is False and not os.access(directory, os.W_OK)): return True else: return False
def logging_start_block(directory): """ Logs an information block to the console and file which indicates the start of an upload run. Includes the uploader version number set in this module :return: """ if config.read_config_option("readonly", bool, False) is False: logger.add_log_to_directory(directory) logging.info("==================================================") logging.info("---------------STARTING UPLOAD RUN----------------") logging.info("Uploader Version {}".format(VERSION_NUMBER)) logging.info("Logging to file in: " + logger.get_user_log_dir()) logging.info("==================================================")
def _set_and_write_directory_status(directory_status, status, message=None, run_id=None): """ Given a DirectoryStatus object, sets the status and message, and then writes to the directory status directory :param directory_status: DirectoryStatus object :param status: a valid DirectoryStatus status :param message: string :param run_id: optional, if provided, the run id and irida instance will be included when written :return: """ if config.read_config_option("readonly", bool, False) is False: directory_status.status = status directory_status.message = message progress.write_directory_status(directory_status, run_id)
def test_read_config_option(self): """ Test writing to config file, make sure writen values are written correctly :return: """ # set up config config.set_config_file(os.path.join(path_to_module, "test_config.conf")) config.setup() # Test that all the parameters loaded from file are correct self.assertEqual(config.read_config_option('client_id'), 'uploader') self.assertEqual(config.read_config_option('client_secret'), 'secret') self.assertEqual(config.read_config_option('username'), 'admin') self.assertEqual(config.read_config_option('password'), 'password1') self.assertEqual(config.read_config_option('base_url'), 'http://localhost:8080/irida-latest/api/') self.assertEqual(config.read_config_option('parser'), 'miseq') self.assertEqual(config.read_config_option('readonly', bool), False)
def write_directory_status(directory_status): """ Writes a status to the status file: Overwrites anything that is in the file Writes a timestamp to the time of last written :param directory_status: DirectoryStatus object containing status to write to directory :return: None """ if config.read_config_option("readonly", bool, False) is False: if not os.access(directory_status.directory, os.W_OK): # Cannot access upload directory raise exceptions.DirectoryError("Cannot access directory", directory_status.directory) json_data = directory_status.to_json_dict() uploader_info_file = os.path.join(directory_status.directory, STATUS_FILE_NAME) with open(uploader_info_file, "w") as json_file: json.dump(json_data, json_file, indent=4, sort_keys=True) json_file.write("\n")
def validate_file_size_minimum(sequencing_run): """ Validate the files in a SequencingRun object have the minimum file size requirement from the config :param sequencing_run: SequencingRun object to validate :return: ValidationResult object with list of errors if any """ minimum_file_size = config.read_config_option("minimum_file_size", int, 0) validation_result = model.ValidationResult() for p in sequencing_run.project_list: for s in p.sample_list: # do validation of file size if not _file_size_is_valid(s.sequence_file, minimum_file_size): error_msg = "File size for sample `{}`is smaller than configured minimum of `{} KB`. " \ "Please verify your data.".format(s.sample_name, minimum_file_size) validation_result.add_error( FileSizeError(error_msg, s.sequence_file)) return validation_result
def run_is_ready_with_delay(directory_status): """ Expects a NEW or DELAYED directory status If a NEW run is given, and the config is set to delay new runs, the run will be set to DELAYED, otherwise it's ready If a DELAYED run is given, the run is ready if enough time has passed, otherwise it is not ready yet. Writes to directory status file when set to DELAYED :param directory_status: :return: True when run is ready for upload, otherwise False """ delay_minutes = config.read_config_option("delay", expected_type=int) logging.debug("delay_minutes is set to: " + str(delay_minutes)) # Check if run is new, check if there's a delay if directory_status.status_equals(DirectoryStatus.NEW): if delay_minutes > 0: _set_run_delayed(directory_status) logging.info("Run has been delayed for {} minutes.".format(delay_minutes)) run_is_ready = False else: logging.info("No delay time given for NEW run. Continuing...") run_is_ready = True # If run was delayed, check if run can now be uploaded elif directory_status.status_equals(DirectoryStatus.DELAYED): if _delayed_time_has_passed(directory_status, delay_minutes): logging.info("Delayed run is now ready for upload. Continuing...") run_is_ready = True else: logging.info("Delayed run is still not ready for upload.") run_is_ready = False # This case should be imposable else: raise Exception("Function called with invalid directory status, This should never happen.") return run_is_ready
def get_directory_status(directory, required_file_list): """ Gets the directory status based off using 'irida_uploader_status.info' files to track progress :param directory: the directory to search for a run :param required_file_list: optional param: a list of required files that are required for that run to be considered valid. Example: ['SampleSheet.csv'] :return: directory and status dictionary """ result = DirectoryStatus(directory) # Verify directory is readable if not os.access(directory, os.R_OK): result.status = DirectoryStatus.INVALID result.message = 'Directory cannot be read. Please check permissions' return result # If readonly is not set, verify directory is writable if config.read_config_option("readonly", bool, False) is False: if not os.access(directory, os.W_OK): result.status = DirectoryStatus.INVALID result.message = 'Directory cannot be written to. Please check permissions or use readonly mode' return result file_list = next( os.walk(directory))[2] # Gets the list of files in the directory # Legacy upload catch # When the irida-miseq-uploader (old uploader) ran it generated a .miseqUploaderInfo file # To prevent uploading runs that used this old system, we assume runs with this file are COMPLETE # By default they will not be picked up automatically with --batch because they are set to COMPLETE, # but they can still be uploaded using the --force option if '.miseqUploaderInfo' in file_list: result.status = DirectoryStatus.COMPLETE result.message = "Legacy uploader run. Set to complete to avoid uploading duplicate data." return result for file_name in required_file_list: if file_name not in file_list: result.status = DirectoryStatus.INVALID result.message = 'Directory is missing required file with filename {}'.format( file_name) return result if STATUS_FILE_NAME not in file_list: # no irida_uploader_status.info file yet, has not been uploaded result.status = DirectoryStatus.NEW return result # Must check status of upload to determine if upload is completed uploader_info_file = os.path.join(directory, STATUS_FILE_NAME) with open(uploader_info_file, "rb") as reader: data = reader.read().decode() info_file = json.loads(data) status = info_file[STATUS_FIELD] if status in DirectoryStatus.VALID_STATUS_LIST: result.status = status if MESSAGE_FIELD in info_file: result.message = info_file[MESSAGE_FIELD] else: result.message = None else: # the status found in the file is not in the defined list raise exceptions.DirectoryError("Invalid Status in status file", directory) return result
def irida_instance(self): if self._irida_instance is None: self._irida_instance = config.read_config_option('base_url') return self._irida_instance