def test_set_config_options(self):
"""
Test writing to config file, make sure writen values are written correctly
:return:
"""
# set up config
config.set_config_file(
os.path.join(path_to_module, "example_config.conf"))
config.setup()
# Make sure id is initially set to what we expect
self.assertEqual(config.read_config_option('client_id'), 'uploader')
# Set and test to a new id
config.set_config_options(client_id="new_id")
self.assertEqual(config.read_config_option('client_id'), "new_id")
def get_parser_from_config():
"""
Uses the 'parser' field in the config file to find and return the parser
:return:
"""
parser_instance = config.read_config_option("parser")
return parsers.parser_factory(parser_instance)
def write_directory_status(directory_status, run_id=None):
"""
Writes a status to the status file:
Overwrites anything that is in the file
Writes a timestamp to the time of last written
:param directory_status: DirectoryStatus object containing status to write to directory
:param run_id: optional, when used, the run id will be included in the status file,
along with the irida instance the run is uploaded to.
:return: None
"""
if not os.access(directory_status.directory, os.W_OK): # Cannot access upload directory
raise exceptions.DirectoryError("Cannot access directory", directory_status.directory)
uploader_info_file = os.path.join(directory_status.directory, STATUS_FILE_NAME)
if run_id:
json_data = {STATUS_FIELD: directory_status.status,
MESSAGE_FIELD: directory_status.message,
DATE_TIME_FIELD: _get_date_time_field(),
RUN_ID_FIELD: run_id,
IRIDA_INSTANCE_FIELD: config.read_config_option('base_url')}
else:
json_data = {STATUS_FIELD: directory_status.status,
MESSAGE_FIELD: directory_status.message,
DATE_TIME_FIELD: _get_date_time_field()}
with open(uploader_info_file, "w") as json_file:
json.dump(json_data, json_file, indent=4, sort_keys=True)
json_file.write("\n")
def initialize_api_from_config():
"""
Loads the api parameters from the config file and initializes the api with them
:return: the api instance
"""
client_id = config.read_config_option("client_id")
client_secret = config.read_config_option("client_secret")
base_url = config.read_config_option("base_url")
username = config.read_config_option("username")
password = config.read_config_option("password")
return _initialize_api(client_id=client_id,
client_secret=client_secret,
base_url=base_url,
username=username,
password=password)
def logging_end_block():
"""
Logs an block to the console and file that indicates the end of an upload run.
:return:
"""
logging.info("==================================================")
logging.info("----------------ENDING UPLOAD RUN-----------------")
logging.info("==================================================")
if config.read_config_option("readonly", bool, False) is False:
logger.remove_directory_logger()
def directory_has_readonly_conflict(directory):
"""
Returns True if directory is not writable and readonly is not True
Else returns False
:param directory:
:return: boolean
"""
if (config.read_config_option("readonly", bool, False) is False
and not os.access(directory, os.W_OK)):
return True
else:
return False
def logging_start_block(directory):
"""
Logs an information block to the console and file which indicates the start of an upload run.
Includes the uploader version number set in this module
:return:
"""
if config.read_config_option("readonly", bool, False) is False:
logger.add_log_to_directory(directory)
logging.info("==================================================")
logging.info("---------------STARTING UPLOAD RUN----------------")
logging.info("Uploader Version {}".format(VERSION_NUMBER))
logging.info("Logging to file in: " + logger.get_user_log_dir())
logging.info("==================================================")
def _set_and_write_directory_status(directory_status,
status,
message=None,
run_id=None):
"""
Given a DirectoryStatus object, sets the status and message, and then writes to the directory status directory
:param directory_status: DirectoryStatus object
:param status: a valid DirectoryStatus status
:param message: string
:param run_id: optional, if provided, the run id and irida instance will be included when written
:return:
"""
if config.read_config_option("readonly", bool, False) is False:
directory_status.status = status
directory_status.message = message
progress.write_directory_status(directory_status, run_id)
def test_read_config_option(self):
"""
Test writing to config file, make sure writen values are written correctly
:return:
"""
# set up config
config.set_config_file(os.path.join(path_to_module,
"test_config.conf"))
config.setup()
# Test that all the parameters loaded from file are correct
self.assertEqual(config.read_config_option('client_id'), 'uploader')
self.assertEqual(config.read_config_option('client_secret'), 'secret')
self.assertEqual(config.read_config_option('username'), 'admin')
self.assertEqual(config.read_config_option('password'), 'password1')
self.assertEqual(config.read_config_option('base_url'),
'http://localhost:8080/irida-latest/api/')
self.assertEqual(config.read_config_option('parser'), 'miseq')
self.assertEqual(config.read_config_option('readonly', bool), False)
def write_directory_status(directory_status):
"""
Writes a status to the status file:
Overwrites anything that is in the file
Writes a timestamp to the time of last written
:param directory_status: DirectoryStatus object containing status to write to directory
:return: None
"""
if config.read_config_option("readonly", bool, False) is False:
if not os.access(directory_status.directory, os.W_OK): # Cannot access upload directory
raise exceptions.DirectoryError("Cannot access directory", directory_status.directory)
json_data = directory_status.to_json_dict()
uploader_info_file = os.path.join(directory_status.directory, STATUS_FILE_NAME)
with open(uploader_info_file, "w") as json_file:
json.dump(json_data, json_file, indent=4, sort_keys=True)
json_file.write("\n")
def validate_file_size_minimum(sequencing_run):
"""
Validate the files in a SequencingRun object have the minimum file size requirement from the config
:param sequencing_run: SequencingRun object to validate
:return: ValidationResult object with list of errors if any
"""
minimum_file_size = config.read_config_option("minimum_file_size", int, 0)
validation_result = model.ValidationResult()
for p in sequencing_run.project_list:
for s in p.sample_list:
# do validation of file size
if not _file_size_is_valid(s.sequence_file, minimum_file_size):
error_msg = "File size for sample `{}`is smaller than configured minimum of `{} KB`. " \
"Please verify your data.".format(s.sample_name, minimum_file_size)
validation_result.add_error(
FileSizeError(error_msg, s.sequence_file))
return validation_result
def run_is_ready_with_delay(directory_status):
"""
Expects a NEW or DELAYED directory status
If a NEW run is given, and the config is set to delay new runs, the run will be set to DELAYED, otherwise it's ready
If a DELAYED run is given, the run is ready if enough time has passed, otherwise it is not ready yet.
Writes to directory status file when set to DELAYED
:param directory_status:
:return: True when run is ready for upload, otherwise False
"""
delay_minutes = config.read_config_option("delay", expected_type=int)
logging.debug("delay_minutes is set to: " + str(delay_minutes))
# Check if run is new, check if there's a delay
if directory_status.status_equals(DirectoryStatus.NEW):
if delay_minutes > 0:
_set_run_delayed(directory_status)
logging.info("Run has been delayed for {} minutes.".format(delay_minutes))
run_is_ready = False
else:
logging.info("No delay time given for NEW run. Continuing...")
run_is_ready = True
# If run was delayed, check if run can now be uploaded
elif directory_status.status_equals(DirectoryStatus.DELAYED):
if _delayed_time_has_passed(directory_status, delay_minutes):
logging.info("Delayed run is now ready for upload. Continuing...")
run_is_ready = True
else:
logging.info("Delayed run is still not ready for upload.")
run_is_ready = False
# This case should be imposable
else:
raise Exception("Function called with invalid directory status, This should never happen.")
return run_is_ready
def get_directory_status(directory, required_file_list):
"""
Gets the directory status based off using 'irida_uploader_status.info' files to track progress
:param directory: the directory to search for a run
:param required_file_list: optional param: a list of required files that
are required for that run to be considered valid. Example: ['SampleSheet.csv']
:return: directory and status dictionary
"""
result = DirectoryStatus(directory)
# Verify directory is readable
if not os.access(directory, os.R_OK):
result.status = DirectoryStatus.INVALID
result.message = 'Directory cannot be read. Please check permissions'
return result
# If readonly is not set, verify directory is writable
if config.read_config_option("readonly", bool, False) is False:
if not os.access(directory, os.W_OK):
result.status = DirectoryStatus.INVALID
result.message = 'Directory cannot be written to. Please check permissions or use readonly mode'
return result
file_list = next(
os.walk(directory))[2] # Gets the list of files in the directory
# Legacy upload catch
# When the irida-miseq-uploader (old uploader) ran it generated a .miseqUploaderInfo file
# To prevent uploading runs that used this old system, we assume runs with this file are COMPLETE
# By default they will not be picked up automatically with --batch because they are set to COMPLETE,
# but they can still be uploaded using the --force option
if '.miseqUploaderInfo' in file_list:
result.status = DirectoryStatus.COMPLETE
result.message = "Legacy uploader run. Set to complete to avoid uploading duplicate data."
return result
for file_name in required_file_list:
if file_name not in file_list:
result.status = DirectoryStatus.INVALID
result.message = 'Directory is missing required file with filename {}'.format(
file_name)
return result
if STATUS_FILE_NAME not in file_list: # no irida_uploader_status.info file yet, has not been uploaded
result.status = DirectoryStatus.NEW
return result
# Must check status of upload to determine if upload is completed
uploader_info_file = os.path.join(directory, STATUS_FILE_NAME)
with open(uploader_info_file, "rb") as reader:
data = reader.read().decode()
info_file = json.loads(data)
status = info_file[STATUS_FIELD]
if status in DirectoryStatus.VALID_STATUS_LIST:
result.status = status
if MESSAGE_FIELD in info_file:
result.message = info_file[MESSAGE_FIELD]
else:
result.message = None
else: # the status found in the file is not in the defined list
raise exceptions.DirectoryError("Invalid Status in status file",
directory)
return result
def irida_instance(self):
if self._irida_instance is None:
self._irida_instance = config.read_config_option('base_url')
return self._irida_instance