def post(self, runfolder):
monitored_dir = self.config["monitored_directory"]
StartHandler._validate_runfolder_exists(runfolder, monitored_dir)
request_data = json.loads(self.request.body)
path_to_runfolder = os.path.join(monitored_dir, runfolder)
path_to_md5_sum_file = os.path.join(monitored_dir, runfolder, request_data["path_to_md5_sum_file"])
if not StartHandler._validate_md5sum_path(path_to_runfolder, path_to_md5_sum_file):
raise ArteriaUsageException("{} is not a valid file!".format(path_to_md5_sum_file))
md5sum_log_dir = self.config["md5_log_directory"]
if not StartHandler._is_valid_log_dir(md5sum_log_dir):
raise ArteriaUsageException("{} is not a directory.!".format(md5sum_log_dir))
md5sum_log_file = "{}/{}_{}".format(md5sum_log_dir,
runfolder,
datetime.datetime.now().isoformat())
cmd = " ".join(["md5sum -c", path_to_md5_sum_file])
job_id = self.runner_service.start(cmd,
nbr_of_cores=1,
run_dir=monitored_dir,
stdout=md5sum_log_file,
stderr=md5sum_log_file)
status_end_point = "{0}://{1}{2}".format(
self.request.protocol,
self.request.host,
self.reverse_url("status", job_id))
response_data = {
"job_id": job_id,
"service_version": version,
"link": status_end_point,
"state": State.STARTED,
"md5sum_log": md5sum_log_file}
self.set_status(202, reason="started processing")
self.write_object(response_data)
def post(self, runfolder_archive):
monitored_dir = self.config["path_to_archive_root"]
if not UploadHandler._validate_runfolder_exists(runfolder_archive, monitored_dir):
response_data = {"service_version": version, "state": State.ERROR}
self.set_status(500, reason="{} is not found under {}!".format(runfolder_archive, path_to_archive_root))
self.write_object(response_data)
return
path_to_runfolder = os.path.join(monitored_dir, runfolder_archive)
dsmc_log_dir = self.config["dsmc_log_directory"]
uniq_id = str(uuid.uuid4())
if not UploadHandler._is_valid_log_dir(dsmc_log_dir):
raise ArteriaUsageException("{} is not a directory!".format(dsmc_log_dir))
# TODO: Need to put the logs in the commands as well.
dsmc_log_file = "{}/dsmc_{}_{}-{}".format(dsmc_log_dir,
runfolder_archive,
uniq_id,
# description,
datetime.datetime.now().isoformat())
# cmd = "export DSM_LOG={} && dsmc archive {} -subdir=yes -desc={}".format(dsmc_log_file,
# runfolder,
# description)
cmd = "dsmc archive {}/ -subdir=yes -description={}".format(path_to_runfolder, uniq_id)
# FIXME: echo is just used when testing return codes locally.
#cmd = "echo 'ANS1809W ANS2000W Test run started.' && echo ANS9999W && echo ANS1809W && exit 8" #false
#cmd = "echo 'ANS1809W Test run started.' && echo ANS1809W && exit 8"
job_id = self.runner_service.start(cmd, nbr_of_cores=1, run_dir=monitored_dir, stdout=dsmc_log_file, stderr=dsmc_log_file)
status_end_point = "{0}://{1}{2}".format(
self.request.protocol,
self.request.host,
self.reverse_url("status", job_id))
response_data = {
"job_id": job_id,
"service_version": version,
"link": status_end_point,
"state": State.STARTED,
"dsmc_log": dsmc_log_file}
self.set_status(202, reason="started processing")
self.write_object(response_data)
def validate_output(self):
def _parent_dir(d):
return os.path.abspath(os.path.join(d, os.path.pardir))
abs_path_of_allowed_dirs = map(
os.path.abspath,
self.config.general_config['allowed_output_folders'])
is_located_in_parent_dir = _parent_dir(
self.config.output) in abs_path_of_allowed_dirs
if not is_located_in_parent_dir:
error_string = "Invalid output directory {} was specified." \
" Allowed dirs were: {}".format(self.config.output,
self.config.general_config['allowed_output_folders'])
log.error(error_string)
raise ArteriaUsageException(error_string)
def post(self, job_id):
"""
Stops the job with the specified id.
:param job_id: of job to stop, or set to "all" to stop all jobs
"""
try:
if job_id == "all":
log.info("Attempting to stop all jobs.")
self.runner_service.stop_all()
log.info("Stopped all jobs!")
self.set_status(200)
elif job_id:
log.info("Attempting to stop job: {}".format(job_id))
self.runner_service.stop(job_id)
self.set_status(200)
else:
ArteriaUsageException("Unknown job to stop")
except ArteriaUsageException as e:
log.warning("Failed stopping job: {}. Message: ".format(job_id, e.message))
self.send_error(500, reason=e.message)
def _create_archive(oldtree, newtree, exclude_dirs=[], exclude_extensions=[]):
try:
content = os.listdir(oldtree)
for entry in content:
oldpath = os.path.join(oldtree, entry)
newpath = os.path.join(newtree, entry)
if os.path.isdir(oldpath) and entry not in exclude_dirs:
os.mkdir(newpath)
CreateDirHandler._create_archive(oldpath, newpath, exclude_dirs, exclude_extensions)
elif os.path.isfile(oldpath):
_, ext = os.path.splitext(oldpath)
if ext not in exclude_extensions:
os.symlink(oldpath, newpath)
else:
log.debug("Skipping {} because it is excluded".format(oldpath))
else:
log.debug("Skipping {} because it is excluded".format(oldpath))
except OSError, msg:
errmsg = "Error when creating archive directory: {}".format(msg)
log.debug(errmsg)
raise ArteriaUsageException(errmsg)
def build_index_string(length_tuple):
"""
Builds the index mask string
:param length_tuple: a tuple of the length of the index in the samplesheet and in the read, e.g. (3, 5)
:return: a index string, e.g. "i5n*" or "n*" or "i3", depending on the situation.
"""
length_of_index_in_samplesheet = length_tuple[0]
length_of_index_read = length_tuple[1]
difference = length_of_index_read - length_of_index_in_samplesheet
if not difference >= 0:
raise ArteriaUsageException(
"Sample sheet indicates that index is "
"longer than what was read by the sequencer!")
if length_of_index_in_samplesheet == 0:
# If there is no index in the samplesheet, ignore it in the base-mask
return "n*"
if difference > 0:
# Pad the end if there is a difference here.
return "i" + str(length_of_index_in_samplesheet) + "n*"
else:
return "i" + str(length_of_index_in_samplesheet)
def construct_command(self):
##################################
# First run configureBcl2fastq.pl
##################################
# Assumes configureBclToFastq.pl on path
commandline_collection = [
"configureBclToFastq.pl",
"--input-dir",
self.config.base_calls_input,
"--sample-sheet",
self.config.samplesheet_file,
"--output-dir",
self.config.output,
"--fastq-cluster-count 0", # No upper-limit on number of clusters per output file.
"--force" # overwrite output if it exists.
]
if self.config.barcode_mismatches:
commandline_collection.append("--mismatches " +
self.config.barcode_mismatches)
if self.config.tiles:
commandline_collection.append("--tiles " + self.config.tiles)
if self.config.use_base_mask:
commandline_collection.append("--use_bases_mask " +
self.config.use_base_mask)
else:
length_of_indexes = Bcl2FastqConfig.get_length_of_indexes(
self.config.runfolder_input)
samplesheet = Samplesheet(self.config.samplesheet_file)
is_single_read_run = Bcl2FastqConfig.is_single_read(
self.config.runfolder_input)
lanes_and_base_mask = \
Bcl2FastqConfig.get_bases_mask_per_lane_from_samplesheet(
samplesheet,
length_of_indexes,
is_single_read_run)
base_masks_as_set = set(lanes_and_base_mask.values())
if len(base_masks_as_set) is 1:
raise ArteriaUsageException(
"For bcl2fastq 1.8.4 there is no support for "
"mixing different bases masks for different lanes")
# Here we are forced to use the same bases mask was always used for all lanes.
commandline_collection.append("--use_bases_mask " +
lanes_and_base_mask.values()[0])
if self.config.additional_args:
commandline_collection.append(self.config.additional_args)
##################################
# Then run make
##################################
commandline_collection.append(" && make -j{0}".format(
self.config.nbr_of_cores))
command = " ".join(commandline_collection)
log.debug("Generated command: " + command)
return command
def create_config_from_request(self, runfolder, request_body):
"""
For the specified runfolder, will look it up from the place setup in the
configuration, and then parse additional data from the request_data object.
This can be used to override any default setting in the resulting Bcl2FastqConfig
instance.
:param runfolder: name of the runfolder we want to create a config for
:param request_body: the body of the request. Can be empty, in which case if will not be loaded.
:return: an instances of Bcl2FastqConfig
"""
if request_body:
request_data = json.loads(request_body)
else:
request_data = {}
# TODO Make sure to escape them for sec. reasons.
bcl2fastq_version = ""
runfolder_input = ""
samplesheet = ""
output = ""
barcode_mismatches = ""
tiles = ""
use_base_mask = ""
create_indexes = False
additional_args = ""
runfolder_base_path = self.config["runfolder_path"]
runfolder_input = "{0}/{1}".format(runfolder_base_path, runfolder)
if not os.path.isdir(runfolder_input):
raise ArteriaUsageException(
"No such file: {0}".format(runfolder_input))
if "bcl2fastq_version" in request_data:
bcl2fastq_version = request_data["bcl2fastq_version"]
if "output" in request_data:
output = request_data["output"]
if "samplesheet" in request_data:
samplesheet = request_data["samplesheet"]
if "barcode_mismatches" in request_data:
barcode_mismatches = request_data["barcode_mismatches"]
if "tiles" in request_data:
tiles = request_data["tiles"]
if "use_base_mask" in request_data:
use_base_mask = request_data["use_base_mask"]
if "create_indexes" in request_data:
if request_data["create_indexes"] == "True":
create_indexes = True
if "additional_args" in request_data:
additional_args = request_data["additional_args"]
config = Bcl2FastqConfig(self.config, bcl2fastq_version,
runfolder_input, output, samplesheet,
barcode_mismatches, tiles, use_base_mask,
create_indexes, additional_args)
return config