def create_report(self):
""" Create a sample report and an aggregate report via a system call """
logprefix = os.path.abspath(
self.expand_path(os.path.join(self.logpath, "{}-{}".format(
self.projectid, self.sampleid))))
try:
if not create_folder(os.path.dirname(logprefix)):
logprefix = None
except AttributeError:
logprefix = None
with chdir(self.expand_path(self.reportpath)):
# create the ign_sample_report for this sample
cl = self.report_sample.split(' ')
cl.extend(["--samples",self.sampleid])
call_external_command(
cl,
with_log_files=(logprefix is not None),
prefix="{}_sample".format(logprefix))
# estimate the delivery date for this sample to 0.5 days ahead
cl = self.report_aggregate.split(' ')
cl.extend([
"--samples_extra",
json.dumps({
self.sampleid: {
"delivered": "{}(expected)".format(
_timestamp(days=0.5))}})
])
call_external_command(
cl,
with_log_files=(logprefix is not None),
prefix="{}_aggregate".format(logprefix))
def _archive_run((run, days, force, compress_only)):
""" Archive a specific run to swestore
:param str run: Run directory
:param int days: Days to consider a run old
:param bool force: Force the archiving even if the run is not complete
:param bool compress_only: Only compress the run without sending it to swestore
"""
def _send_to_swestore(f, dest, remove=True):
""" Send file to swestore checking adler32 on destination and eventually
removing the file from disk
:param str f: File to remove
:param str dest: Destination directory in Swestore
:param bool remove: If True, remove original file from source
"""
if not filesystem.is_in_swestore(f):
logger.info("Sending {} to swestore".format(f))
misc.call_external_command("iput -K -P {file} {dest}".format(file=f, dest=dest), with_log_files=True)
logger.info("Run {} sent correctly and checksum was okay.".format(f))
if remove:
logger.info("Removing run".format(f))
os.remove(f)
else:
logger.warn("Run {} is already in Swestore, not sending it again nor removing from the disk".format(f))
# Create state file to say that the run is being archived
open("{}.archiving".format(run.split(".")[0]), "w").close()
if run.endswith("bz2"):
if os.stat(run).st_mtime < time.time() - (86400 * days):
_send_to_swestore(run, CONFIG.get("storage").get("irods").get("irodsHome"))
else:
logger.info("Run {} is not {} days old yet. Not archiving".format(run, str(days)))
else:
rta_file = os.path.join(run, "RTAComplete.txt")
if not os.path.exists(rta_file) and not force:
logger.warn(
(
"Run {} doesn't seem to be completed and --force option was "
"not enabled, not archiving the run".format(run)
)
)
if force or (os.path.exists(rta_file) and os.stat(rta_file).st_mtime < time.time() - (86400 * days)):
logger.info("Compressing run {}".format(run))
# Compress with pbzip2
misc.call_external_command("tar --use-compress-program=pbzip2 -cf {run}.tar.bz2 {run}".format(run=run))
logger.info("Run {} successfully compressed! Removing from disk...".format(run))
shutil.rmtree(run)
if not compress_only:
_send_to_swestore("{}.tar.bz2".format(run), CONFIG.get("storage").get("irods").get("irodsHome"))
else:
logger.info("Run {} is not completed or is not {} days old yet. Not archiving".format(run, str(days)))
os.remove("{}.archiving".format(run.split(".")[0]))
def transfer_run(run, analysis=True):
""" Transfer a run to the analysis server. Will add group R/W permissions to
the run directory in the destination server so that the run can be processed
by any user/account in that group (i.e a functional account...). Run will be
moved to data_dir/nosync after transferred.
:param str run: Run directory
:param bool analysis: Trigger analysis on remote server
"""
command_line = ['rsync', '-av']
# Add R/W permissions to the group
command_line.append('--chmod=g+rw')
# rsync works in a really funny way, if you don't understand this, refer to
# this note: http://silentorbit.com/notes/2013/08/rsync-by-extension/
command_line.append("--include=*/")
for to_include in CONFIG['analysis']['analysis_server']['sync']['include']:
command_line.append("--include={}".format(to_include))
command_line.extend(["--exclude=*", "--prune-empty-dirs"])
r_user = CONFIG['analysis']['analysis_server']['user']
r_host = CONFIG['analysis']['analysis_server']['host']
r_dir = CONFIG['analysis']['analysis_server']['sync']['data_archive']
remote = "{}@{}:{}".format(r_user, r_host, r_dir)
command_line.extend([run, remote])
# Create temp file indicating that the run is being transferred
try:
open(os.path.join(run, 'transferring'), 'w').close()
except IOError as e:
logger.error("Cannot create a file in {}. Check the run name, and the permissions.".format(run))
raise e
started = ("Started transfer of run {} on {}"
.format(os.path.basename(run), datetime.now()))
logger.info(started)
# In this particular case we want to capture the exception because we want
# to delete the transfer file
try:
misc.call_external_command(command_line, with_log_files=True)
except subprocess.CalledProcessError as exception:
os.remove(os.path.join(run, 'transferring'))
raise exception
t_file = os.path.join(CONFIG['analysis']['status_dir'], 'transfer.tsv')
logger.info('Adding run {} to {}'
.format(os.path.basename(run), t_file))
with open(t_file, 'a') as tranfer_file:
tsv_writer = csv.writer(tranfer_file, delimiter='\t')
tsv_writer.writerow([os.path.basename(run), str(datetime.now())])
os.remove(os.path.join(run, 'transferring'))
#Now, let's move the run to nosync
archive_run(run)
if analysis:
trigger_analysis(run)
def transfer_run(self, t_file, analysis):
""" Transfer a run to the analysis server. Will add group R/W permissions to
the run directory in the destination server so that the run can be processed
by any user/account in that group (i.e a functional account...).
:param str t_file: File where to put the transfer information
:param bool analysis: Trigger analysis on remote server
"""
# TODO: check the run type and build the correct rsync command
command_line = ['rsync', '-Lav']
# Add R/W permissions to the group
command_line.append('--chmod=g+rw')
# rsync works in a really funny way, if you don't understand this, refer to
# this note: http://silentorbit.com/notes/2013/08/rsync-by-extension/
# this horrible thing here avoids data dup when we use multiple indexes in a lane/FC
command_line.append("--exclude=Demultiplexing_*/*_*")
command_line.append("--include=*/")
for to_include in self.CONFIG['analysis_server']['sync']['include']:
command_line.append("--include={}".format(to_include))
command_line.extend(["--exclude=*", "--prune-empty-dirs"])
r_user = self.CONFIG['analysis_server']['user']
r_host = self.CONFIG['analysis_server']['host']
r_dir = self.CONFIG['analysis_server']['sync']['data_archive']
remote = "{}@{}:{}".format(r_user, r_host, r_dir)
command_line.extend([self.run_dir, remote])
# Create temp file indicating that the run is being transferred
try:
open(os.path.join(self.run_dir, 'transferring'), 'w').close()
except IOError as e:
logger.error("Cannot create a file in {}. "
"Check the run name, and the permissions.".format(self.id))
raise e
started = ("Started transfer of run {} on {}".format(self.id, datetime.now()))
logger.info(started)
# In this particular case we want to capture the exception because we want
# to delete the transfer file
try:
misc.call_external_command(command_line, with_log_files=True,
prefix="", log_dir=self.run_dir)
except subprocess.CalledProcessError as exception:
os.remove(os.path.join(self.run_dir, 'transferring'))
raise exception
logger.info('Adding run {} to {}'.format(self.id, t_file))
with open(t_file, 'a') as tranfer_file:
tsv_writer = csv.writer(tranfer_file, delimiter='\t')
tsv_writer.writerow([self.id, str(datetime.now())])
os.remove(os.path.join(self.run_dir, 'transferring'))
if analysis:
# This needs to pass the runtype (i.e., Xten or HiSeq) and start the correct pipeline
self.trigger_analysis()
def _archive_run((run, seconds, force, compress_only)):
""" Archive a specific run to swestore
:param str run: Run directory
:param int seconds: Days/hours converted as seconds to check
:param bool force: Force the archiving even if the run is not complete
:param bool compress_only: Only compress the run without sending it to swestore
"""
def _send_to_swestore(f, dest, remove=True):
""" Send file to swestore checking adler32 on destination and eventually
removing the file from disk
:param str f: File to remove
:param str dest: Destination directory in Swestore
:param bool remove: If True, remove original file from source
"""
if not filesystem.is_in_swestore(f):
logger.info("Sending {} to swestore".format(f))
misc.call_external_command('iput -R swestoreArchCacheResc -P {file} {dest}'.format(file=f, dest=dest),
with_log_files=True, prefix=f.replace('.tar.bz2',''), log_dir="swestore_logs")
logger.info('Run {} sent to swestore.'.format(f))
if remove:
logger.info('Removing run'.format(f))
os.remove(f)
else:
logger.warn('Run {} is already in Swestore, not sending it again nor removing from the disk'.format(f))
# Create state file to say that the run is being archived
open("{}.archiving".format(run.split('.')[0]), 'w').close()
if run.endswith('bz2'):
if os.stat(run).st_mtime < time.time() - seconds:
_send_to_swestore(run, CONFIG.get('storage').get('irods').get('irodsHome'))
else:
logger.info("Run {} is not older than given time yet. Not archiving".format(run))
else:
rta_file = os.path.join(run, finished_run_indicator)
if not os.path.exists(rta_file) and not force:
logger.warn(("Run {} doesn't seem to be completed and --force option was "
"not enabled, not archiving the run".format(run)))
if force or (os.path.exists(rta_file) and os.stat(rta_file).st_mtime < time.time() - seconds):
logger.info("Compressing run {}".format(run))
# Compress with pbzip2
misc.call_external_command('tar --use-compress-program=pbzip2 -cf {run}.tar.bz2 {run}'.format(run=run))
logger.info('Run {} successfully compressed! Removing from disk...'.format(run))
shutil.rmtree(run)
if not compress_only:
_send_to_swestore('{}.tar.bz2'.format(run), CONFIG.get('storage').get('irods').get('irodsHome'))
else:
logger.info("Run {} is not completed or is not older than given time yet. Not archiving".format(run))
os.remove("{}.archiving".format(run.split('.')[0]))
def create_report(self):
""" Create a final aggregate report via a system call """
logprefix = os.path.abspath(
self.expand_path(os.path.join(self.logpath, self.projectid)))
try:
if not create_folder(os.path.dirname(logprefix)):
logprefix = None
except AttributeError:
logprefix = None
with chdir(self.expand_path(self.reportpath)):
cl = self.report_aggregate.split(' ')
call_external_command(
cl,
with_log_files=(logprefix is not None),
prefix="{}_aggregate".format(logprefix))
def cleanup_swestore(days, dry_run=False):
"""Remove archived runs from swestore
:param int days: Threshold days to check and remove
"""
days = check_days('swestore', days, config)
if not days:
return
runs = filesystem.list_runs_in_swestore(path=CONFIG.get('cleanup').get('swestore').get('root'))
for run in runs:
date = run.split('_')[0]
if misc.days_old(date) > days:
if dry_run:
logger.info('Will remove file {} from swestore'.format(run))
continue
misc.call_external_command('irm -f {}'.format(run))
logger.info('Removed file {} from swestore'.format(run))
def cleanup_swestore(seconds, dry_run=False):
"""Remove archived runs from swestore
:param int seconds: Days/hours converted as seconds to check
"""
seconds = check_default(site, seconds, CONFIG)
if not seconds:
return
runs = filesystem.list_runs_in_swestore(path=CONFIG.get('cleanup').get('swestore').get('root'))
for run in runs:
date = run.split('_')[0]
if misc.to_seconds(misc.days_old(date)) > seconds:
if dry_run:
logger.info('Will remove file {} from swestore'.format(run))
continue
misc.call_external_command('irm -f {}'.format(run))
logger.info('Removed file {} from swestore'.format(run))
def _send_to_swestore(f, dest, remove=True):
""" Send file to swestore checking adler32 on destination and eventually
removing the file from disk
:param str f: File to remove
:param str dest: Destination directory in Swestore
:param bool remove: If True, remove original file from source
"""
if not filesystem.is_in_swestore(f):
logger.info("Sending {} to swestore".format(f))
misc.call_external_command("iput -K -P {file} {dest}".format(file=f, dest=dest), with_log_files=True)
logger.info("Run {} sent correctly and checksum was okay.".format(f))
if remove:
logger.info("Removing run".format(f))
os.remove(f)
else:
logger.warn("Run {} is already in Swestore, not sending it again nor removing from the disk".format(f))
def _send_to_swestore(f, dest, remove=True):
""" Send file to swestore checking adler32 on destination and eventually
removing the file from disk
:param str f: File to remove
:param str dest: Destination directory in Swestore
:param bool remove: If True, remove original file from source
"""
if not filesystem.is_in_swestore(f):
logger.info("Sending {} to swestore".format(f))
misc.call_external_command('iput -R swestoreArchCacheResc -P {file} {dest}'.format(file=f, dest=dest),
with_log_files=True, prefix=f.replace('.tar.bz2',''), log_dir="swestore_logs")
logger.info('Run {} sent to swestore.'.format(f))
if remove:
logger.info('Removing run'.format(f))
os.remove(f)
else:
logger.warn('Run {} is already in Swestore, not sending it again nor removing from the disk'.format(f))
def transfer(self, transfer_log=None):
"""
Execute the transfer as set up by this instance and, if requested,
validate the transfer.
:param string transfer_log: path prefix to log files where stderr
and stdout streams will be directed if this option is specified
:returns True on success, False if the validation failed
:raises transfer.TransferError: if src_path or dest_path were not valid
:raises transfer.RsyncError: if the rsync command did not exit successfully
"""
self.validate_src_path()
self.validate_dest_path()
command = [self.CMD] + self.format_options() + [self.src_path,self.remote_path()]
try:
call_external_command(
command,
with_log_files=(transfer_log is not None),
prefix=transfer_log)
except subprocess.CalledProcessError as e:
raise RsyncError(e)
return (not self.validate) or self.validate_transfer()
def transfer(self, transfer_log=None):
"""Execute the transfer as set up by this instance and, if requested,
validate the transfer.
:param string transfer_log: path prefix to log files where stderr
and stdout streams will be directed if this option is specified
:returns True on success, False if the validation failed
:raises transfer.TransferError: if src_path or dest_path were not valid
:raises transfer.RsyncError: if the rsync command did not exit successfully
"""
self.validate_src_path()
self.validate_dest_path()
command = [self.CMD] + self.format_options() + [
self.src_path, self.remote_path()
]
try:
call_external_command(command,
with_log_files=(transfer_log is not None),
prefix=transfer_log)
except subprocess.CalledProcessError as e:
raise RsyncError(e)
return (not self.validate) or self.validate_transfer()
def _send_to_swestore(f, dest, remove=True):
""" Send file to swestore checking adler32 on destination and eventually
removing the file from disk
:param str f: File to remove
:param str dest: Destination directory in Swestore
:param bool remove: If True, remove original file from source
"""
if not filesystem.is_in_swestore(f):
logger.info("Sending {} to swestore".format(f))
misc.call_external_command(
'iput -R swestoreArchCacheResc -P {file} {dest}'.format(
file=f, dest=dest),
with_log_files=True,
prefix=f.replace('.tar.bz2', ''),
log_dir="swestore_logs")
logger.info('Run {} sent to swestore.'.format(f))
if remove:
logger.info('Removing run'.format(f))
os.remove(f)
else:
logger.warn(
'Run {} is already in Swestore, not sending it again nor removing from the disk'
.format(f))
def transfer_run(self, t_file, analysis, mail_recipients=None):
""" Transfer a run to the analysis server. Will add group R/W permissions to
the run directory in the destination server so that the run can be processed
by any user/account in that group (i.e a functional account...).
:param str t_file: File where to put the transfer information
:param bool analysis: Trigger analysis on remote server
"""
# TODO: check the run type and build the correct rsync command
# The option -a implies -o and -g which is not the desired behaviour
command_line = ['rsync', '-Lav', '--no-o', '--no-g']
# Add R/W permissions to the group
command_line.append('--chmod=g+rw')
# This horrible thing here avoids data dup when we use multiple indexes in a lane/FC
command_line.append("--exclude=Demultiplexing_*/*_*")
command_line.append("--include=*/")
for to_include in self.CONFIG['analysis_server']['sync']['include']:
command_line.append("--include={}".format(to_include))
command_line.extend(["--exclude=*", "--prune-empty-dirs"])
r_user = self.CONFIG['analysis_server']['user']
r_host = self.CONFIG['analysis_server']['host']
r_dir = self.CONFIG['analysis_server']['sync']['data_archive']
remote = "{}@{}:{}".format(r_user, r_host, r_dir)
command_line.extend([self.run_dir, remote])
# Create temp file indicating that the run is being transferred
try:
open(os.path.join(self.run_dir, 'transferring'), 'w').close()
except IOError as e:
logger.error("Cannot create a file in {}. "
"Check the run name, and the permissions.".format(
self.id))
raise e
started = ("Started transfer of run {} on {}".format(
self.id, datetime.now()))
logger.info(started)
# In this particular case we want to capture the exception because we want
# to delete the transfer file
try:
msge_text = "I am about to transfer with this command \n{}".format(
command_line)
logger.info(msge_text)
misc.call_external_command(command_line,
with_log_files=True,
prefix="",
log_dir=self.run_dir)
except subprocess.CalledProcessError as exception:
os.remove(os.path.join(self.run_dir, 'transferring'))
#Send an email notifying that the transfer failed
runname = self.id
sbt = ("Rsync of run {} failed".format(runname))
msg = """ Rsync of data for run {run} has failed!
Raised the following exception: {e}
""".format(run=runname, e=exception)
if mail_recipients:
send_mail(sbt, msg, mail_recipients)
raise exception
logger.info('Adding run {} to {}'.format(self.id, t_file))
with open(t_file, 'a') as tranfer_file:
tsv_writer = csv.writer(tranfer_file, delimiter='\t')
tsv_writer.writerow([self.id, str(datetime.now())])
os.remove(os.path.join(self.run_dir, 'transferring'))
#Send an email notifying that the transfer was successful
runname = self.id
sbt = ("Rsync of data for run {} to Irma has finished".format(runname))
msg = """ Rsync of data for run {run} to Irma has finished!
The run is available at : https://genomics-status.scilifelab.se/flowcells/{run}
""".format(run=runname)
if mail_recipients:
send_mail(sbt, msg, mail_recipients)
if analysis:
# This needs to pass the runtype (i.e., Xten or HiSeq) and start the correct pipeline
self.trigger_analysis()
def _archive_run((run, seconds, force, compress_only)):
""" Archive a specific run to swestore
:param str run: Run directory
:param int seconds: Days/hours converted as seconds to check
:param bool force: Force the archiving even if the run is not complete
:param bool compress_only: Only compress the run without sending it to swestore
"""
def _send_to_swestore(f, dest, remove=True):
""" Send file to swestore checking adler32 on destination and eventually
removing the file from disk
:param str f: File to remove
:param str dest: Destination directory in Swestore
:param bool remove: If True, remove original file from source
"""
if not filesystem.is_in_swestore(f):
logger.info("Sending {} to swestore".format(f))
misc.call_external_command(
'iput -R swestoreArchCacheResc -P {file} {dest}'.format(
file=f, dest=dest),
with_log_files=True,
prefix=f.replace('.tar.bz2', ''),
log_dir="swestore_logs")
logger.info('Run {} sent to swestore.'.format(f))
if remove:
logger.info('Removing run'.format(f))
os.remove(f)
else:
logger.warn(
'Run {} is already in Swestore, not sending it again nor removing from the disk'
.format(f))
# Create state file to say that the run is being archived
open("{}.archiving".format(run.split('.')[0]), 'w').close()
if run.endswith('bz2'):
if os.stat(run).st_mtime < time.time() - seconds:
_send_to_swestore(
run,
CONFIG.get('storage').get('irods').get('irodsHome'))
else:
logger.info(
"Run {} is not older than given time yet. Not archiving".
format(run))
else:
rta_file = os.path.join(run, finished_run_indicator)
if not os.path.exists(rta_file) and not force:
logger.warn(
("Run {} doesn't seem to be completed and --force option was "
"not enabled, not archiving the run".format(run)))
if force or (os.path.exists(rta_file)
and os.stat(rta_file).st_mtime < time.time() - seconds):
logger.info("Compressing run {}".format(run))
# Compress with pbzip2
misc.call_external_command(
'tar --use-compress-program=pbzip2 -cf {run}.tar.bz2 {run}'.
format(run=run))
logger.info(
'Run {} successfully compressed! Removing from disk...'.format(
run))
shutil.rmtree(run)
if not compress_only:
_send_to_swestore(
'{}.tar.bz2'.format(run),
CONFIG.get('storage').get('irods').get('irodsHome'))
else:
logger.info(
"Run {} is not completed or is not older than given time yet. Not archiving"
.format(run))
os.remove("{}.archiving".format(run.split('.')[0]))
def test_call_external_command_fail(self):
"""Call external command should handle error."""
command = 'ls -E'
with self.assertRaises(subprocess.CalledProcessError):
misc.call_external_command(command)
