def create_report(self):
     """ Create a sample report and an aggregate report via a system call """
     logprefix = os.path.abspath(
         self.expand_path(os.path.join(self.logpath, "{}-{}".format(
             self.projectid, self.sampleid))))
     try:
         if not create_folder(os.path.dirname(logprefix)):
             logprefix = None
     except AttributeError:
         logprefix = None
     with chdir(self.expand_path(self.reportpath)):
         # create the ign_sample_report for this sample
         cl = self.report_sample.split(' ')
         cl.extend(["--samples",self.sampleid])
         call_external_command(
             cl,
             with_log_files=(logprefix is not None),
             prefix="{}_sample".format(logprefix))
         # estimate the delivery date for this sample to 0.5 days ahead
         cl = self.report_aggregate.split(' ')
         cl.extend([
             "--samples_extra",
             json.dumps({
                 self.sampleid: {
                     "delivered": "{}(expected)".format(
                         _timestamp(days=0.5))}})
         ])
         call_external_command(
             cl,
             with_log_files=(logprefix is not None),
             prefix="{}_aggregate".format(logprefix))
Exemple #2
0
def _archive_run((run, days, force, compress_only)):
    """ Archive a specific run to swestore

    :param str run: Run directory
    :param int days: Days to consider a run old
    :param bool force: Force the archiving even if the run is not complete
    :param bool compress_only: Only compress the run without sending it to swestore
    """

    def _send_to_swestore(f, dest, remove=True):
        """ Send file to swestore checking adler32 on destination and eventually
        removing the file from disk

        :param str f: File to remove
        :param str dest: Destination directory in Swestore
        :param bool remove: If True, remove original file from source
        """
        if not filesystem.is_in_swestore(f):
            logger.info("Sending {} to swestore".format(f))
            misc.call_external_command("iput -K -P {file} {dest}".format(file=f, dest=dest), with_log_files=True)
            logger.info("Run {} sent correctly and checksum was okay.".format(f))
            if remove:
                logger.info("Removing run".format(f))
                os.remove(f)
        else:
            logger.warn("Run {} is already in Swestore, not sending it again nor removing from the disk".format(f))

    # Create state file to say that the run is being archived
    open("{}.archiving".format(run.split(".")[0]), "w").close()
    if run.endswith("bz2"):
        if os.stat(run).st_mtime < time.time() - (86400 * days):
            _send_to_swestore(run, CONFIG.get("storage").get("irods").get("irodsHome"))
        else:
            logger.info("Run {} is not {} days old yet. Not archiving".format(run, str(days)))
    else:
        rta_file = os.path.join(run, "RTAComplete.txt")
        if not os.path.exists(rta_file) and not force:
            logger.warn(
                (
                    "Run {} doesn't seem to be completed and --force option was "
                    "not enabled, not archiving the run".format(run)
                )
            )
        if force or (os.path.exists(rta_file) and os.stat(rta_file).st_mtime < time.time() - (86400 * days)):
            logger.info("Compressing run {}".format(run))
            # Compress with pbzip2
            misc.call_external_command("tar --use-compress-program=pbzip2 -cf {run}.tar.bz2 {run}".format(run=run))
            logger.info("Run {} successfully compressed! Removing from disk...".format(run))
            shutil.rmtree(run)
            if not compress_only:
                _send_to_swestore("{}.tar.bz2".format(run), CONFIG.get("storage").get("irods").get("irodsHome"))
        else:
            logger.info("Run {} is not completed or is not {} days old yet. Not archiving".format(run, str(days)))
    os.remove("{}.archiving".format(run.split(".")[0]))
Exemple #3
0
def transfer_run(run, analysis=True):
    """ Transfer a run to the analysis server. Will add group R/W permissions to
    the run directory in the destination server so that the run can be processed
    by any user/account in that group (i.e a functional account...). Run will be
    moved to data_dir/nosync after transferred.

    :param str run: Run directory
    :param bool analysis: Trigger analysis on remote server
    """
    command_line = ['rsync', '-av']
    # Add R/W permissions to the group
    command_line.append('--chmod=g+rw')
    # rsync works in a really funny way, if you don't understand this, refer to
    # this note: http://silentorbit.com/notes/2013/08/rsync-by-extension/
    command_line.append("--include=*/")
    for to_include in CONFIG['analysis']['analysis_server']['sync']['include']:
        command_line.append("--include={}".format(to_include))
    command_line.extend(["--exclude=*", "--prune-empty-dirs"])
    r_user = CONFIG['analysis']['analysis_server']['user']
    r_host = CONFIG['analysis']['analysis_server']['host']
    r_dir = CONFIG['analysis']['analysis_server']['sync']['data_archive']
    remote = "{}@{}:{}".format(r_user, r_host, r_dir)
    command_line.extend([run, remote])

    # Create temp file indicating that the run is being transferred
    try:
        open(os.path.join(run, 'transferring'), 'w').close()
    except IOError as e:
        logger.error("Cannot create a file in {}. Check the run name, and the permissions.".format(run))
        raise e
    started = ("Started transfer of run {} on {}"
               .format(os.path.basename(run), datetime.now()))
    logger.info(started)
    # In this particular case we want to capture the exception because we want
    # to delete the transfer file
    try:
        misc.call_external_command(command_line, with_log_files=True)
    except subprocess.CalledProcessError as exception:
        os.remove(os.path.join(run, 'transferring'))
        raise exception

    t_file = os.path.join(CONFIG['analysis']['status_dir'], 'transfer.tsv')
    logger.info('Adding run {} to {}'
                .format(os.path.basename(run), t_file))
    with open(t_file, 'a') as tranfer_file:
        tsv_writer = csv.writer(tranfer_file, delimiter='\t')
        tsv_writer.writerow([os.path.basename(run), str(datetime.now())])
    os.remove(os.path.join(run, 'transferring'))

    #Now, let's move the run to nosync
    archive_run(run)

    if analysis:
        trigger_analysis(run)
Exemple #4
0
    def transfer_run(self, t_file, analysis):
        """ Transfer a run to the analysis server. Will add group R/W permissions to
            the run directory in the destination server so that the run can be processed
            by any user/account in that group (i.e a functional account...). 
            :param str t_file: File where to put the transfer information
            :param bool analysis: Trigger analysis on remote server
        """
        # TODO: check the run type and build the correct rsync command
        command_line = ['rsync', '-Lav']
        # Add R/W permissions to the group
        command_line.append('--chmod=g+rw')
        # rsync works in a really funny way, if you don't understand this, refer to
        # this note: http://silentorbit.com/notes/2013/08/rsync-by-extension/
        # this horrible thing here avoids data dup when we use multiple indexes in a lane/FC
        command_line.append("--exclude=Demultiplexing_*/*_*") 
        command_line.append("--include=*/")
        for to_include in self.CONFIG['analysis_server']['sync']['include']:
            command_line.append("--include={}".format(to_include))
        command_line.extend(["--exclude=*", "--prune-empty-dirs"])
        r_user = self.CONFIG['analysis_server']['user']
        r_host = self.CONFIG['analysis_server']['host']
        r_dir = self.CONFIG['analysis_server']['sync']['data_archive']
        remote = "{}@{}:{}".format(r_user, r_host, r_dir)
        command_line.extend([self.run_dir, remote])

        # Create temp file indicating that the run is being transferred
        try:
            open(os.path.join(self.run_dir, 'transferring'), 'w').close()
        except IOError as e:
            logger.error("Cannot create a file in {}. "
                         "Check the run name, and the permissions.".format(self.id))
            raise e
        started = ("Started transfer of run {} on {}".format(self.id, datetime.now()))
        logger.info(started)
        # In this particular case we want to capture the exception because we want
        # to delete the transfer file
        try:
            misc.call_external_command(command_line, with_log_files=True, 
                                       prefix="", log_dir=self.run_dir)
        except subprocess.CalledProcessError as exception:
            os.remove(os.path.join(self.run_dir, 'transferring'))
            raise exception

        logger.info('Adding run {} to {}'.format(self.id, t_file))
        with open(t_file, 'a') as tranfer_file:
            tsv_writer = csv.writer(tranfer_file, delimiter='\t')
            tsv_writer.writerow([self.id, str(datetime.now())])
        os.remove(os.path.join(self.run_dir, 'transferring'))

        if analysis:
            # This needs to pass the runtype (i.e., Xten or HiSeq) and start the correct pipeline
            self.trigger_analysis()
Exemple #5
0
def _archive_run((run, seconds, force, compress_only)):
    """ Archive a specific run to swestore

    :param str run: Run directory
    :param int seconds: Days/hours converted as seconds to check
    :param bool force: Force the archiving even if the run is not complete
    :param bool compress_only: Only compress the run without sending it to swestore
    """

    def _send_to_swestore(f, dest, remove=True):
        """ Send file to swestore checking adler32 on destination and eventually
        removing the file from disk

        :param str f: File to remove
        :param str dest: Destination directory in Swestore
        :param bool remove: If True, remove original file from source
        """
        if not filesystem.is_in_swestore(f):
            logger.info("Sending {} to swestore".format(f))
            misc.call_external_command('iput -R swestoreArchCacheResc -P {file} {dest}'.format(file=f, dest=dest),
                    with_log_files=True, prefix=f.replace('.tar.bz2',''), log_dir="swestore_logs")
            logger.info('Run {} sent to swestore.'.format(f))
            if remove:
                logger.info('Removing run'.format(f))
                os.remove(f)
        else:
            logger.warn('Run {} is already in Swestore, not sending it again nor removing from the disk'.format(f))

    # Create state file to say that the run is being archived
    open("{}.archiving".format(run.split('.')[0]), 'w').close()
    if run.endswith('bz2'):
        if os.stat(run).st_mtime < time.time() - seconds:
            _send_to_swestore(run, CONFIG.get('storage').get('irods').get('irodsHome'))
        else:
            logger.info("Run {} is not older than given time yet. Not archiving".format(run))
    else:
        rta_file = os.path.join(run, finished_run_indicator)
        if not os.path.exists(rta_file) and not force:
            logger.warn(("Run {} doesn't seem to be completed and --force option was "
                      "not enabled, not archiving the run".format(run)))
        if force or (os.path.exists(rta_file) and os.stat(rta_file).st_mtime < time.time() - seconds):
            logger.info("Compressing run {}".format(run))
            # Compress with pbzip2
            misc.call_external_command('tar --use-compress-program=pbzip2 -cf {run}.tar.bz2 {run}'.format(run=run))
            logger.info('Run {} successfully compressed! Removing from disk...'.format(run))
            shutil.rmtree(run)
            if not compress_only:
                _send_to_swestore('{}.tar.bz2'.format(run), CONFIG.get('storage').get('irods').get('irodsHome'))
        else:
            logger.info("Run {} is not completed or is not older than given time yet. Not archiving".format(run))
    os.remove("{}.archiving".format(run.split('.')[0]))
 def create_report(self):
     """ Create a final aggregate report via a system call """
     logprefix = os.path.abspath(
         self.expand_path(os.path.join(self.logpath, self.projectid)))
     try:
         if not create_folder(os.path.dirname(logprefix)):
             logprefix = None
     except AttributeError:
         logprefix = None
     with chdir(self.expand_path(self.reportpath)):
         cl = self.report_aggregate.split(' ')
         call_external_command(
             cl,
             with_log_files=(logprefix is not None),
             prefix="{}_aggregate".format(logprefix))
Exemple #7
0
def cleanup_swestore(days, dry_run=False):
    """Remove archived runs from swestore

    :param int days: Threshold days to check and remove
    """
    days = check_days('swestore', days, config)
    if not days:
        return
    runs = filesystem.list_runs_in_swestore(path=CONFIG.get('cleanup').get('swestore').get('root'))
    for run in runs:
        date = run.split('_')[0]
        if misc.days_old(date) > days:
            if dry_run:
                logger.info('Will remove file {} from swestore'.format(run))
                continue
            misc.call_external_command('irm -f {}'.format(run))
            logger.info('Removed file {} from swestore'.format(run))
Exemple #8
0
def cleanup_swestore(seconds, dry_run=False):
    """Remove archived runs from swestore

    :param int seconds: Days/hours converted as seconds to check
    """
    seconds = check_default(site, seconds, CONFIG)
    if not seconds:
        return
    runs = filesystem.list_runs_in_swestore(path=CONFIG.get('cleanup').get('swestore').get('root'))
    for run in runs:
        date = run.split('_')[0]
        if misc.to_seconds(misc.days_old(date)) > seconds:
            if dry_run:
                logger.info('Will remove file {} from swestore'.format(run))
                continue
            misc.call_external_command('irm -f {}'.format(run))
            logger.info('Removed file {} from swestore'.format(run))
Exemple #9
0
    def _send_to_swestore(f, dest, remove=True):
        """ Send file to swestore checking adler32 on destination and eventually
        removing the file from disk

        :param str f: File to remove
        :param str dest: Destination directory in Swestore
        :param bool remove: If True, remove original file from source
        """
        if not filesystem.is_in_swestore(f):
            logger.info("Sending {} to swestore".format(f))
            misc.call_external_command("iput -K -P {file} {dest}".format(file=f, dest=dest), with_log_files=True)
            logger.info("Run {} sent correctly and checksum was okay.".format(f))
            if remove:
                logger.info("Removing run".format(f))
                os.remove(f)
        else:
            logger.warn("Run {} is already in Swestore, not sending it again nor removing from the disk".format(f))
Exemple #10
0
    def _send_to_swestore(f, dest, remove=True):
        """ Send file to swestore checking adler32 on destination and eventually
        removing the file from disk

        :param str f: File to remove
        :param str dest: Destination directory in Swestore
        :param bool remove: If True, remove original file from source
        """
        if not filesystem.is_in_swestore(f):
            logger.info("Sending {} to swestore".format(f))
            misc.call_external_command('iput -R swestoreArchCacheResc -P {file} {dest}'.format(file=f, dest=dest),
                    with_log_files=True, prefix=f.replace('.tar.bz2',''), log_dir="swestore_logs")
            logger.info('Run {} sent to swestore.'.format(f))
            if remove:
                logger.info('Removing run'.format(f))
                os.remove(f)
        else:
            logger.warn('Run {} is already in Swestore, not sending it again nor removing from the disk'.format(f))
Exemple #11
0
 def transfer(self, transfer_log=None):
     """ 
         Execute the transfer as set up by this instance and, if requested, 
         validate the transfer. 
         :param string transfer_log: path prefix to log files where stderr 
             and stdout streams will be directed if this option is specified
         :returns True on success, False if the validation failed
         :raises transfer.TransferError: if src_path or dest_path were not valid
         :raises transfer.RsyncError: if the rsync command did not exit successfully
     """
     self.validate_src_path()
     self.validate_dest_path()
     command = [self.CMD] + self.format_options() + [self.src_path,self.remote_path()]
     try:
         call_external_command(
             command,
             with_log_files=(transfer_log is not None),
             prefix=transfer_log)
     except subprocess.CalledProcessError as e:
         raise RsyncError(e)
     return (not self.validate) or self.validate_transfer()
Exemple #12
0
    def transfer(self, transfer_log=None):
        """Execute the transfer as set up by this instance and, if requested,
            validate the transfer.

            :param string transfer_log: path prefix to log files where stderr
                and stdout streams will be directed if this option is specified
            :returns True on success, False if the validation failed
            :raises transfer.TransferError: if src_path or dest_path were not valid
            :raises transfer.RsyncError: if the rsync command did not exit successfully
        """
        self.validate_src_path()
        self.validate_dest_path()
        command = [self.CMD] + self.format_options() + [
            self.src_path, self.remote_path()
        ]
        try:
            call_external_command(command,
                                  with_log_files=(transfer_log is not None),
                                  prefix=transfer_log)
        except subprocess.CalledProcessError as e:
            raise RsyncError(e)
        return (not self.validate) or self.validate_transfer()
Exemple #13
0
    def _send_to_swestore(f, dest, remove=True):
        """ Send file to swestore checking adler32 on destination and eventually
        removing the file from disk

        :param str f: File to remove
        :param str dest: Destination directory in Swestore
        :param bool remove: If True, remove original file from source
        """
        if not filesystem.is_in_swestore(f):
            logger.info("Sending {} to swestore".format(f))
            misc.call_external_command(
                'iput -R swestoreArchCacheResc -P {file} {dest}'.format(
                    file=f, dest=dest),
                with_log_files=True,
                prefix=f.replace('.tar.bz2', ''),
                log_dir="swestore_logs")
            logger.info('Run {} sent to swestore.'.format(f))
            if remove:
                logger.info('Removing run'.format(f))
                os.remove(f)
        else:
            logger.warn(
                'Run {} is already in Swestore, not sending it again nor removing from the disk'
                .format(f))
Exemple #14
0
    def transfer_run(self, t_file, analysis, mail_recipients=None):
        """ Transfer a run to the analysis server. Will add group R/W permissions to
            the run directory in the destination server so that the run can be processed
            by any user/account in that group (i.e a functional account...). 
            :param str t_file: File where to put the transfer information
            :param bool analysis: Trigger analysis on remote server
        """
        # TODO: check the run type and build the correct rsync command
        # The option -a implies -o and -g which is not the desired behaviour
        command_line = ['rsync', '-Lav', '--no-o', '--no-g']
        # Add R/W permissions to the group
        command_line.append('--chmod=g+rw')
        # This horrible thing here avoids data dup when we use multiple indexes in a lane/FC
        command_line.append("--exclude=Demultiplexing_*/*_*")
        command_line.append("--include=*/")
        for to_include in self.CONFIG['analysis_server']['sync']['include']:
            command_line.append("--include={}".format(to_include))
        command_line.extend(["--exclude=*", "--prune-empty-dirs"])
        r_user = self.CONFIG['analysis_server']['user']
        r_host = self.CONFIG['analysis_server']['host']
        r_dir = self.CONFIG['analysis_server']['sync']['data_archive']
        remote = "{}@{}:{}".format(r_user, r_host, r_dir)
        command_line.extend([self.run_dir, remote])

        # Create temp file indicating that the run is being transferred
        try:
            open(os.path.join(self.run_dir, 'transferring'), 'w').close()
        except IOError as e:
            logger.error("Cannot create a file in {}. "
                         "Check the run name, and the permissions.".format(
                             self.id))
            raise e
        started = ("Started transfer of run {} on {}".format(
            self.id, datetime.now()))
        logger.info(started)
        # In this particular case we want to capture the exception because we want
        # to delete the transfer file
        try:
            msge_text = "I am about to transfer with this command \n{}".format(
                command_line)
            logger.info(msge_text)
            misc.call_external_command(command_line,
                                       with_log_files=True,
                                       prefix="",
                                       log_dir=self.run_dir)
        except subprocess.CalledProcessError as exception:
            os.remove(os.path.join(self.run_dir, 'transferring'))
            #Send an email notifying that the transfer failed
            runname = self.id
            sbt = ("Rsync of run {} failed".format(runname))
            msg = """ Rsync of data for run {run} has failed!
                Raised the following exception:     {e}
            """.format(run=runname, e=exception)
            if mail_recipients:
                send_mail(sbt, msg, mail_recipients)

            raise exception

        logger.info('Adding run {} to {}'.format(self.id, t_file))
        with open(t_file, 'a') as tranfer_file:
            tsv_writer = csv.writer(tranfer_file, delimiter='\t')
            tsv_writer.writerow([self.id, str(datetime.now())])
        os.remove(os.path.join(self.run_dir, 'transferring'))

        #Send an email notifying that the transfer was successful
        runname = self.id
        sbt = ("Rsync of data for run {} to Irma has finished".format(runname))
        msg = """ Rsync of data for run {run} to Irma has finished!
                          
        The run is available at : https://genomics-status.scilifelab.se/flowcells/{run}
        """.format(run=runname)
        if mail_recipients:
            send_mail(sbt, msg, mail_recipients)

        if analysis:
            # This needs to pass the runtype (i.e., Xten or HiSeq) and start the correct pipeline
            self.trigger_analysis()
Exemple #15
0
def _archive_run((run, seconds, force, compress_only)):
    """ Archive a specific run to swestore

    :param str run: Run directory
    :param int seconds: Days/hours converted as seconds to check
    :param bool force: Force the archiving even if the run is not complete
    :param bool compress_only: Only compress the run without sending it to swestore
    """
    def _send_to_swestore(f, dest, remove=True):
        """ Send file to swestore checking adler32 on destination and eventually
        removing the file from disk

        :param str f: File to remove
        :param str dest: Destination directory in Swestore
        :param bool remove: If True, remove original file from source
        """
        if not filesystem.is_in_swestore(f):
            logger.info("Sending {} to swestore".format(f))
            misc.call_external_command(
                'iput -R swestoreArchCacheResc -P {file} {dest}'.format(
                    file=f, dest=dest),
                with_log_files=True,
                prefix=f.replace('.tar.bz2', ''),
                log_dir="swestore_logs")
            logger.info('Run {} sent to swestore.'.format(f))
            if remove:
                logger.info('Removing run'.format(f))
                os.remove(f)
        else:
            logger.warn(
                'Run {} is already in Swestore, not sending it again nor removing from the disk'
                .format(f))

    # Create state file to say that the run is being archived
    open("{}.archiving".format(run.split('.')[0]), 'w').close()
    if run.endswith('bz2'):
        if os.stat(run).st_mtime < time.time() - seconds:
            _send_to_swestore(
                run,
                CONFIG.get('storage').get('irods').get('irodsHome'))
        else:
            logger.info(
                "Run {} is not older than given time yet. Not archiving".
                format(run))
    else:
        rta_file = os.path.join(run, finished_run_indicator)
        if not os.path.exists(rta_file) and not force:
            logger.warn(
                ("Run {} doesn't seem to be completed and --force option was "
                 "not enabled, not archiving the run".format(run)))
        if force or (os.path.exists(rta_file)
                     and os.stat(rta_file).st_mtime < time.time() - seconds):
            logger.info("Compressing run {}".format(run))
            # Compress with pbzip2
            misc.call_external_command(
                'tar --use-compress-program=pbzip2 -cf {run}.tar.bz2 {run}'.
                format(run=run))
            logger.info(
                'Run {} successfully compressed! Removing from disk...'.format(
                    run))
            shutil.rmtree(run)
            if not compress_only:
                _send_to_swestore(
                    '{}.tar.bz2'.format(run),
                    CONFIG.get('storage').get('irods').get('irodsHome'))
        else:
            logger.info(
                "Run {} is not completed or is not older than given time yet. Not archiving"
                .format(run))
    os.remove("{}.archiving".format(run.split('.')[0]))
Exemple #16
0
 def test_call_external_command_fail(self):
     """Call external command should handle error."""
     command = 'ls -E'
     with self.assertRaises(subprocess.CalledProcessError):
         misc.call_external_command(command)