Ejemplo n.º 1
0
def update_dmfilestat_diskusage(resultpk):
    '''
    Task to update DMFileStat.diskspace for all associated with this resultpk
    This task is launched at the end of pipeline execution.
    NOTE: This can be a long-lived task
    '''
    logid = {'logid':"%s" % ('tasks')}
    try:
        result = Results.objects.get(pk=resultpk)
        search_dirs = [result.get_report_dir(), result.experiment.expDir]
        cached_file_list = dm_utils.get_walk_filelist(search_dirs, list_dir=result.get_report_dir(), save_list=True)
        for dmtype in FILESET_TYPES:
            dmfilestat = result.get_filestat(dmtype)
            dmfilestat_utils.update_diskspace(dmfilestat, cached=cached_file_list)
    except SoftTimeLimitExceeded:
        logger.warn("Time exceeded update_diskusage for (%d) %s" % (resultpk,result.resultsName), extra = logid)
    except:
        raise
    
    try:
        disk_total = 0
        for dmfilestat in [ result.get_filestat(dmtype) for dmtype in FILESET_TYPES]:
            if dmfilestat.dmfileset.type == dmactions_types.SIG:
                dmfilestat.result.experiment.diskusage = dmfilestat.diskspace if dmfilestat.diskspace != None else 0
                dmfilestat.result.experiment.save()
            else:
                partial = dmfilestat.diskspace
                disk_total += int(partial) if partial != None else 0
                result.diskusage = disk_total
                result.save()
        # See dmaction._update_related_objects() which also updates Exp & Results diskusage fields
    except:
        logger.error(traceback.format_exc(), extra = logid)
        raise    
Ejemplo n.º 2
0
def update_dmfilestats_diskspace(dmfilestat):
    ''' Task to update DMFileStat.diskspace '''
    search_dirs = [dmfilestat.result.get_report_dir(), dmfilestat.result.experiment.expDir]
    try:
        cached_file_list = dm_utils.get_walk_filelist(search_dirs, list_dir=dmfilestat.result.get_report_dir())
        dmfilestat_utils.update_diskspace(dmfilestat, cached=cached_file_list)
    except:
        logger.exception(traceback.format_exc(), extra = logid)
Ejemplo n.º 3
0
def update_dmfilestats_diskspace(dmfilestat):
    ''' Task to update DMFileStat.diskspace '''
    search_dirs = [
        dmfilestat.result.get_report_dir(), dmfilestat.result.experiment.expDir
    ]
    try:
        cached_file_list = dm_utils.get_walk_filelist(
            search_dirs, list_dir=dmfilestat.result.get_report_dir())
        dmfilestat_utils.update_diskspace(dmfilestat, cached=cached_file_list)
    except:
        logger.exception(traceback.format_exc(), extra=logid)
Ejemplo n.º 4
0
def backfill_dmfilestats_diskspace():
    ''' Backfill records with DMFileStat.diskspace = None, one at a time
        These could be older data sets or new ones where update_diskusage task failed
    '''
    dmfilestats = DMFileStat.objects.filter(diskspace=None, action_state='L', files_in_use='').order_by('-created')
    if dmfilestats.count() > 0:
        dmfilestat = dmfilestats[0]
        search_dirs = [dmfilestat.result.get_report_dir(), dmfilestat.result.experiment.expDir]
        try:
            cached_file_list = dm_utils.get_walk_filelist(search_dirs, list_dir=dmfilestat.result.get_report_dir(), save_list=True)
            dmfilestat_utils.update_diskspace(dmfilestat, cached=cached_file_list)
        except:
            logger.error(traceback.format_exc(), extra = logid)
            raise
Ejemplo n.º 5
0
def main(cl_args):
    '''Main function'''
    for dmtype in FILESET_TYPES:
        sys.stdout.write("%s\n" % dmtype)
        stats = models.DMFileStat.objects.filter(dmfileset__type=dmtype) \
            .filter(action_state__in=['L', 'S', 'N', 'A']) \
            .filter(diskspace=None) \
            .order_by('created')
        num_datasets = 0
        for index, item in enumerate([stat for stat in stats if stat.diskspace is None]):
            if index == 0:
                sys.stdout.write("Missing disk space:\n")
            num_datasets = index
            if cl_args.repair:
                sys.stdout.write("\t%s..." % (item.result.resultsName))
                try:
                    if DEBUG:
                        sys.stdout.write("\nWe would run update_diskspace() here, if not in debug mode")
                    else:
                        sys.stdout.write("%d\n" % update_diskspace(item))
                except:
                    print traceback.format_exc()
            else:
                sys.stdout.write("\t%s\n" % item.result.resultsName)
        if num_datasets == 0:
            sys.stdout.write("...No datasets found\n")
    sys.exit(0)
Ejemplo n.º 6
0
def main(cl_args):
    """Main function"""
    for dmtype in FILESET_TYPES:
        sys.stdout.write("%s\n" % dmtype)
        stats = (models.DMFileStat.objects.filter(
            dmfileset__type=dmtype).filter(
                action_state__in=["L", "S", "N", "A"]).filter(
                    diskspace=None).order_by("created"))
        num_datasets = 0
        for index, item in enumerate(
            [stat for stat in stats if stat.diskspace is None]):
            if index == 0:
                sys.stdout.write("Missing disk space:\n")
            num_datasets = index
            if cl_args.repair:
                sys.stdout.write("\t%s..." % (item.result.resultsName))
                try:
                    if DEBUG:
                        sys.stdout.write(
                            "\nWe would run update_diskspace() here, if not in debug mode"
                        )
                    else:
                        sys.stdout.write("%d\n" % update_diskspace(item))
                except Exception:
                    print(traceback.format_exc())
            else:
                sys.stdout.write("\t%s\n" % item.result.resultsName)
        if num_datasets == 0:
            sys.stdout.write("...No datasets found\n")
    sys.exit(0)
Ejemplo n.º 7
0
def update_dmfilestat_diskusage(resultpk):
    """
    Task to update DMFileStat.diskspace for all associated with this resultpk
    This task is launched at the end of pipeline execution.
    NOTE: This can be a long-lived task
    """
    logid = {"logid": "%s" % ("tasks")}
    try:
        result = Results.objects.get(pk=resultpk)
        search_dirs = [result.get_report_dir(), result.experiment.expDir]
        cached_file_list = dm_utils.get_walk_filelist(
            search_dirs, list_dir=result.get_report_dir(), save_list=True)
        for dmtype in FILESET_TYPES:
            dmfilestat = result.get_filestat(dmtype)
            dmfilestat_utils.update_diskspace(dmfilestat,
                                              cached=cached_file_list)
    except SoftTimeLimitExceeded:
        logger.warn(
            "Time exceeded update_diskusage for (%d) %s" %
            (resultpk, result.resultsName),
            extra=logid,
        )
    except:
        raise

    try:
        disk_total = 0
        for dmfilestat in [
                result.get_filestat(dmtype) for dmtype in FILESET_TYPES
        ]:
            if dmfilestat.dmfileset.type == dmactions_types.SIG:
                dmfilestat.result.experiment.diskusage = (
                    dmfilestat.diskspace
                    if dmfilestat.diskspace != None else 0)
                dmfilestat.result.experiment.save()
            else:
                partial = dmfilestat.diskspace
                disk_total += int(partial) if partial != None else 0
                result.diskusage = disk_total
                result.save()
        # See dmaction._update_diskspace_and_diskusage() which also updates Exp & Results diskusage fields
    except Exception:
        logger.error(traceback.format_exc(), extra=logid)
        raise
Ejemplo n.º 8
0
def backfill_dmfilestats_diskspace():
    ''' Backfill records with DMFileStat.diskspace = None, one at a time
        These could be older data sets or new ones where update_diskusage task failed
    '''
    dmfilestats = DMFileStat.objects.filter(
        diskspace=None, action_state='L', files_in_use='').order_by('-created')
    if dmfilestats.count() > 0:
        dmfilestat = dmfilestats[0]
        search_dirs = [
            dmfilestat.result.get_report_dir(),
            dmfilestat.result.experiment.expDir
        ]
        try:
            cached_file_list = dm_utils.get_walk_filelist(
                search_dirs,
                list_dir=dmfilestat.result.get_report_dir(),
                save_list=True)
            dmfilestat_utils.update_diskspace(dmfilestat,
                                              cached=cached_file_list)
        except:
            logger.error(traceback.format_exc(), extra=logid)
            raise
Ejemplo n.º 9
0
def _action_complete_update(user, user_comment, dmfilestat, action):
    logger.debug("Function: %s()" % sys._getframe().f_code.co_name, extra = logid)

    if action == ARCHIVE:
        action_state = 'AD'
        msg = "%0.1f MB %s archived to %s." % (dmfilestat.diskspace, dmfilestat.dmfileset.type, dmfilestat.archivepath)
    elif action == DELETE:
        action_state = 'DD'
        msg = "%0.1f MB %s deleted." % (dmfilestat.diskspace, dmfilestat.dmfileset.type)
    elif action == EXPORT:
        action_state = 'L'
        msg = "%0.1f MB %s exported to %s." % (dmfilestat.diskspace, dmfilestat.dmfileset.type, dmfilestat.archivepath)
        dmfilestat.archivepath = None
        dmfilestat.save()
    elif action == TEST:
        return
    
    # Update disk usage values in dmfilestat objects after a delete or archive action.  Should be zero or close.
    # update diskusage on Experiment or Results object if data was deleted/moved
    if action != EXPORT:
        
        diskspace = update_diskspace(dmfilestat)
        
        if dmfilestat.dmfileset.type == dmactions_types.SIG:
            dmfilestat.result.experiment.diskusage = diskspace if diskspace != None else 0
            dmfilestat.result.experiment.save()
        else:
            result = dmfilestat.result
            disk_total = 0
            mylist = [
                result.get_filestat(dmactions_types.BASE).diskspace,
                result.get_filestat(dmactions_types.OUT).diskspace,
                result.get_filestat(dmactions_types.INTR).diskspace
            ]
            for partial in mylist:
                disk_total += int(partial) if partial != None else 0
            result.diskusage = disk_total
            result.save()
        # See data/tasks.update_dmfilestat_diskusage() which also updates Exp & Results diskusage fields
        
    _update_related_objects(user, user_comment, dmfilestat, action, msg, action_state)
Ejemplo n.º 10
0
def _update_diskspace_and_diskusage(dmfilestat):
    logger.debug("Function: %s()" % sys._getframe().f_code.co_name, extra=logid)

    diskspace = update_diskspace(dmfilestat)

    # update diskusage on Experiment or Results object if data was deleted/moved
    # See data/tasks.update_dmfilestat_diskusage() which also updates Exp & Results diskusage fields
    if dmfilestat.dmfileset.type == dmactions_types.SIG:
        dmfilestat.result.experiment.diskusage = diskspace if diskspace != None else 0
        dmfilestat.result.experiment.save()
    else:
        result = dmfilestat.result
        disk_total = 0
        mylist = [
            result.get_filestat(dmactions_types.BASE).diskspace,
            result.get_filestat(dmactions_types.OUT).diskspace,
            result.get_filestat(dmactions_types.INTR).diskspace
        ]
        for partial in mylist:
            disk_total += int(partial) if partial != None else 0
        result.diskusage = disk_total
        result.save()
Ejemplo n.º 11
0
def destination_validation(dmfilestat, backup_directory=None, manual_action=False):
    '''
    Tests to validate destination directory:
    Does destination directory exist.
    Is there sufficient disk space.
    Write permissions.
    '''
    def _skipdiskspacecheck(directory):
        '''
        The hidden file .no_size_check should be placed into the root directory of the scratch drive mounted on the local
        system for the tape drive system.
        '''
        if os.path.exists(os.path.join(directory, ".no_size_check")):
            logger.info("%s: Exists: %s" %
                        (sys._getframe().f_code.co_name, os.path.join(directory, ".no_size_check")), extra=logid)
            return True
        else:
            logger.info("%s: Not Found: %s" %
                        (sys._getframe().f_code.co_name, os.path.join(directory, ".no_size_check")), extra=logid)
            return False

    logger.debug("Function: %s()" % sys._getframe().f_code.co_name, extra=logid)
    if backup_directory in [None, 'None', '']:
        backup_directory = dmfilestat.dmfileset.backup_directory

    # check for valid destination
    try:
        if backup_directory in [None, 'None', '', '/']:
            raise DMExceptions.MediaNotSet(
                "Backup media for %s is not configured. Please use Data Management Configuration page." % dmfilestat.dmfileset.type)

        if not os.path.isdir(backup_directory):
            raise DMExceptions.MediaNotAvailable(
                "Backup media for %s is not a directory: %s" % (dmfilestat.dmfileset.type, backup_directory))

        # check if destination is external filesystem
        # ie, catch the error of writing to a mountpoint which is unmounted.
        # Use the tool 'mountpoint' which returns 0 if its mountpoint and mounted
        # If its a subdirectory to a mountpoint, then the isdir() test will fail when not mounted.
        if not is_mounted(backup_directory):
            raise DMExceptions.MediaNotAvailable(
                "Backup media for %s is not mounted: %s" % (dmfilestat.dmfileset.type, backup_directory))

    except Exception as e:
        logger.error("%s" % e, extra=logid)
        raise

    # check for sufficient disk space (units: kilobytes)
    if _skipdiskspacecheck(backup_directory):
        logger.warn("%s - skipping destination disk space check" %
                    (sys._getframe().f_code.co_name), extra=logid)
        pass
    else:
        if dmfilestat.diskspace is None:
            diskspace = update_diskspace(dmfilestat)
        else:
            diskspace = dmfilestat.diskspace

        try:
            freespace = getSpaceMB(backup_directory)
            pending = 0
            if manual_action:
                # add up all objects that will be processed before this one
                exp_ids = []
                for obj in DMFileStat.objects.filter(action_state__in=['AG', 'EG', 'SA', 'SE']):
                    try:
                        if obj.archivepath and not os.path.normpath(obj.archivepath).startswith(os.path.normpath(backup_directory)):
                            continue
                        elif not os.path.normpath(obj.dmfileset.backup_directory).startswith(os.path.normpath(backup_directory)):
                            continue
                        if obj.dmfileset.type == dmactions_types.SIG:
                            if obj.result.experiment_id not in exp_ids:
                                exp_ids.append(obj.result.experiment_id)
                                pending += obj.diskspace
                        else:
                            pending += obj.diskspace
                    except:
                        pass

                logger.debug("Required %dMB, pending %dMB, free %dMB" %
                             (diskspace, pending, freespace), extra=logid)

            if diskspace >= freespace:
                raise DMExceptions.InsufficientDiskSpace(
                    "Not enough space to write files at %s (free=%dMB)" % (backup_directory, freespace))
            elif diskspace >= (freespace - pending):
                raise DMExceptions.InsufficientDiskSpace(
                    "Not enough space to write files at %s (free=%dMB, pending=%dMB)" % (backup_directory, freespace, pending))
        except Exception as e:
            logger.debug("%s" % str(e), extra=logid)
            raise

    # check for write permission.
    cmd = ['sudo', '/opt/ion/iondb/bin/sudo_utils.py', 'check_write_permission', backup_directory]
    try:
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        _, stderrdata = process.communicate()
    except Exception as err:
        logger.debug("Sub Process execution failed %s" % err, extra=logid)

    if process.returncode:
        raise DMExceptions.FilePermission(stderrdata)

    # check for existence of source directory - needed first to create destination folder
    # and, of course, as source of files to copy.  but we check here instead of failing halfway
    # thru the setup.  This shows inconsistency between dmfilestat action_status and filesystem.
    if dmfilestat.dmfileset.type == dmactions_types.SIG:
        src_dir = dmfilestat.result.experiment.expDir
    else:
        src_dir = dmfilestat.result.get_report_dir()
    if not os.path.exists(src_dir):
        raise DMExceptions.SrcDirDoesNotExist(src_dir)
Ejemplo n.º 12
0
def destination_validation(dmfilestat, backup_directory=None, manual_action=False):
    '''
    Tests to validate destination directory:
    Does destination directory exist.
    Is there sufficient disk space.
    Write permissions.
    '''
    def _skipdiskspacecheck(directory):
        '''
        The hidden file .no_size_check should be placed into the root directory of the scratch drive mounted on the local
        system for the tape drive system.
        '''
        if os.path.exists(os.path.join(directory, ".no_size_check")):
            logger.info("%s: Exists: %s" % (sys._getframe().f_code.co_name, os.path.join(directory, ".no_size_check")))
            return True
        else:
            logger.info("%s: Not Found: %s" % (sys._getframe().f_code.co_name, os.path.join(directory, ".no_size_check")))
            return False

    logger.debug("Function: %s()" % sys._getframe().f_code.co_name, extra = logid)
    if backup_directory in [None, 'None', '']:
        backup_directory = dmfilestat.dmfileset.backup_directory

    # check for valid destination
    try:
        if backup_directory in [None, 'None', '', '/']:
            raise DMExceptions.MediaNotSet("Backup media for %s is not configured. Please use Data Management Configuration page." % dmfilestat.dmfileset.type)

        if not os.path.isdir(backup_directory):
            raise DMExceptions.MediaNotAvailable("Backup media for %s is not a directory: %s" % (dmfilestat.dmfileset.type, backup_directory))

        # check if destination is external filesystem
        # ie, catch the error of writing to a mountpoint which is unmounted.
        # Use the tool 'mountpoint' which returns 0 if its mountpoint and mounted
        # If its a subdirectory to a mountpoint, then the isdir() test will fail when not mounted.
        if not is_mounted(backup_directory):
            raise DMExceptions.MediaNotAvailable("Backup media for %s is not mounted: %s" % (dmfilestat.dmfileset.type, backup_directory))

    except Exception as e:
        logger.error("%s" % e, extra = logid)
        raise

    # check for sufficient disk space (units: kilobytes)
    if _skipdiskspacecheck(backup_directory):
        logger.warn("%s - skipping destination disk space check" % (sys._getframe().f_code.co_name), extra = logid)
        pass
    else:
        if dmfilestat.diskspace is None:
            diskspace = update_diskspace(dmfilestat)
        else:
            diskspace = dmfilestat.diskspace

        try:
            freespace = getSpaceMB(backup_directory)
            pending = 0
            if manual_action:
                # add up all objects that will be processed before this one
                exp_ids = []
                for obj in DMFileStat.objects.filter(action_state__in=['AG', 'EG', 'SA', 'SE']):
                    try:
                        if obj.archivepath and not os.path.normpath(obj.archivepath).startswith(os.path.normpath(backup_directory)):
                            continue
                        elif not os.path.normpath(obj.dmfileset.backup_directory).startswith(os.path.normpath(backup_directory)):
                            continue
                        if obj.dmfileset.type == dmactions_types.SIG:
                            if obj.result.experiment_id not in exp_ids:
                                exp_ids.append(obj.result.experiment_id)
                                pending += obj.diskspace
                        else:
                            pending += obj.diskspace
                    except:
                        pass

                logger.debug("Required %dMB, pending %dMB, free %dMB" % (diskspace, pending, freespace), extra = logid)

            if diskspace >= freespace:
                raise DMExceptions.InsufficientDiskSpace("Not enough space to write files at %s (free=%dMB)" % (backup_directory, freespace))
            elif diskspace >= (freespace - pending):
                raise DMExceptions.InsufficientDiskSpace("Not enough space to write files at %s (free=%dMB, pending=%dMB)" % (backup_directory, freespace, pending))
        except Exception as e:
            logger.debug("%s" % str(e), extra = logid)
            raise

    # check for write permission.  NOTE: this function is called by apache2 user while the action function
    # will be executed within a celery task which takes the uid/gid of celeryd process - in our case that is currently root.
    # This test is too restrictive.
    try:
        foo = tempfile.NamedTemporaryFile(dir=backup_directory)
        foo.close()
    except Exception as e:
        if e.errno in [errno.EPERM, errno.EACCES]: # Operation not permitted
            errmsg = "Insufficient write permission in %s" % backup_directory
        else:
            errmsg = e
        logger.error(errmsg, extra = logid)
        raise DMExceptions.FilePermission(errmsg)

    # check for existence of source directory - needed first to create destination folder
    # and, of course, as source of files to copy.  but we check here instead of failing halfway
    # thru the setup.  This shows inconsistency between dmfilestat action_status and filesystem.
    if dmfilestat.dmfileset.type == dmactions_types.SIG:
        src_dir = dmfilestat.result.experiment.expDir
    else:
        src_dir = dmfilestat.result.get_report_dir()
    if not os.path.exists(src_dir):
        raise DMExceptions.SrcDirDoesNotExist(src_dir)