def validate_destination(categories):
dest = {}
for category in categories:
if category["copy_files"]:
destination = category["dest_path"]
if destination:
if os.path.normpath(destination) == os.path.normpath(
category["src_path"]):
raise Exception(
"%s destination is the same as source path: %s" %
(category["dmtype"], destination))
if category["diskspace"]:
dest[destination] = (dest.setdefault(destination, 0) +
category["diskspace"])
else:
raise Exception("Missing %s copy files destination" %
category["dmtype"])
for destination, diskspace in list(dest.items()):
# create destination
try:
os.makedirs(destination)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# check diskspace
freespace = getSpaceMB(destination)
if diskspace >= freespace:
raise Exception(
"Not enough space to copy files at %s (required=%dMB, free=%dMB)"
% (destination, diskspace, freespace))
def validate_destination(categories):
dest = {}
for category in categories:
if category['copy_files']:
destination = category['dest_path']
if destination:
if category['diskspace']:
dest[destination] = dest.setdefault(
destination, 0) + category['diskspace']
else:
raise Exception("Missing %s copy files destination" %
category['dmtype'])
for destination, diskspace in dest.items():
# create destination
try:
os.makedirs(destination)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# check diskspace
freespace = getSpaceMB(destination)
if diskspace >= freespace:
raise Exception(
"Not enough space to copy files at %s (required=%dMB, free=%dMB)"
% (destination, diskspace, freespace))
def validate_destination(categories):
dest = {}
for category in categories:
if category['copy_files']:
destination = category['dest_path']
if destination:
if os.path.normpath(destination) == os.path.normpath(category['src_path']):
raise Exception("%s destination is the same as source path: %s" %
(category['dmtype'], destination))
if category['diskspace']:
dest[destination] = dest.setdefault(destination, 0) + category['diskspace']
else:
raise Exception("Missing %s copy files destination" % category['dmtype'])
for destination, diskspace in dest.items():
# create destination
try:
os.makedirs(destination)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# check diskspace
freespace = getSpaceMB(destination)
if diskspace >= freespace:
raise Exception("Not enough space to copy files at %s (required=%dMB, free=%dMB)" %
(destination, diskspace, freespace))
def destination_validation(dmfilestat, backup_directory=None, manual_action=False):
'''
Tests to validate destination directory:
Does destination directory exist.
Is there sufficient disk space.
Write permissions.
'''
def _skipdiskspacecheck(directory):
'''
The hidden file .no_size_check should be placed into the root directory of the scratch drive mounted on the local
system for the tape drive system.
'''
if os.path.exists(os.path.join(directory, ".no_size_check")):
logger.info("%s: Exists: %s" %
(sys._getframe().f_code.co_name, os.path.join(directory, ".no_size_check")), extra=logid)
return True
else:
logger.info("%s: Not Found: %s" %
(sys._getframe().f_code.co_name, os.path.join(directory, ".no_size_check")), extra=logid)
return False
logger.debug("Function: %s()" % sys._getframe().f_code.co_name, extra=logid)
if backup_directory in [None, 'None', '']:
backup_directory = dmfilestat.dmfileset.backup_directory
# check for valid destination
try:
if backup_directory in [None, 'None', '', '/']:
raise DMExceptions.MediaNotSet(
"Backup media for %s is not configured. Please use Data Management Configuration page." % dmfilestat.dmfileset.type)
if not os.path.isdir(backup_directory):
raise DMExceptions.MediaNotAvailable(
"Backup media for %s is not a directory: %s" % (dmfilestat.dmfileset.type, backup_directory))
# check if destination is external filesystem
# ie, catch the error of writing to a mountpoint which is unmounted.
# Use the tool 'mountpoint' which returns 0 if its mountpoint and mounted
# If its a subdirectory to a mountpoint, then the isdir() test will fail when not mounted.
if not is_mounted(backup_directory):
raise DMExceptions.MediaNotAvailable(
"Backup media for %s is not mounted: %s" % (dmfilestat.dmfileset.type, backup_directory))
except Exception as e:
logger.error("%s" % e, extra=logid)
raise
# check for sufficient disk space (units: kilobytes)
if _skipdiskspacecheck(backup_directory):
logger.warn("%s - skipping destination disk space check" %
(sys._getframe().f_code.co_name), extra=logid)
pass
else:
if dmfilestat.diskspace is None:
diskspace = update_diskspace(dmfilestat)
else:
diskspace = dmfilestat.diskspace
try:
freespace = getSpaceMB(backup_directory)
pending = 0
if manual_action:
# add up all objects that will be processed before this one
exp_ids = []
for obj in DMFileStat.objects.filter(action_state__in=['AG', 'EG', 'SA', 'SE']):
try:
if obj.archivepath and not os.path.normpath(obj.archivepath).startswith(os.path.normpath(backup_directory)):
continue
elif not os.path.normpath(obj.dmfileset.backup_directory).startswith(os.path.normpath(backup_directory)):
continue
if obj.dmfileset.type == dmactions_types.SIG:
if obj.result.experiment_id not in exp_ids:
exp_ids.append(obj.result.experiment_id)
pending += obj.diskspace
else:
pending += obj.diskspace
except:
pass
logger.debug("Required %dMB, pending %dMB, free %dMB" %
(diskspace, pending, freespace), extra=logid)
if diskspace >= freespace:
raise DMExceptions.InsufficientDiskSpace(
"Not enough space to write files at %s (free=%dMB)" % (backup_directory, freespace))
elif diskspace >= (freespace - pending):
raise DMExceptions.InsufficientDiskSpace(
"Not enough space to write files at %s (free=%dMB, pending=%dMB)" % (backup_directory, freespace, pending))
except Exception as e:
logger.debug("%s" % str(e), extra=logid)
raise
# check for write permission.
cmd = ['sudo', '/opt/ion/iondb/bin/sudo_utils.py', 'check_write_permission', backup_directory]
try:
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
_, stderrdata = process.communicate()
except Exception as err:
logger.debug("Sub Process execution failed %s" % err, extra=logid)
if process.returncode:
raise DMExceptions.FilePermission(stderrdata)
# check for existence of source directory - needed first to create destination folder
# and, of course, as source of files to copy. but we check here instead of failing halfway
# thru the setup. This shows inconsistency between dmfilestat action_status and filesystem.
if dmfilestat.dmfileset.type == dmactions_types.SIG:
src_dir = dmfilestat.result.experiment.expDir
else:
src_dir = dmfilestat.result.get_report_dir()
if not os.path.exists(src_dir):
raise DMExceptions.SrcDirDoesNotExist(src_dir)
def destination_validation(dmfilestat, backup_directory=None, manual_action=False):
'''
Tests to validate destination directory:
Does destination directory exist.
Is there sufficient disk space.
Write permissions.
'''
logger.debug("Function: %s()" % sys._getframe().f_code.co_name)
if backup_directory in [None, 'None', '']:
backup_directory = dmfilestat.dmfileset.backup_directory
# check for valid destination
try:
if backup_directory in [None, 'None', '']:
raise DMExceptions.MediaNotSet("Backup media for %s is not configured. Please use Data Management Configuration page." % dmfilestat.dmfileset.type)
if not os.path.isdir(backup_directory):
raise DMExceptions.MediaNotAvailable("Backup media for %s is not available: %s" % (dmfilestat.dmfileset.type, backup_directory))
except Exception as e:
logger.error("%s" % e)
raise
# check for sufficient disk space (units: kilobytes)
if dmfilestat.diskspace is None:
diskspace = update_diskspace(dmfilestat)
else:
diskspace = dmfilestat.diskspace
try:
freespace = getSpaceMB(backup_directory)
pending = 0
if manual_action:
# add up all objects that will be processed before this one
exp_ids = []
for obj in DMFileStat.objects.filter(action_state__in=['AG','EG','SA','SE']):
if obj.archivepath and not os.path.normpath(obj.archivepath).startswith(os.path.normpath(backup_directory)):
continue
elif not os.path.normpath(obj.dmfileset.backup_directory).startswith(os.path.normpath(backup_directory)):
continue
if obj.dmfileset.type == dmactions_types.SIG:
if obj.result.experiment_id not in exp_ids:
exp_ids.append(obj.result.experiment_id)
pending += obj.diskspace
else:
pending += obj.diskspace
logger.debug("Required %dMB, pending %dMB, free %dMB" % (diskspace, pending, freespace))
if diskspace >= freespace:
raise DMExceptions.InsufficientDiskSpace("Not enough space to write files at %s (free=%dMB)" % (backup_directory,freespace))
elif diskspace >= (freespace - pending):
raise DMExceptions.InsufficientDiskSpace("Not enough space to write files at %s (free=%dMB, pending=%dMB)" % (backup_directory,freespace, pending))
except Exception as e:
logger.debug("%s" % str(e))
raise
# check for write permission. NOTE: this function is called by apache2 user while the action function
# will be executed within a celery task which takes the uid/gid of celeryd process - in our case that is currently root.
# This test is too restrictive.
try:
tempfile.TemporaryFile(dir=backup_directory).close()
except Exception as e:
if e.errno == errno.EPERM or errno.EACCES: # Operation not permitted
errmsg = "Insufficient write permission in %s" % backup_directory
else:
errmsg = e
logger.error(errmsg)
raise DMExceptions.FilePermission(errmsg)
# check for existence of source directory - needed first to create destination folder
# and, of course, as source of files to copy. but we check here instead of failing halfway
# thru the setup. This shows inconsistency between dmfilestat action_status and filesystem.
if dmfilestat.dmfileset.type == dmactions_types.SIG:
src_dir = dmfilestat.result.experiment.expDir
if not os.path.exists(src_dir):
raise DMExceptions.SrcDirDoesNotExist(src_dir)
def destination_validation(dmfilestat,
backup_directory=None,
manual_action=False):
'''
Tests to validate destination directory:
Does destination directory exist.
Is there sufficient disk space.
Write permissions.
'''
logger.debug("Function: %s()" % sys._getframe().f_code.co_name)
if backup_directory in [None, 'None', '']:
backup_directory = dmfilestat.dmfileset.backup_directory
# check for valid destination
try:
if backup_directory in [None, 'None', '']:
raise DMExceptions.MediaNotSet(
"Backup media for %s is not configured. Please use Data Management Configuration page."
% dmfilestat.dmfileset.type)
if not os.path.isdir(backup_directory):
raise DMExceptions.MediaNotAvailable(
"Backup media for %s is not available: %s" %
(dmfilestat.dmfileset.type, backup_directory))
except Exception as e:
logger.error("%s" % e)
raise
# check for sufficient disk space (units: kilobytes)
if dmfilestat.diskspace is None:
diskspace = update_diskspace(dmfilestat)
else:
diskspace = dmfilestat.diskspace
try:
freespace = getSpaceMB(backup_directory)
pending = 0
if manual_action:
# add up all objects that will be processed before this one
exp_ids = []
for obj in DMFileStat.objects.filter(
action_state__in=['AG', 'EG', 'SA', 'SE']):
if obj.archivepath and not os.path.normpath(
obj.archivepath).startswith(
os.path.normpath(backup_directory)):
continue
elif not os.path.normpath(
obj.dmfileset.backup_directory).startswith(
os.path.normpath(backup_directory)):
continue
if obj.dmfileset.type == dmactions_types.SIG:
if obj.result.experiment_id not in exp_ids:
exp_ids.append(obj.result.experiment_id)
pending += obj.diskspace
else:
pending += obj.diskspace
logger.debug("Required %dMB, pending %dMB, free %dMB" %
(diskspace, pending, freespace))
if diskspace >= freespace:
raise DMExceptions.InsufficientDiskSpace(
"Not enough space to write files at %s (free=%dMB)" %
(backup_directory, freespace))
elif diskspace >= (freespace - pending):
raise DMExceptions.InsufficientDiskSpace(
"Not enough space to write files at %s (free=%dMB, pending=%dMB)"
% (backup_directory, freespace, pending))
except Exception as e:
logger.debug("%s" % str(e))
raise
# check for write permission. NOTE: this function is called by apache2 user while the action function
# will be executed within a celery task which takes the uid/gid of celeryd process - in our case that is currently root.
# This test is too restrictive.
try:
tempfile.TemporaryFile(dir=backup_directory).close()
except Exception as e:
if e.errno == errno.EPERM or errno.EACCES: # Operation not permitted
errmsg = "Insufficient write permission in %s" % backup_directory
else:
errmsg = e
logger.error(errmsg)
raise DMExceptions.FilePermission(errmsg)
# check for existence of source directory - needed first to create destination folder
# and, of course, as source of files to copy. but we check here instead of failing halfway
# thru the setup. This shows inconsistency between dmfilestat action_status and filesystem.
if dmfilestat.dmfileset.type == dmactions_types.SIG:
src_dir = dmfilestat.result.experiment.expDir
else:
src_dir = dmfilestat.result.get_report_dir()
if not os.path.exists(src_dir):
raise DMExceptions.SrcDirDoesNotExist(src_dir)
def destination_validation(dmfilestat, backup_directory=None, manual_action=False):
'''
Tests to validate destination directory:
Does destination directory exist.
Is there sufficient disk space.
Write permissions.
'''
def _skipdiskspacecheck(directory):
'''
The hidden file .no_size_check should be placed into the root directory of the scratch drive mounted on the local
system for the tape drive system.
'''
if os.path.exists(os.path.join(directory, ".no_size_check")):
logger.info("%s: Exists: %s" % (sys._getframe().f_code.co_name, os.path.join(directory, ".no_size_check")))
return True
else:
logger.info("%s: Not Found: %s" % (sys._getframe().f_code.co_name, os.path.join(directory, ".no_size_check")))
return False
logger.debug("Function: %s()" % sys._getframe().f_code.co_name, extra = logid)
if backup_directory in [None, 'None', '']:
backup_directory = dmfilestat.dmfileset.backup_directory
# check for valid destination
try:
if backup_directory in [None, 'None', '', '/']:
raise DMExceptions.MediaNotSet("Backup media for %s is not configured. Please use Data Management Configuration page." % dmfilestat.dmfileset.type)
if not os.path.isdir(backup_directory):
raise DMExceptions.MediaNotAvailable("Backup media for %s is not a directory: %s" % (dmfilestat.dmfileset.type, backup_directory))
# check if destination is external filesystem
# ie, catch the error of writing to a mountpoint which is unmounted.
# Use the tool 'mountpoint' which returns 0 if its mountpoint and mounted
# If its a subdirectory to a mountpoint, then the isdir() test will fail when not mounted.
if not is_mounted(backup_directory):
raise DMExceptions.MediaNotAvailable("Backup media for %s is not mounted: %s" % (dmfilestat.dmfileset.type, backup_directory))
except Exception as e:
logger.error("%s" % e, extra = logid)
raise
# check for sufficient disk space (units: kilobytes)
if _skipdiskspacecheck(backup_directory):
logger.warn("%s - skipping destination disk space check" % (sys._getframe().f_code.co_name), extra = logid)
pass
else:
if dmfilestat.diskspace is None:
diskspace = update_diskspace(dmfilestat)
else:
diskspace = dmfilestat.diskspace
try:
freespace = getSpaceMB(backup_directory)
pending = 0
if manual_action:
# add up all objects that will be processed before this one
exp_ids = []
for obj in DMFileStat.objects.filter(action_state__in=['AG', 'EG', 'SA', 'SE']):
try:
if obj.archivepath and not os.path.normpath(obj.archivepath).startswith(os.path.normpath(backup_directory)):
continue
elif not os.path.normpath(obj.dmfileset.backup_directory).startswith(os.path.normpath(backup_directory)):
continue
if obj.dmfileset.type == dmactions_types.SIG:
if obj.result.experiment_id not in exp_ids:
exp_ids.append(obj.result.experiment_id)
pending += obj.diskspace
else:
pending += obj.diskspace
except:
pass
logger.debug("Required %dMB, pending %dMB, free %dMB" % (diskspace, pending, freespace), extra = logid)
if diskspace >= freespace:
raise DMExceptions.InsufficientDiskSpace("Not enough space to write files at %s (free=%dMB)" % (backup_directory, freespace))
elif diskspace >= (freespace - pending):
raise DMExceptions.InsufficientDiskSpace("Not enough space to write files at %s (free=%dMB, pending=%dMB)" % (backup_directory, freespace, pending))
except Exception as e:
logger.debug("%s" % str(e), extra = logid)
raise
# check for write permission. NOTE: this function is called by apache2 user while the action function
# will be executed within a celery task which takes the uid/gid of celeryd process - in our case that is currently root.
# This test is too restrictive.
try:
foo = tempfile.NamedTemporaryFile(dir=backup_directory)
foo.close()
except Exception as e:
if e.errno in [errno.EPERM, errno.EACCES]: # Operation not permitted
errmsg = "Insufficient write permission in %s" % backup_directory
else:
errmsg = e
logger.error(errmsg, extra = logid)
raise DMExceptions.FilePermission(errmsg)
# check for existence of source directory - needed first to create destination folder
# and, of course, as source of files to copy. but we check here instead of failing halfway
# thru the setup. This shows inconsistency between dmfilestat action_status and filesystem.
if dmfilestat.dmfileset.type == dmactions_types.SIG:
src_dir = dmfilestat.result.experiment.expDir
else:
src_dir = dmfilestat.result.get_report_dir()
if not os.path.exists(src_dir):
raise DMExceptions.SrcDirDoesNotExist(src_dir)
