def getPlainFileName(datadir, item, dir_size_limit, zipdir = True):
"""
item is an entry in the dataset directory, which may be file or directory
if a plain file, return the name
if a directory, zip it up and return the zip file name
if anything that can't be transferred, return None
"""
path = os.path.join(datadir, os.path.basename(item))
if (not os.path.exists(path)):
print path
return None
if os.path.islink(path):
return None
if os.path.isfile(path):
return item
elif os.path.isdir(path) and zipdir:
size = futils.getDirSize(path) # this is in bytes, limit is in MB
if size < 0:
return None
if dir_size_limit and (size > dir_size_limit * 1048576):
return None
resp = futils.zipDir(path)
if not resp:
return None
zip_file = resp.data
resp = futils.deleteDir(path)
return zip_file
elif os.path.isdir(path) and zipdir == False:
return item
else:
return None
def deleteFilesWhileVeryLowDisk(self, dconfig, deletions):
"""
Keep deleting files from a data_stream while disk state is
very low. Return a True/False value for whether a
better (i.e. not VLOW) disk state was reached.
dconfig is the DatasetConfig object
'deletions' argument is an array provided by the caller;
it will be appended to with pathnames deleted, so that
the caller can log these
"""
for tu_path in self.getTUsForDeletion(dconfig):
# test disk space before deleting
if self.getDiskState() != DiskState.VLOW:
return True
deletions.append(tu_path)
if os.path.isdir(tu_path):
# recursive deletion - may take a while, so
# move it inside a temporary dot-dir first (and then
# delete from the level of the dot-dir itself) to
# reduce chance of races with TransferUnitController
# trying to transfer it
parent_dir = os.path.dirname(tu_path)
del_dir = tempfile.mkdtemp(dir = parent_dir,
prefix = ".del_tmp_")
os.rename(tu_path, del_dir)
status = futils.deleteDir(del_dir)
else:
status = futils.deleteFile(tu_path)
if not status:
self.warn("could not delete %s: %s" % \
(tu_path, status))
# repeat the test one final time (after last deletion)
# to determine return value
return (self.getDiskState() != DiskState.VLOW)
