def getPlainFileName(datadir, item, dir_size_limit, zipdir = True): """ item is an entry in the dataset directory, which may be file or directory if a plain file, return the name if a directory, zip it up and return the zip file name if anything that can't be transferred, return None """ path = os.path.join(datadir, os.path.basename(item)) if (not os.path.exists(path)): print path return None if os.path.islink(path): return None if os.path.isfile(path): return item elif os.path.isdir(path) and zipdir: size = futils.getDirSize(path) # this is in bytes, limit is in MB if size < 0: return None if dir_size_limit and (size > dir_size_limit * 1048576): return None resp = futils.zipDir(path) if not resp: return None zip_file = resp.data resp = futils.deleteDir(path) return zip_file elif os.path.isdir(path) and zipdir == False: return item else: return None
def deleteFilesWhileVeryLowDisk(self, dconfig, deletions): """ Keep deleting files from a data_stream while disk state is very low. Return a True/False value for whether a better (i.e. not VLOW) disk state was reached. dconfig is the DatasetConfig object 'deletions' argument is an array provided by the caller; it will be appended to with pathnames deleted, so that the caller can log these """ for tu_path in self.getTUsForDeletion(dconfig): # test disk space before deleting if self.getDiskState() != DiskState.VLOW: return True deletions.append(tu_path) if os.path.isdir(tu_path): # recursive deletion - may take a while, so # move it inside a temporary dot-dir first (and then # delete from the level of the dot-dir itself) to # reduce chance of races with TransferUnitController # trying to transfer it parent_dir = os.path.dirname(tu_path) del_dir = tempfile.mkdtemp(dir = parent_dir, prefix = ".del_tmp_") os.rename(tu_path, del_dir) status = futils.deleteDir(del_dir) else: status = futils.deleteFile(tu_path) if not status: self.warn("could not delete %s: %s" % \ (tu_path, status)) # repeat the test one final time (after last deletion) # to determine return value return (self.getDiskState() != DiskState.VLOW)