def getSize(path, unit='b', humanReadable=False, decimal=2):
def __convertSize(size, unit):
unit = unit.lower()
if unit in ['k', 'ko', 'kilo']:
size = size / 1024
elif unit in ['m', 'mo', 'mega']:
size = size / 1024 / 1024
elif unit in ['g', 'go', 'giga']:
size = size / 1024 / 1024 / 1024
else: # unit in ['b', 'bytes']
pass
return size
size = None
if isFile(path):
size = os.path.getsize(path)
size = __convertSize(size, unit)
elif isDir(path):
totalSize = 0
for current in sortedGlob(path + "/*"):
totalSize += getSize(current, unit='b')
size = __convertSize(totalSize, unit)
if unit in ['a', 'auto', None]:
tempSize = size
for u in ['k', 'm', 'g']:
tempSize = tempSize / 1024
if tempSize < 1024 and tempSize > 0:
size = tempSize
unit = u
break
if humanReadable:
return str(truncateFloat(size, decimal)) + unit
else:
return size
def remove(path, secure=True, minSlashCount=5, doRaise=True, skipDirs=False, skipFiles=False, decreaseMinSlashCountForTmp=True):
if path is None or len(path) == 0:
return
if isinstance(path, str):
path = [path]
for currentPath in path:
if secure and decreaseMinSlashCountForTmp and minSlashCount > 0:
minSlashCount -= minSlashCount
if secure and currentPath.count('/') < minSlashCount:
errorMsg = "Not enough slashes in " + currentPath
if doRaise:
raise Exception(errorMsg)
else:
print(errorMsg)
return
if isDir(currentPath) and not skipDirs:
try:
return shutil.rmtree(currentPath, True)
except Exception as e:
if doRaise:
raise e
else:
print(str(e))
if isFile(currentPath) and not skipFiles:
try:
os.remove(currentPath)
except OSError as e: # this would be "except OSError, e:" before Python 2.6
if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
raise # re-raise exception if a different error occurred
return
def fileToStrListYielder(path,
strip=True,
skipBlank=True,
commentStart="###",
logger=None,
verbose=False):
if path is not None and isFile(path):
commentCount = 0
with open(path) as f:
for line in f.readlines():
isComment = False
if strip:
line = line.strip()
if commentStart is not None and len(commentStart) > 0 and line.startswith(commentStart):
commentCount += 1
isComment = True
if not isComment:
if skipBlank and len(line) == 0:
pass
else:
yield line
if commentCount > 0:
basicLog("We found " + str(commentCount) + " comments in " + path, logger, verbose)
else:
basicLog(str(path) + " file not found.", logger, verbose)
def extract(filePath, destinationDir=None, upIfUnique=True, doDoubleExtract=True):
if not isFile(filePath):
print(filePath + " does not exist")
return None
# We get the dir of the file to extract:
(dirPath, _, _, filenameExt) = decomposePath(filePath)
# We extract it:
extractedDirPath = xtract.xtract(filePath)
# Here we check if the file end with ".tar":
if doDoubleExtract and extractedDirPath[-4:] == ".tar":
# So we re-extract it:
previousPath = extractedDirPath
extractedDirPath = xtract.xtract(extractedDirPath)
# We remove the previous element:
if isDir(previousPath):
remove(previousPath, minSlashCount=4)
elif isFile(previousPath):
remove(previousPath, minSlashCount=4)
# If there is only one folder or file under extractedDirPath, we up it:
if upIfUnique and len(sortedGlob(extractedDirPath + "/*")) == 1:
# We get the element path:
elementPath = sortedGlob(extractedDirPath + "/*")[0]
# We make the dst path:
dst = dirPath + "/" + elementPath.split("/")[-1]
# First we check if the element exists inthe parent dir:
if isFile(dst) or isDir(dst):
dst += time.strftime("-%Y.%m.%d-%H.%M.%S")
# then we move it:
shutil.move(elementPath, dst)
# And finally we remove the dir:
remove(extractedDirPath, minSlashCount=4)
# We update extractedDirPath:
extractedDirPath = dst
# We move the element:
if destinationDir is not None:
# We move it:
newDestFilePath = destinationDir + "/" + decomposePath(extractedDirPath)[3]
shutil.move(extractedDirPath, newDestFilePath)
# We update extractedDirPath:
extractedDirPath = newDestFilePath
# Finally we return the new path:
return extractedDirPath
def download(url, dirPath=None, skipIfExists=False):
"""
Based on https://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py/39217788
"""
if dirPath is None:
dirPath = tmpDir("downloads")
fileName = strToFilename(url.split('/')[-1])
filePath = dirPath + "/" + fileName
if skipIfExists and isFile(filePath):
return filePath
else:
r = requests.get(url, stream=True)
with open(filePath, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
return filePath