def testFileToMultiParts():
directory = getExecDir(__file__) + "/testdata"
filePath = sortedGlob(directory + "/*")[0]
workingDir = tmpDir("vectors-test")
result = extract(filePath, destinationDir=workingDir)
outputDir = fileToMultiParts(result, checkLineCount=True, compress=True)
print(outputDir)
def strToTmpFile(text, name=None, ext="", addRandomStr=False, *args, **kwargs):
if text is None:
text = ""
if ext is None:
ext = ""
if ext != "":
if not ext.startswith("."):
ext = "." + ext
if name is None:
name = getRandomStr()
elif addRandomStr:
name += "-" + getRandomStr()
path = tmpDir(*args, **kwargs) + "/" + name + ext
strToFile(text, path)
return path
def download(url, dirPath=None, skipIfExists=False):
"""
Based on https://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py/39217788
"""
if dirPath is None:
dirPath = tmpDir("downloads")
fileName = strToFilename(url.split('/')[-1])
filePath = dirPath + "/" + fileName
if skipIfExists and isFile(filePath):
return filePath
else:
r = requests.get(url, stream=True)
with open(filePath, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
return filePath
if logger is not None:
try:
logger.log(msg)
except: pass
else:
print(msg)
except Exception as e:
print(e)
if __name__ == '__main__':
# testRM()
# print(download("http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz"))
# extract("/home/hayj/tmp/downloads/aclImdb_v1.tar.gz")
# print(extract("/home/hayj/tmp/downloads/aclImdb_v1.tar.gz", tmpDir("aaa")))
# normalizeNumericalFilePaths("/home/hayj/test/test1/*.txt")
# normalizeNumericalFilePaths("/users/modhel/hayj/NoSave/Data/TwitterArchiveOrg/Converted/*.bz2")
# strToTmpFile("hoho", subDir="test", ext="txt")
# strToFile("haha", tmpDir(subDir="test") + "/test.txt")
# key = 'AAA'
# text = "bbb"
# print(encryptFile(homeDir() + '/tmp/titi.txt', key, text=text))
#
#
# text = decryptFile(homeDir() + '/tmp/titi.txt', key)
#
# print(text)
cleanDir(tmpDir(), startsWith=None, olderHour=4, verbose=True, dryRun=True)
# Calculate the nmber of digit:
digitCountHasToBe = len(str(maxInt))
# Replace all :
i = 0
for i in range(len(allNumbers)):
currentPath = allPaths[i]
(dir, filename, ext, filenameExt) = decomposePath(currentPath)
currentInt = allNumbers[i]
currentRegex = "0*" + str(currentInt)
zerosCountToAdd = digitCountHasToBe - len(str(currentInt))
zerosStr = "0" * zerosCountToAdd
newFilename = re.sub(currentRegex, zerosStr + str(currentInt), filename, count=1)
newFilename = dir + newFilename + "." + ext
if currentPath != newFilename:
os.rename(currentPath, newFilename)
print(newFilename + " done.")
i += 1
return True
if __name__ == '__main__':
# normalizeNumericalFilePaths("/home/hayj/test/test1/*.txt")
# normalizeNumericalFilePaths("/users/modhel/hayj/NoSave/Data/TwitterArchiveOrg/Converted/*.bz2")
strToTmpFile("hoho", subDir="test", ext="txt")
strToFile("haha", tmpDir(subDir="test") + "/test.txt")