def getDataDirFullPath():
result = config.dataDir
if not os.path.isabs(config.dataDir):
result = os.path.dirname(config.moduleFile) + os.path.sep + config.dataDir
result = os.path.normpath(result)
result = pathWithLastSlash(result)
return result
def downloadURLtoFile(self, url, fileIndex, filesCount):
# Downloads to temporary file, if suceeded, then rename result
tmpFileName = pathWithLastSlash(self.targetDir) + "tmpfile.bin"
logger.debug("RUIANDownloader.downloadURLtoFile")
file_name = self.targetDir + url.split('/')[-1]
startTime = datetime.datetime.now()
if os.path.exists(file_name):
logger.info("File " + extractFileName(file_name) + " is already downloaded, skipping it.")
fileSize = os.stat(file_name).st_size
else:
req = urllib2.urlopen(url)
meta = req.info()
fileSize = int(meta.getheaders("Content-Length")[0])
logger.info("Downloading file %s [%d/%d %d Bytes]" % (extractFileName(file_name), fileIndex, filesCount, fileSize))
fileDownloadInfo(file_name, fileSize)
CHUNK = 1024*1024
file_size_dl = 0
with open(tmpFileName, 'wb') as fp:
while True:
chunk = req.read(CHUNK)
if not chunk:
break
fp.write(chunk)
file_size_dl += len(chunk)
logger.info(r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100.0 / fileSize))
self.downloadInfo.compressedFileSize = file_size_dl
#self.buildIndexHTML()
fp.close()
os.rename(tmpFileName, file_name)
self.downloadInfo.downloadTime = formatTimeDelta(str(datetime.datetime.now() - startTime)[5:])
self.downloadInfo.fileName = file_name
self.downloadInfo.compressedFileSize = fileSize
return file_name
def processDownloadedDirectory(path):
logger.info("Načítám stažené soubory do databáze...")
logger.info("--------------------------------------")
logger.info("Zdrojová data : " + path)
path = pathWithLastSlash(path)
stateFileList = ""
updatesFileList = []
for file in os.listdir(path):
fileName = file.lower()
if file.endswith(".txt"):
if fileName.startswith("download_"):
stateFileList = join(path, fileName)
elif fileName.startswith("patch_"):
updatesFileList.append(join(path, fileName))
result = False
if stateFileList != "":
createStateDatabase(path, stateFileList)
result = True
else:
logger.info("Stavová data nejsou obsahem zdrojových dat.")
if len(updatesFileList) == 0:
logger.info("Denní aktualizace nejsou obsahem zdrojových dat.")
else:
result = True
for updateFileName in updatesFileList:
updateDatabase(updateFileName)
logger.info(u"Generuji sestavu importů.")
buildhtmllog.buildHTMLLog()
logger.info("Načítání stažených souborů do databáze - hotovo.")
return result
def removeDataFiles():
dataPath = pathWithLastSlash(os.path.split(updateFileList)[0])
inFile = open(updateFileList, "r")
try:
for line in inFile:
fileName = os.path.basename(line)
if os.path.exists(dataPath + fileName):
os.remove(dataPath + fileName)
finally:
inFile.close()
pass
def deleteFilesInLists(path, fileLists, extension):
path = pathWithLastSlash(path)
for fileList in fileLists:
listFile = open(fileList, "r")
i = 0
for line in listFile:
i += 1
fileName = path + line.rstrip() + extension
if os.path.exists(fileName):
os.remove(fileName)
logger.debug(str(i), ":", fileName)
listFile.close()
os.remove(fileList)
pass
def getFullPath(configFileName, path):
if not os.path.exists(path):
path = pathWithLastSlash(configFileName) + path
return path
