def getDataDirFullPath(): result = config.dataDir if not os.path.isabs(config.dataDir): result = os.path.dirname(config.moduleFile) + os.path.sep + config.dataDir result = os.path.normpath(result) result = pathWithLastSlash(result) return result
def downloadURLtoFile(self, url, fileIndex, filesCount): # Downloads to temporary file, if suceeded, then rename result tmpFileName = pathWithLastSlash(self.targetDir) + "tmpfile.bin" logger.debug("RUIANDownloader.downloadURLtoFile") file_name = self.targetDir + url.split('/')[-1] startTime = datetime.datetime.now() if os.path.exists(file_name): logger.info("File " + extractFileName(file_name) + " is already downloaded, skipping it.") fileSize = os.stat(file_name).st_size else: req = urllib2.urlopen(url) meta = req.info() fileSize = int(meta.getheaders("Content-Length")[0]) logger.info("Downloading file %s [%d/%d %d Bytes]" % (extractFileName(file_name), fileIndex, filesCount, fileSize)) fileDownloadInfo(file_name, fileSize) CHUNK = 1024*1024 file_size_dl = 0 with open(tmpFileName, 'wb') as fp: while True: chunk = req.read(CHUNK) if not chunk: break fp.write(chunk) file_size_dl += len(chunk) logger.info(r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100.0 / fileSize)) self.downloadInfo.compressedFileSize = file_size_dl #self.buildIndexHTML() fp.close() os.rename(tmpFileName, file_name) self.downloadInfo.downloadTime = formatTimeDelta(str(datetime.datetime.now() - startTime)[5:]) self.downloadInfo.fileName = file_name self.downloadInfo.compressedFileSize = fileSize return file_name
def processDownloadedDirectory(path): logger.info("Načítám stažené soubory do databáze...") logger.info("--------------------------------------") logger.info("Zdrojová data : " + path) path = pathWithLastSlash(path) stateFileList = "" updatesFileList = [] for file in os.listdir(path): fileName = file.lower() if file.endswith(".txt"): if fileName.startswith("download_"): stateFileList = join(path, fileName) elif fileName.startswith("patch_"): updatesFileList.append(join(path, fileName)) result = False if stateFileList != "": createStateDatabase(path, stateFileList) result = True else: logger.info("Stavová data nejsou obsahem zdrojových dat.") if len(updatesFileList) == 0: logger.info("Denní aktualizace nejsou obsahem zdrojových dat.") else: result = True for updateFileName in updatesFileList: updateDatabase(updateFileName) logger.info(u"Generuji sestavu importů.") buildhtmllog.buildHTMLLog() logger.info("Načítání stažených souborů do databáze - hotovo.") return result
def removeDataFiles(): dataPath = pathWithLastSlash(os.path.split(updateFileList)[0]) inFile = open(updateFileList, "r") try: for line in inFile: fileName = os.path.basename(line) if os.path.exists(dataPath + fileName): os.remove(dataPath + fileName) finally: inFile.close() pass
def deleteFilesInLists(path, fileLists, extension): path = pathWithLastSlash(path) for fileList in fileLists: listFile = open(fileList, "r") i = 0 for line in listFile: i += 1 fileName = path + line.rstrip() + extension if os.path.exists(fileName): os.remove(fileName) logger.debug(str(i), ":", fileName) listFile.close() os.remove(fileList) pass
def getFullPath(configFileName, path): if not os.path.exists(path): path = pathWithLastSlash(configFileName) + path return path