def writeZipFile(archiveType, fileNameRoot, shouldMakeSnapshot, wikiAddress): 'Write zip file.' print('Copying:') print(wikiAddress) print('') almoner.makeDirectory(fileNameRoot) previousLetter = '0' lastModifiedText = datetime.datetime.today().strftime(globalDateTimeFormat) titles = getRecentTitles(archiveType, fileNameRoot, wikiAddress) print('Number of titles: %s' % len(titles)) almoner.writeFileText(os.path.join(fileNameRoot, 'last_modified.txt'), lastModifiedText) for title in titles: letter = title[0] if letter != previousLetter: previousLetter = letter print('Copying articles starting with %s.' % letter.upper()) sourceText = devtome.getSourceText(wikiAddress + '/doku.php?id=%s&do=edit' % title) time.sleep(2) fileName = os.path.join(fileNameRoot, title) almoner.writeFileText(fileName, sourceText) print('There were %s files in the wiki.\n' % len(titles)) archiveFileName = fileNameRoot + '.' + archiveType if archiveType == 'zip': almoner.writeZipFileByFolder(fileNameRoot) else: writeTarFileByFolder(archiveType, archiveFileName, fileNameRoot) if shouldMakeSnapshot: snapshotSuffix = datetime.datetime.today().strftime('_%y-%m-%d_%H') destination = fileNameRoot + snapshotSuffix + '.' + archiveType shutil.copyfile(archiveFileName, destination) print('The snapshot zip file has been written to:\n%s\n' % destination)
def getAuthors(backupFolder, categoryDictionary, lines, ratingDictionary, titles, viewDictionary): 'Get the authors.' averageRating = 0.0 authors = [] authorSet = set([]) ratingValues = ratingDictionary.values() almoner.makeDirectory(backupFolder) zipFileName = backupFolder + '.zip' backupFileSet = set([]) if os.path.isfile(zipFileName): zipArchive = zipfile.ZipFile(zipFileName, 'r', compression=zipfile.ZIP_DEFLATED) backupFileSet = set(zipArchive.namelist()) zipArchive.close() for ratingValue in ratingValues: averageRating += ratingValue if averageRating == 0.0: averageRating = 1.0 else: averageRating /= float(len(ratingDictionary.values())) for line in lines[1:]: words = line.split(',') if len(words) > 0: if len(words[0]) > 0: author = Author(averageRating, backupFolder, backupFileSet, categoryDictionary, ratingDictionary, titles, viewDictionary, words) if author.name not in authorSet: authorSet.add(author.name) authors.append(author) if len(authors) > 100: almoner.writeZipFileByFolder(backupFolder) return authors
def getAuthors(backupFolder, categoryDictionary, lines, ratingDictionary, titles, viewDictionary): 'Get the authors.' averageRating = 0.0 authors = [] authorSet = set([]) ratingValues = ratingDictionary.values() almoner.makeDirectory(backupFolder) zipFileName = backupFolder + '.zip' backupFileSet = set([]) if os.path.isfile(zipFileName): zipArchive = zipfile.ZipFile(zipFileName, 'r', compression=zipfile.ZIP_DEFLATED) backupFileSet = set(zipArchive.namelist()) zipArchive.close() for ratingValue in ratingValues: averageRating += ratingValue if averageRating == 0.0: averageRating = 1.0 else: averageRating /= float(len(ratingDictionary.values())) for line in lines[1 :]: words = line.split(',') if len(words) > 0: if len(words[0]) > 0: author = Author(averageRating, backupFolder, backupFileSet, categoryDictionary, ratingDictionary, titles, viewDictionary, words) if author.name not in authorSet: authorSet.add(author.name) authors.append(author) if len(authors) > 100: almoner.writeZipFileByFolder(backupFolder) return authors
def writeCategoryFiles(categoryDictionary, rootFileName): 'Write category files to a folder.' categoryFolder = rootFileName + '_categories' almoner.makeDirectory(categoryFolder) categoryKeys = categoryDictionary.keys() for categoryKey in categoryDictionary.keys(): writeCategoryFile(categoryDictionary, categoryFolder, categoryKey, rootFileName) almoner.writeZipFileByFolder(categoryFolder)
def getAuthors(backupFolder, lines, titles): 'Get the authors.' authors = [] almoner.makeDirectory(backupFolder) for line in lines[1:]: words = line.split(',') if len(words) > 0: if len(words[0]) > 0: authors.append(Author(backupFolder, titles, words)) almoner.writeZipFileByFolder(backupFolder) return authors
def getAuthors(backupFolder, lines, titles): 'Get the authors.' authors = [] almoner.makeDirectory(backupFolder) for line in lines[1 :]: words = line.split(',') if len(words) > 0: if len(words[0]) > 0: authors.append(Author(backupFolder, titles, words)) almoner.writeZipFileByFolder(backupFolder) return authors
def writeFolder(folderName, peerRootAddress, round): "Write zip file." print("Making directory:") print(folderName) print("") almoner.makeDirectory(folderName) peerAddress = almoner.getSuffixedFileName(peerRootAddress, str(round)) print("Getting peer:") print(peerAddress) print("") peerText = almoner.getLocationText(peerAddress) peerTextUntilCoins = peerText[: peerText.find("_begincoins")] unthinkingbitString = "https://raw.github.com/Unthinkingbit/charity/master/receiver.csv" fuzzyString = "http://devcoinpool.btc-music.com/receiver/receiver.csv" peerTextUntilCoins = peerTextUntilCoins.replace(unthinkingbitString, fuzzyString) print("Peer text until coins:") print(peerTextUntilCoins) print("") print("Writing widened files:") for widenRound in xrange(round + 1): writeWidenedFile(folderName, peerRootAddress, peerTextUntilCoins, widenRound) print("")
def getAuthors(backupFolder, lines, titles, viewDictionary): 'Get the authors.' authors = [] authorSet = set([]) almoner.makeDirectory(backupFolder) for line in lines[1 :]: words = line.split(',') if len(words) > 0: if len(words[0]) > 0: author = Author(backupFolder, titles, viewDictionary, words) if author.name not in authorSet: authorSet.add(author.name) authors.append(author) almoner.writeZipFileByFolder(backupFolder) for author in authors: if len(author.warnings) > 0: print('BIG WARNING') print(author.name) for warning in author.warnings: print(warning) print('') return authors
def writeFolder(folderName, peerRootAddress, round): 'Write zip file.' print('Making directory:') print(folderName) print('') almoner.makeDirectory(folderName) peerAddress = almoner.getSuffixedFileName(peerRootAddress, str(round)) print('Getting peer:') print(peerAddress) print('') peerText = almoner.getLocationText(peerAddress) peerTextUntilCoins = peerText[:peerText.find('_begincoins')] unthinkingbitString = 'https://raw.github.com/Unthinkingbit/charity/master/receiver.csv' fuzzyString = 'http://devcoinpool.btc-music.com/receiver/receiver.csv' peerTextUntilCoins = peerTextUntilCoins.replace(unthinkingbitString, fuzzyString) print('Peer text until coins:') print(peerTextUntilCoins) print('') print('Writing widened files:') for widenRound in xrange(round + 1): writeWidenedFile(folderName, peerRootAddress, peerTextUntilCoins, widenRound) print('')