def writeOutput(arguments): 'Write output.' if '-h' in arguments or '-help' in arguments: print(__doc__) return suffixNumberString = almoner.getParameter(arguments, '24', 'round') suffixNumber = int(suffixNumberString) outputAccountTo = almoner.getSuffixedFileName(almoner.getParameter(arguments, 'account.csv', 'account'), suffixNumberString) accountLines = getAccountLines(arguments, suffixNumberString) peerLines = getPeerLines(arguments) peerText = '_beginpeers\n%s_endpeers\n' % almoner.getTextByLines(peerLines) accountText = getPluribusunumText(peerText, accountLines) if almoner.sendOutputTo(outputAccountTo, accountText): print('The account file has been written to:\n%s\n' % outputAccountTo) outputReceiverTo = almoner.getSuffixedFileName(almoner.getParameter(arguments, 'receiver.csv', 'receiver'), suffixNumberString) outputSummaryTo = almoner.getParameter(arguments, 'receiver_summary.txt', 'summary') denominatorSequences = getDenominatorSequencesByAccountLines(accountLines) originalReceiverLines = getReceiverLinesByDenominatorSequences(denominatorSequences) receiverLines = getPackedReceiverLines(denominatorSequences, originalReceiverLines, suffixNumber) receiverText = getPluribusunumText(peerText, receiverLines) if almoner.sendOutputTo(outputReceiverTo, receiverText): print('The receiver file has been written to:\n%s\n' % outputReceiverTo) shaOutputPrefix = almoner.getParameter(arguments, '', 'sha') if len(shaOutputPrefix) != 0: sha256FileName = almoner.getSuffixedFileName(outputReceiverTo, shaOutputPrefix) almoner.writeFileText(sha256FileName, hashlib.sha256(receiverText).hexdigest()) print('The sha256 receiver file has been written to:\n%s\n' % sha256FileName) if almoner.sendOutputTo(outputSummaryTo, getSummaryText(accountLines, originalReceiverLines, peerLines, suffixNumber)): print('The summary file has been written to:\n%s\n' % outputSummaryTo)
def writeZipFile(archiveType, fileNameRoot, shouldMakeSnapshot, wikiAddress): 'Write zip file.' print('Copying:') print(wikiAddress) print('') almoner.makeDirectory(fileNameRoot) previousLetter = '0' lastModifiedText = datetime.datetime.today().strftime(globalDateTimeFormat) titles = getRecentTitles(archiveType, fileNameRoot, wikiAddress) print('Number of titles: %s' % len(titles)) almoner.writeFileText(os.path.join(fileNameRoot, 'last_modified.txt'), lastModifiedText) for title in titles: letter = title[0] if letter != previousLetter: previousLetter = letter print('Copying articles starting with %s.' % letter.upper()) sourceText = devtome.getSourceText(wikiAddress + '/doku.php?id=%s&do=edit' % title) time.sleep(2) fileName = os.path.join(fileNameRoot, title) almoner.writeFileText(fileName, sourceText) print('There were %s files in the wiki.\n' % len(titles)) archiveFileName = fileNameRoot + '.' + archiveType if archiveType == 'zip': almoner.writeZipFileByFolder(fileNameRoot) else: writeTarFileByFolder(archiveType, archiveFileName, fileNameRoot) if shouldMakeSnapshot: snapshotSuffix = datetime.datetime.today().strftime('_%y-%m-%d_%H') destination = fileNameRoot + snapshotSuffix + '.' + archiveType shutil.copyfile(archiveFileName, destination) print('The snapshot zip file has been written to:\n%s\n' % destination)
def saveArticle(self, categoryDictionary, linkName, linkText, underscoredLinkName): 'Save article and if new add to new articles list.' if underscoredLinkName not in self.backupFileSet: self.newArticles.append(underscoredLinkName) almoner.writeFileText( os.path.join(self.backupFolder, underscoredLinkName), linkText) self.tomecount.articleCount += 1 linkTextLower = linkText.lower() categoryIndex = linkTextLower.find('[[category:') if categoryIndex != -1: categoryPrefixLength = len('[[category:') self.tomecount.categorizedArticleCount += 1 while categoryIndex != -1: startIndex = categoryIndex + categoryPrefixLength endBracketIndex = linkTextLower.find(']]', startIndex) if endBracketIndex == -1: return categoryName = linkTextLower[startIndex:endBracketIndex].strip( ).replace('_', ' ') if categoryName in categoryDictionary: categoryDictionary[categoryName].append(linkName) else: categoryDictionary[categoryName] = [linkName] categoryIndex = linkTextLower.find('[[category:', endBracketIndex)
def writeOutput(arguments): 'Write output.' if '-h' in arguments or '-help' in arguments: print(__doc__) return round = int(almoner.getParameter(arguments, '23', 'round')) rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki') currentFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % round, 'current') previousFileName = almoner.getParameter( arguments, rootFileName + '_%s.csv' % (round - 1), 'previous') lines = almoner.getTextLines(almoner.getFileText(previousFileName)) titleLine = lines[0] titles = titleLine.split(',') backupFolder = rootFileName + '_articles' authors = getAuthors(backupFolder, lines, titles) totalTomecount = getTotalTomecount(authors) tomecountText = getTomecountText(authors, totalTomecount) earningsText = getEarningsText(authors) outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt', 'summary') almoner.writeFileText(currentFileName, tomecountText) outputEarningsTo = almoner.getParameter(arguments, 'devtome_earnings_%s.csv' % round, 'earnings') if almoner.sendOutputTo(outputEarningsTo, earningsText): print('The devtome earnings file has been written to:\n%s\n' % outputEarningsTo) if almoner.sendOutputTo( outputSummaryTo, getSummaryText(earningsText, round, totalTomecount)): print('The summary file has been written to:\n%s\n' % outputSummaryTo)
def writeOutput(arguments): 'Write output.' if '-h' in arguments or '-help' in arguments: print(__doc__) return round = int(almoner.getParameter(arguments, '23', 'round')) rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki') currentFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % round, 'current') previousFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % (round - 1), 'previous') viewFileName = almoner.getParameter(arguments, 'devtome_analytics_%s.csv' % round, 'view') lines = almoner.getTextLines(almoner.getFileText(previousFileName)) titleLine = lines[0] titles = titleLine.split(',') backupFolder = rootFileName + '_articles' viewDictionary = getViewDictionary(viewFileName) authors = getAuthors(backupFolder, lines, titles, viewDictionary) totalTomecount = getTotalTomecount(authors) tomecountText = getTomecountText(authors, totalTomecount) earningsText = getEarningsText(authors) outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt', 'summary') almoner.writeFileText(currentFileName, tomecountText) outputEarningsTo = almoner.getParameter(arguments, 'devtome_earnings_%s.csv' % round, 'earnings') if almoner.sendOutputTo(outputEarningsTo, earningsText): print('The devtome earnings file has been written to:\n%s\n' % outputEarningsTo) if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, round, totalTomecount)): print('The summary file has been written to:\n%s\n' % outputSummaryTo)
def writeWidenedFile(folderName, peerRootAddress, peerTextUntilCoins, widenRound): "Write widened file." title = almoner.getSuffixedFileName("receiver.csv", str(widenRound)) widenPath = os.path.join(folderName, title) receiverAddress = almoner.getSuffixedFileName(peerRootAddress, str(widenRound)) receiverText = almoner.getLocationText(receiverAddress) receiverTextFromCoins = receiverText[receiverText.find("_begincoins") :] widenText = peerTextUntilCoins + receiverTextFromCoins almoner.writeFileText(widenPath, widenText)
def writeWriterFile(fileName, previousDevtomeName, wikiAddress): 'Write writer file.' nowDatetime = datetime.datetime.today() cString = cStringIO.StringIO() cString.write('Date,%s' % nowDatetime.strftime(globalDateTimeFormat)) names = getRecentNames(fileName, nowDatetime, previousDevtomeName, wikiAddress) print('Number of names: %s' % len(names)) for name in names: cString.write('\nhttp://devtome.com/doku.php?id=wiki:user:%s' % name) almoner.writeFileText(fileName, cString.getvalue())
def writeWidenedFile(folderName, peerRootAddress, peerTextUntilCoins, widenRound): 'Write widened file.' title = almoner.getSuffixedFileName('receiver.csv', str(widenRound)) widenPath = os.path.join(folderName, title) receiverAddress = almoner.getSuffixedFileName(peerRootAddress, str(widenRound)) receiverText = almoner.getLocationText(receiverAddress) receiverTextFromCoins = receiverText[receiverText.find('_begincoins'):] widenText = peerTextUntilCoins + receiverTextFromCoins almoner.writeFileText(widenPath, widenText)
def writeOutput(arguments): 'Write output.' if '-h' in arguments or '-help' in arguments: print(__doc__) return advertisingRevenue = int(almoner.getParameter(arguments, '0', 'revenue')) round = int(almoner.getParameter(arguments, '23', 'round')) advertisingFileName = almoner.getParameter(arguments, 'devtome', 'wiki') rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki') currentFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % round, 'current') previousFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % (round - 1), 'previous') ratingFileName = almoner.getParameter(arguments, 'rating_%s.csv' % round, 'rating') viewFileName = almoner.getParameter(arguments, 'devtome_analytics_%s.csv' % round, 'view') categoryDictionary = {} nolines = almoner.getTextLines(almoner.getFileText(previousFileName)) lines = [nolines[0]] titleLine = lines[0] titles = titleLine.split(',') backupFolder = rootFileName + '_articles' ratingDictionary = getRatingDictionary(ratingFileName) viewDictionary = getViewDictionary(viewFileName) authors = getAuthors(backupFolder, categoryDictionary, lines, ratingDictionary, titles, viewDictionary) totalTomecount = getTotalTomecount(advertisingRevenue, authors) tomecountText = getTomecountText(authors, totalTomecount) advertisingRevenueText = getAdvertisingRevenueText(authors) earningsText = getEarningsText(authors) activeWritersText = getActiveWritersText(authors, round) newArticlesText = getNewArticlesText(authors, round) warningsText = getWarningsText(authors) outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt', 'summary') almoner.writeFileText(currentFileName, tomecountText) outputActiveWritersTo = almoner.getParameter(arguments, 'devtome_active_writers.txt', 'writers') outputAdvertisingRevenueTo = almoner.getParameter(arguments, 'devtome_advertising_revenue.csv', 'advertising') outputEarningsTo = almoner.getParameter(arguments, 'devtome_earnings_%s.csv' % round, 'earnings') outputNewArticlesTo = almoner.getParameter(arguments, 'devtome_new_articles.txt', 'articles') outputWarningsTo = almoner.getParameter(arguments, 'devtome_warnings.txt', 'warnings') writeCategoryFiles(categoryDictionary, rootFileName) if advertisingRevenue > 0: if almoner.sendOutputTo(outputAdvertisingRevenueTo, advertisingRevenueText): print('The devtome advertising revenue file has been written to:\n%s\n' % outputAdvertisingRevenueTo) if almoner.sendOutputTo(outputEarningsTo, earningsText): print('The devtome earnings file has been written to:\n%s\n' % outputEarningsTo) if almoner.sendOutputTo(outputNewArticlesTo, newArticlesText): print('The devtome new articles file has been written to:\n%s\n' % outputNewArticlesTo) if almoner.sendOutputTo(outputActiveWritersTo, activeWritersText): print('The devtome active writers file has been written to:\n%s\n' % outputActiveWritersTo) if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, round, totalTomecount)): print('The summary file has been written to:\n%s\n' % outputSummaryTo) if almoner.sendOutputTo(outputWarningsTo, warningsText): print('The devtome warnings file has been written to:\n%s\n' % outputWarningsTo)
def writeOutput(arguments): 'Write output.' if '-h' in arguments or '-help' in arguments: print(__doc__) return advertisingRevenue = int(almoner.getParameter(arguments, '0', 'revenue')) round = int(almoner.getParameter(arguments, '23', 'round')) advertisingFileName = almoner.getParameter(arguments, 'devtome', 'wiki') rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki') currentFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % round, 'current') previousFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % (round - 1), 'previous') ratingFileName = almoner.getParameter(arguments, 'rating_%s.csv' % round, 'rating') viewFileName = almoner.getParameter(arguments, 'devtome_analytics_%s.csv' % round, 'view') categoryDictionary = {} lines = almoner.getTextLines(almoner.getFileText(previousFileName)) titleLine = lines[0] titles = titleLine.split(',') backupFolder = rootFileName + '_articles' ratingDictionary = getRatingDictionary(ratingFileName) viewDictionary = getViewDictionary(viewFileName) authors = getAuthors(backupFolder, categoryDictionary, lines, ratingDictionary, titles, viewDictionary) totalTomecount = getTotalTomecount(advertisingRevenue, authors) tomecountText = getTomecountText(authors, totalTomecount) advertisingRevenueText = getAdvertisingRevenueText(authors) earningsText = getEarningsText(authors) activeWritersText = getActiveWritersText(authors, round) newArticlesText = getNewArticlesText(authors, round) warningsText = getWarningsText(authors) outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt', 'summary') almoner.writeFileText(currentFileName, tomecountText) outputActiveWritersTo = almoner.getParameter(arguments, 'devtome_active_writers.txt', 'writers') outputAdvertisingRevenueTo = almoner.getParameter(arguments, 'devtome_advertising_revenue.csv', 'advertising') outputEarningsTo = almoner.getParameter(arguments, 'devtome_earnings_%s.csv' % round, 'earnings') outputNewArticlesTo = almoner.getParameter(arguments, 'devtome_new_articles.txt', 'articles') outputWarningsTo = almoner.getParameter(arguments, 'devtome_warnings.txt', 'warnings') writeCategoryFiles(categoryDictionary, rootFileName) if advertisingRevenue > 0: if almoner.sendOutputTo(outputAdvertisingRevenueTo, advertisingRevenueText): print('The devtome advertising revenue file has been written to:\n%s\n' % outputAdvertisingRevenueTo) if almoner.sendOutputTo(outputEarningsTo, earningsText): print('The devtome earnings file has been written to:\n%s\n' % outputEarningsTo) if almoner.sendOutputTo(outputNewArticlesTo, newArticlesText): print('The devtome new articles file has been written to:\n%s\n' % outputNewArticlesTo) if almoner.sendOutputTo(outputActiveWritersTo, activeWritersText): print('The devtome active writers file has been written to:\n%s\n' % outputActiveWritersTo) if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, round, totalTomecount)): print('The summary file has been written to:\n%s\n' % outputSummaryTo) if almoner.sendOutputTo(outputWarningsTo, warningsText): print('The devtome warnings file has been written to:\n%s\n' % outputWarningsTo)
def saveArticle(self, categoryDictionary, linkName, linkText, underscoredLinkName): 'Save article and if new add to new articles list.' if underscoredLinkName not in self.backupFileSet: self.newArticles.append(underscoredLinkName) almoner.writeFileText(os.path.join(self.backupFolder, underscoredLinkName), linkText) self.tomecount.articleCount += 1 linkTextLower = linkText.lower() categoryIndex = linkTextLower.find('[[category:') if categoryIndex != -1: categoryPrefixLength = len('[[category:') self.tomecount.categorizedArticleCount += 1 while categoryIndex != -1: startIndex = categoryIndex + categoryPrefixLength endBracketIndex = linkTextLower.find(']]', startIndex) if endBracketIndex == -1: return categoryName = linkTextLower[startIndex : endBracketIndex].strip().replace('_', ' ') if categoryName in categoryDictionary: categoryDictionary[categoryName].append(linkName) else: categoryDictionary[categoryName] = [linkName] categoryIndex = linkTextLower.find('[[category:', endBracketIndex)
def main(): 'Replace peers.' beginPeerString = '_beginpeers\n' endPeerString = '_endpeers\n' receiverIndex = 0 receiverFileName = getReceiverFileName(receiverIndex) receiverText = almoner.getFileText(receiverFileName) peerLines = almoner.getTextLines(almoner.getFileText('peer.csv')) replacementString = beginPeerString + almoner.getTextByLines( peerLines) + endPeerString while receiverText != '': beginPeerIndex = receiverText.find(beginPeerString) endPeerEndIndex = receiverText.find(endPeerString) + len(endPeerString) betweenString = receiverText[beginPeerIndex:endPeerEndIndex] almoner.writeFileText('backup_receiver_%s.csv' % receiverIndex, receiverText) replacedText = receiverText.replace(betweenString, replacementString) almoner.writeFileText(receiverFileName, replacedText) print('Replaced peers in file: ' + receiverFileName) receiverIndex += 1 receiverFileName = getReceiverFileName(receiverIndex) receiverText = almoner.getFileText(receiverFileName, False)
def __init__(self, backupFolder, titles, words): 'Initialize.' self.tomecount = Tomecount() self.parameterDictionary = {} for wordIndex, word in enumerate(words): self.parameterDictionary[titles[wordIndex]] = word if 'Cumulative Payout' in self.parameterDictionary: self.tomecount.previousPayout = int(self.parameterDictionary['Cumulative Payout']) name = self.parameterDictionary['Name'] self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % name print('Loading articles from %s' % name) sourceText = getSourceText(self.sourceAddress) almoner.writeFileText(os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower: isCollated = False isOriginal = False if 'collated' in lineStrippedLower: isCollated = True elif 'original' in lineStrippedLower: isOriginal = True if isCollated: linkText = getLinkText(lineStrippedLower) self.tomecount.imageCount += getImageCount(linkText) wordCount = getWordCount(linkText) self.tomecount.collatedWordCount += wordCount if wordCount > 0: print('Collated article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount))) almoner.writeFileText(os.path.join(backupFolder, getLinkName(lineStrippedLower)[1 :]), linkText) if isOriginal: linkText = getLinkText(lineStrippedLower) self.tomecount.imageCount += getImageCount(linkText) wordCount = getWordCount(linkText) self.tomecount.originalWordCount += wordCount if wordCount > 0: print('Original article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount))) almoner.writeFileText(os.path.join(backupFolder, getLinkName(lineStrippedLower)[1 :]), linkText) self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10 self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount if self.tomecount.weightedWordCount >= 1000: self.tomecount.cumulativePayout = int(round(float(self.tomecount.weightedWordCount) * 0.001)) print('Weighted Word Count: %s' % almoner.getCommaNumberString(self.tomecount.weightedWordCount)) self.tomecount.payout = max(self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0) maximumPayout = 80 if self.tomecount.payout > maximumPayout: self.tomecount.payout = maximumPayout self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout
def writeCategoryFile(categoryDictionary, categoryFolder, categoryKey, rootFileName): 'Write category file to a folder.' categorySuffix = 'category:' + categoryKey categoryFileName = os.path.join(categoryFolder, categorySuffix) sourceText = almoner.getSourceText('http://devtome.com/doku.php?id=%s&do=edit' % categorySuffix).replace('"', '"') scriptToken = '{{script}}' scriptIndex = sourceText.find(scriptToken) if scriptIndex == -1: return scriptIndex += len(scriptToken) categoryText = sourceText[: scriptIndex] + '\n' afterScriptText = sourceText[scriptIndex :] lastLetter = None lines = almoner.getTextLines(afterScriptText) isAlphabeticallyGrouped = False scriptEndToken = None titleDictionary = {} for line in lines: if scriptEndToken == None: lineStripped = line.strip() if lineStripped != '': if lineStripped.startswith('=') and lineStripped.endswith('='): heading = lineStripped.replace('=', '').strip() if len(heading) > 1: scriptEndToken = lineStripped elif len(heading) == 1: isAlphabeticallyGrouped = True else: if lineStripped.startswith('*'): lineStripped = lineStripped[1 :] if lineStripped.startswith('[['): lineStripped = lineStripped[2 :] if lineStripped.startswith(':'): lineStripped = lineStripped[1 :] if lineStripped.endswith(']]'): lineStripped = lineStripped[: -2] titleKey = lineStripped.lower().replace('_', ' ') barIndex = titleKey.find('|') if barIndex != -1: titleKey = titleKey[: barIndex] titleDictionary[titleKey] = lineStripped fromTokenText = '' if scriptEndToken != None: fromTokenText = afterScriptText[afterScriptText.find(scriptEndToken) :] articleTitles = categoryDictionary[categoryKey] for articleTitle in articleTitles: articleTitleLower = articleTitle.lower().replace('_', ' ') if articleTitleLower not in titleDictionary: titleDictionary[articleTitleLower] = articleTitle titleKeys = titleDictionary.keys() titleKeys.sort() for titleKey in titleKeys: if isAlphabeticallyGrouped: firstLetter = titleKey[0] if firstLetter != lastLetter: categoryText += '===%s===\n' % firstLetter.capitalize() lastLetter = firstLetter title = titleDictionary[titleKey] if not ']]' in title: title += ']]' categoryText += '[[:%s\n\n' % title categoryText += fromTokenText almoner.writeFileText(os.path.join(categoryFolder, categorySuffix), categoryText)
def __init__(self, averageRating, backupFolder, backupFileSet, categoryDictionary, ratingDictionary, titles, viewDictionary, words): 'Initialize.' self.backupFolder = backupFolder self.backupFileSet = backupFileSet identicalCollatedCount = 0 identicalOriginalCount = 0 self.newArticles = [] self.tomecount = Tomecount() self.parameterDictionary = {} self.sentenceSet = set([]) self.warnings = [] for wordIndex, word in enumerate(words): self.parameterDictionary[titles[wordIndex]] = word if 'Cumulative Payout' in self.parameterDictionary: self.tomecount.previousPayout = int(self.parameterDictionary['Cumulative Payout']) self.name = self.parameterDictionary['Name'] self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name tipAddress = '' print('Loading articles from %s' % self.name) sourceText = almoner.getSourceText(self.sourceAddress) almoner.writeFileText(os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower: if '===' not in lineStrippedLower: isCollated = False isOriginal = False isTip = False if isCollated: linkName = getLinkName(line, self.name) underscoredLinkName = linkName.lower().replace(' ', '_') linkText = getSourceTextIfByAuthor(self, linkName) if linkName != '' and linkText == '': self.printWarning('Warning, could not invoice article link: %s' % linkName) if linkText not in linkTexts: linkTexts.add(linkText) self.tomecount.imageCount += getImageCount(linkText) wordCount = getWordCount(linkText) if underscoredLinkName in viewDictionary: self.tomecount.pageViews += viewDictionary[underscoredLinkName] if wordCount > 0: print('Collated article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount))) self.saveArticle(categoryDictionary, linkName, linkText, underscoredLinkName) identicalCollatedCount += self.getIdenticalWordCount(linkText) self.tomecount.collatedWordCount += wordCount if isOriginal: linkName = getLinkName(line, self.name) underscoredLinkName = linkName.lower().replace(' ', '_') linkText = getSourceTextIfByAuthor(self, linkName) if linkName != '' and linkText == '': self.printWarning('Warning, could not invoice article link: %s' % linkName) if linkText not in linkTexts: linkTexts.add(linkText) self.tomecount.imageCount += getImageCount(linkText) wordCount = getWordCount(linkText) if underscoredLinkName in viewDictionary: self.tomecount.pageViews += viewDictionary[underscoredLinkName] if wordCount > 0: print('Original article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount))) self.saveArticle(categoryDictionary, linkName, linkText, underscoredLinkName) identicalOriginalCount += self.getIdenticalWordCount(linkText) self.tomecount.originalWordCount += wordCount if isTip: tipLine = line.strip().replace("'", '') colonIndex = tipLine.find(':') if colonIndex >= 0: addressName = tipLine[: colonIndex].strip().lower() if 'dvc' in addressName or 'devcoin' in addressName or 'coin address' in addressName: tipAddress = tipLine[colonIndex + 1 :].strip() if '==' in lineStrippedLower: if 'collated' in lineStrippedLower: isCollated = True elif 'original' in lineStrippedLower: isOriginal = True elif 'tip' in lineStrippedLower: isTip = True if identicalCollatedCount > 0: self.tomecount.collatedWeightedWordCount -= identicalCollatedCount print('Identical Collated Word Count: %s' % almoner.getCommaNumberString(identicalCollatedCount)) if identicalOriginalCount > 0: self.tomecount.originalWordCount -= identicalOriginalCount print('Identical Original Word Count: %s' % almoner.getCommaNumberString(identicalOriginalCount)) self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10 self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount if self.tomecount.weightedWordCount >= 1000: self.tomecount.cumulativePayout = int(round(float(self.tomecount.weightedWordCount) * 0.001)) print('Weighted Word Count: %s' % almoner.getCommaNumberString(self.tomecount.weightedWordCount)) self.tomecount.payout = max(self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0) maximumPayout = 50 if tipAddress != self.parameterDictionary['Coin Address'] and self.name != 'Mosinnagant': self.printWarning('Warning, the coin address is not the same as the tip address, so nothing will be paid.') maximumPayout = 0 if self.tomecount.payout > maximumPayout: self.tomecount.payout = maximumPayout self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout if self.tomecount.cumulativePayout > 0: self.tomecount.categorization = float(self.tomecount.categorizedArticleCount) / float(self.tomecount.articleCount) self.tomecount.ratingMedian = averageRating lowerName = self.name.lower() if lowerName in ratingDictionary: self.tomecount.ratingMedian = ratingDictionary[lowerName] weightedPageViews = self.tomecount.pageViews if self.tomecount.previousPayout == 0: weightedPageViews += weightedPageViews self.tomecount.viewsPerThousandWords = 1000.0 * float(weightedPageViews) / float(self.tomecount.weightedWordCount) self.tomecount.normalizedCategorization = self.tomecount.categorization self.tomecount.normalizedPopularity = self.tomecount.viewsPerThousandWords self.tomecount.normalizedRatingMedian = self.tomecount.ratingMedian self.tomecount.popularityTimesRating = int(round(self.tomecount.pageViews * float(self.tomecount.ratingMedian) / 99.0))
def __init__(self, backupFolder, titles, words): 'Initialize.' self.tomecount = Tomecount() self.parameterDictionary = {} for wordIndex, word in enumerate(words): self.parameterDictionary[titles[wordIndex]] = word if 'Cumulative Payout' in self.parameterDictionary: self.tomecount.previousPayout = int( self.parameterDictionary['Cumulative Payout']) name = self.parameterDictionary['Name'] self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % name print('Loading articles from %s' % name) sourceText = getSourceText(self.sourceAddress) almoner.writeFileText(os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower: isCollated = False isOriginal = False if 'collated' in lineStrippedLower: isCollated = True elif 'original' in lineStrippedLower: isOriginal = True if isCollated: linkText = getLinkText(lineStrippedLower) self.tomecount.imageCount += getImageCount(linkText) wordCount = getWordCount(linkText) self.tomecount.collatedWordCount += wordCount if wordCount > 0: print('Collated article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount))) almoner.writeFileText( os.path.join(backupFolder, getLinkName(lineStrippedLower)[1:]), linkText) if isOriginal: linkText = getLinkText(lineStrippedLower) self.tomecount.imageCount += getImageCount(linkText) wordCount = getWordCount(linkText) self.tomecount.originalWordCount += wordCount if wordCount > 0: print('Original article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount))) almoner.writeFileText( os.path.join(backupFolder, getLinkName(lineStrippedLower)[1:]), linkText) self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10 self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount if self.tomecount.weightedWordCount >= 1000: self.tomecount.cumulativePayout = int( round(float(self.tomecount.weightedWordCount) * 0.001)) print('Weighted Word Count: %s' % almoner.getCommaNumberString(self.tomecount.weightedWordCount)) self.tomecount.payout = max( self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0) maximumPayout = 80 if self.tomecount.payout > maximumPayout: self.tomecount.payout = maximumPayout self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout
def __init__(self, averageRating, backupFolder, backupFileSet, categoryDictionary, ratingDictionary, titles, viewDictionary, words): 'Initialize.' self.backupFolder = backupFolder self.backupFileSet = backupFileSet identicalCollatedCount = 0 identicalOriginalCount = 0 self.newArticles = [] self.tomecount = Tomecount() self.parameterDictionary = {} self.sentenceSet = set([]) self.warnings = [] for wordIndex, word in enumerate(words): self.parameterDictionary[titles[wordIndex]] = word if 'Cumulative Payout' in self.parameterDictionary: self.tomecount.previousPayout = int( self.parameterDictionary['Cumulative Payout']) self.name = self.parameterDictionary['Name'] self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name tipAddress = '' print('Loading articles from %s' % self.name) sourceText = almoner.getSourceText(self.sourceAddress) almoner.writeFileText( os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower: if '===' not in lineStrippedLower: isCollated = False isOriginal = False isTip = False if isCollated: linkName = getLinkName(line, self.name) underscoredLinkName = linkName.lower().replace(' ', '_') linkText = getSourceTextIfByAuthor(self, linkName) if linkName != '' and linkText == '': self.printWarning( 'Warning, could not invoice article link: %s' % linkName) if linkText not in linkTexts: linkTexts.add(linkText) self.tomecount.imageCount += getImageCount(linkText) wordCount = getWordCount(linkText) if underscoredLinkName in viewDictionary: self.tomecount.pageViews += viewDictionary[ underscoredLinkName] if wordCount > 0: print('Collated article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount))) self.saveArticle(categoryDictionary, linkName, linkText, underscoredLinkName) identicalCollatedCount += self.getIdenticalWordCount( linkText) self.tomecount.collatedWordCount += wordCount if isOriginal: linkName = getLinkName(line, self.name) underscoredLinkName = linkName.lower().replace(' ', '_') linkText = getSourceTextIfByAuthor(self, linkName) if linkName != '' and linkText == '': self.printWarning( 'Warning, could not invoice article link: %s' % linkName) if linkText not in linkTexts: linkTexts.add(linkText) self.tomecount.imageCount += getImageCount(linkText) wordCount = getWordCount(linkText) if underscoredLinkName in viewDictionary: self.tomecount.pageViews += viewDictionary[ underscoredLinkName] if wordCount > 0: print('Original article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount))) self.saveArticle(categoryDictionary, linkName, linkText, underscoredLinkName) identicalOriginalCount += self.getIdenticalWordCount( linkText) self.tomecount.originalWordCount += wordCount if isTip: tipLine = line.strip().replace("'", '') colonIndex = tipLine.find(':') if colonIndex >= 0: addressName = tipLine[:colonIndex].strip().lower() if 'dvc' in addressName or 'devcoin' in addressName or 'coin address' in addressName: tipAddress = tipLine[colonIndex + 1:].strip() if '==' in lineStrippedLower: if 'collated' in lineStrippedLower: isCollated = True elif 'original' in lineStrippedLower: isOriginal = True elif 'tip' in lineStrippedLower: isTip = True if identicalCollatedCount > 0: self.tomecount.collatedWeightedWordCount -= identicalCollatedCount print('Identical Collated Word Count: %s' % almoner.getCommaNumberString(identicalCollatedCount)) if identicalOriginalCount > 0: self.tomecount.originalWordCount -= identicalOriginalCount print('Identical Original Word Count: %s' % almoner.getCommaNumberString(identicalOriginalCount)) self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10 self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount if self.tomecount.weightedWordCount >= 1000: self.tomecount.cumulativePayout = int( round(float(self.tomecount.weightedWordCount) * 0.001)) print('Weighted Word Count: %s' % almoner.getCommaNumberString(self.tomecount.weightedWordCount)) self.tomecount.payout = max( self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0) maximumPayout = 50 if tipAddress != self.parameterDictionary[ 'Coin Address'] and self.name != 'Mosinnagant': self.printWarning( 'Warning, the coin address is not the same as the tip address, so nothing will be paid.' ) maximumPayout = 0 if self.tomecount.payout > maximumPayout: self.tomecount.payout = maximumPayout self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout if self.tomecount.cumulativePayout > 0: self.tomecount.categorization = float( self.tomecount.categorizedArticleCount) / float( self.tomecount.articleCount) self.tomecount.ratingMedian = averageRating lowerName = self.name.lower() if lowerName in ratingDictionary: self.tomecount.ratingMedian = ratingDictionary[lowerName] weightedPageViews = self.tomecount.pageViews if self.tomecount.previousPayout == 0: weightedPageViews += weightedPageViews self.tomecount.viewsPerThousandWords = 1000.0 * float( weightedPageViews) / float(self.tomecount.weightedWordCount) self.tomecount.normalizedCategorization = self.tomecount.categorization self.tomecount.normalizedPopularity = self.tomecount.viewsPerThousandWords self.tomecount.normalizedRatingMedian = self.tomecount.ratingMedian self.tomecount.popularityTimesRating = int( round(self.tomecount.pageViews * float(self.tomecount.ratingMedian) / 99.0))
def writeCategoryFile(categoryDictionary, categoryFolder, categoryKey, rootFileName): 'Write category file to a folder.' categorySuffix = 'category:' + categoryKey categoryFileName = os.path.join(categoryFolder, categorySuffix) sourceText = almoner.getSourceText( 'http://devtome.com/doku.php?id=%s&do=edit' % categorySuffix).replace( '"', '"') scriptToken = '{{script}}' scriptIndex = sourceText.find(scriptToken) if scriptIndex == -1: return scriptIndex += len(scriptToken) categoryText = sourceText[:scriptIndex] + '\n' afterScriptText = sourceText[scriptIndex:] lastLetter = None lines = almoner.getTextLines(afterScriptText) isAlphabeticallyGrouped = False scriptEndToken = None titleDictionary = {} for line in lines: if scriptEndToken == None: lineStripped = line.strip() if lineStripped != '': if lineStripped.startswith('=') and lineStripped.endswith('='): heading = lineStripped.replace('=', '').strip() if len(heading) > 1: scriptEndToken = lineStripped elif len(heading) == 1: isAlphabeticallyGrouped = True else: if lineStripped.startswith('*'): lineStripped = lineStripped[1:] if lineStripped.startswith('[['): lineStripped = lineStripped[2:] if lineStripped.startswith(':'): lineStripped = lineStripped[1:] if lineStripped.endswith(']]'): lineStripped = lineStripped[:-2] titleKey = lineStripped.lower().replace('_', ' ') barIndex = titleKey.find('|') if barIndex != -1: titleKey = titleKey[:barIndex] titleDictionary[titleKey] = lineStripped fromTokenText = '' if scriptEndToken != None: fromTokenText = afterScriptText[afterScriptText.find(scriptEndToken):] articleTitles = categoryDictionary[categoryKey] for articleTitle in articleTitles: articleTitleLower = articleTitle.lower().replace('_', ' ') if articleTitleLower not in titleDictionary: titleDictionary[articleTitleLower] = articleTitle titleKeys = titleDictionary.keys() titleKeys.sort() for titleKey in titleKeys: if isAlphabeticallyGrouped: firstLetter = titleKey[0] if firstLetter != lastLetter: categoryText += '===%s===\n' % firstLetter.capitalize() lastLetter = firstLetter title = titleDictionary[titleKey] if not ']]' in title: title += ']]' categoryText += '[[:%s\n\n' % title categoryText += fromTokenText almoner.writeFileText(os.path.join(categoryFolder, categorySuffix), categoryText)
def __init__(self, backupFolder, titles, viewDictionary, words): 'Initialize.' self.tomecount = Tomecount() self.parameterDictionary = {} self.warnings = [] for wordIndex, word in enumerate(words): self.parameterDictionary[titles[wordIndex]] = word if 'Cumulative Payout' in self.parameterDictionary: self.tomecount.previousPayout = int(self.parameterDictionary['Cumulative Payout']) self.name = self.parameterDictionary['Name'] self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name tipAddress = '' print('Loading articles from %s' % self.name) sourceText = getSourceText(self.sourceAddress) almoner.writeFileText(os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower: isCollated = False isOriginal = False isTip = False if isCollated: lowerLinkName = getLinkName(line).lower() linkText = getLinkText(lowerLinkName, self.name) if linkText not in linkTexts: linkTexts.add(linkText) self.tomecount.imageCount += getImageCount(linkText) wordCount = getWordCount(linkText) self.tomecount.collatedWordCount += wordCount if lowerLinkName in viewDictionary: self.tomecount.pageViews += int(viewDictionary[lowerLinkName]) if wordCount > 0: print('Collated article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount))) almoner.writeFileText(os.path.join(backupFolder, lowerLinkName), linkText) if isOriginal: lowerLinkName = getLinkName(line).lower() linkText = getLinkText(lowerLinkName, self.name) if lowerLinkName != '' and linkText == '': self.printWarning('Warning, could not invoice article link: %s' % lowerLinkName) if linkText not in linkTexts: linkTexts.add(linkText) self.tomecount.imageCount += getImageCount(linkText) wordCount = getWordCount(linkText) self.tomecount.originalWordCount += wordCount if lowerLinkName in viewDictionary: self.tomecount.pageViews += int(viewDictionary[lowerLinkName]) if wordCount > 0: print('Original article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount))) almoner.writeFileText(os.path.join(backupFolder, lowerLinkName), linkText) if isTip: tipAddress = line.strip() if ':' in tipAddress: tipAddress = tipAddress[tipAddress.find(':') + 1 :].strip() if '==' in lineStrippedLower: if 'collated' in lineStrippedLower: isCollated = True elif 'original' in lineStrippedLower: isOriginal = True elif 'tip' in lineStrippedLower: isTip = True self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10 self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount if self.tomecount.weightedWordCount >= 1000: self.tomecount.cumulativePayout = int(round(float(self.tomecount.weightedWordCount) * 0.001)) print('Weighted Word Count: %s' % almoner.getCommaNumberString(self.tomecount.weightedWordCount)) self.tomecount.payout = max(self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0) maximumPayout = 80 if tipAddress != self.parameterDictionary['Coin Address'] and self.name != 'Mosinnagant': self.printWarning('Warning, the coin address is not the same as the tip address, so nothing will be paid.') maximumPayout = 0 if self.tomecount.payout > maximumPayout: self.tomecount.payout = maximumPayout self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout if self.tomecount.cumulativePayout > 0: worthRatio = float(self.tomecount.pageViews) / float(self.tomecount.weightedWordCount) self.tomecount.normalizedRootWorth = math.sqrt(math.sqrt(worthRatio))