def getRecentTitles(archiveType, fileNameRoot, wikiAddress):
'Get all titles of the dokuwiki.'
archiveFileName = fileNameRoot + '.' + archiveType
if not os.path.exists(archiveFileName):
return getTitles(wikiAddress)
if archiveType == 'zip':
zipArchive = zipfile.ZipFile(archiveFileName, 'r')
zipArchive.extractall(fileNameRoot)
zipArchive.close()
else:
mode = 'r'
if archiveType == 'bz2':
mode = 'r:bz2'
tarArchive = tarfile.open(archiveFileName, mode)
tarArchive.extractall(fileNameRoot)
tarArchive.close()
lastModifiedText = almoner.getFileText(os.path.join(fileNameRoot, 'last_modified.txt'))
lastModifiedDatetime = datetime.datetime.strptime(lastModifiedText, globalDateTimeFormat)
print('Last modified: %s' % lastModifiedDatetime)
nowDatetime = datetime.datetime.today()
nowMinusLast = nowDatetime - lastModifiedDatetime
print('Now minus last: %s' % nowMinusLast)
twentySixHours = 26 * 3600
if getSeconds(nowMinusLast) > (24 * 5 + 22) * 3600:
return getTitles(wikiAddress)
recentPageAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&first[0]'
lines = almoner.getTextLines(almoner.getInternetText(recentPageAddress))
lineDatetime = None
dateTitle = 'class="date">'
linkTitle = 'class="wikilink1" title="'
nameTitle = 'name="'
start = 0
titleSet = set([])
while True:
for lineIndex, line in enumerate(lines):
if dateTitle in line:
dateLine = lines[lineIndex + 1]
dateString = dateLine[: dateLine.find('<')]
if dateString.startswith('20'):
dateString = dateString[2 :]
lineDatetime = datetime.datetime.strptime(dateString, globalDateTimeFormat)
if linkTitle in line:
line = line[line.find(linkTitle) + len(linkTitle) :]
title = line[: line.find('"')]
if title != 'start':
lastMinusLine = lastModifiedDatetime - lineDatetime
if title in titleSet or getSeconds(lastMinusLine) > twentySixHours:
titles = list(titleSet)
titles.sort()
return titles
titleSet.add(title)
if line.startswith('<input') and 'value="less recent' in line and nameTitle in line:
line = line[line.find(nameTitle) + len(nameTitle) :]
name = line[: line.find('"')]
recentPageAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&' + name
lines = almoner.getTextLines(almoner.getInternetText(recentPageAddress))
return getTitles(wikiAddress)
def getRecentNames(fileName, nowDatetime, previousDevtomeName, wikiAddress):
'Get the recent user names.'
lastModifiedText = almoner.getFileText(fileName)
lastModifiedDatetime = nowDatetime - timedelta(30)
if lastModifiedText != '':
lines = almoner.getTextLines(lastModifiedText)
if len(lines) > 0:
words = lines[0].split(',')
if len(words) > 1:
lastModifiedDatetime = datetime.datetime.strptime(
words[1], globalDateTimeFormat)
print('Last modified: %s' % lastModifiedDatetime)
nowMinusLast = nowDatetime - lastModifiedDatetime
paidNameSet = getPaidNameSet(previousDevtomeName)
print('Now minus last: %s' % nowMinusLast)
twentySixHours = 26 * 3600
startChangesAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&'
recentPageAddress = startChangesAddress + 'first[0]'
lineDatetime = None
dateTitle = 'class="date">'
linkTitle = 'class="wikilink1" title="'
nameTitle = 'name="'
names = []
while True:
print('Parsing: %s' % recentPageAddress)
lines = almoner.getTextLines(
almoner.getInternetText(recentPageAddress))
for lineIndex, line in enumerate(lines):
if dateTitle in line:
dateLine = lines[lineIndex + 1]
dateString = dateLine[:dateLine.find('<')]
if dateString.startswith('20'):
dateString = dateString[len('20'):]
lineDatetime = datetime.datetime.strptime(
dateString, globalDateTimeFormat)
if linkTitle in line:
line = line[line.find(linkTitle) + len(linkTitle):]
name = line[:line.find('"')]
if name != 'start':
lastMinusLine = lastModifiedDatetime - lineDatetime
if getSeconds(lastMinusLine) > twentySixHours:
names.sort()
return names
if name.startswith('wiki:user:'******'wiki:user:'******'<input'
) and 'value="less recent' in line and nameTitle in line:
line = line[line.find(nameTitle) + len(nameTitle):]
name = line[:line.find('"')]
recentPageAddress = startChangesAddress + name
time.sleep(1)
return None
def getOutput(arguments):
'Get the output according to the arguments.'
bitcoinFileName = almoner.getParameter(arguments, 'bitcoinshare.html', 'inputbitcoin')
devcoinFileName = almoner.getParameter(arguments, '', 'inputdevcoin')
bitcoinOutput = almoner.getAddressText(bitcoinFileName)
devcoinOutput = almoner.getAddressText(devcoinFileName)
output = bitcoinOutput + devcoinOutput
print('Number of bitcoin lines: %s' % len(almoner.getTextLines(bitcoinOutput)))
print('Number of devcoin lines: %s' % len(almoner.getTextLines(devcoinOutput)))
print('Number of address lines: %s' % len(almoner.getTextLines(output)))
return output
def getOutput(arguments):
'Get the output according to the arguments.'
bitcoinFileName = almoner.getParameter(arguments, 'bitcoinshare.html', 'inputbitcoin')
devcoinFileName = almoner.getParameter(arguments, '', 'inputdevcoin')
bitcoinOutput = almoner.getAddressText(bitcoinFileName)
devcoinOutput = almoner.getAddressText(devcoinFileName)
output = bitcoinOutput + devcoinOutput
print('Number of bitcoin lines: %s' % len(almoner.getTextLines(bitcoinOutput)))
print('Number of devcoin lines: %s' % len(almoner.getTextLines(devcoinOutput)))
print('Number of address lines: %s' % len(almoner.getTextLines(output)))
return output
def getRecentNames(fileName, nowDatetime, previousDevtomeName, wikiAddress):
'Get the recent user names.'
lastModifiedText = almoner.getFileText(fileName)
lastModifiedDatetime = nowDatetime - timedelta(30)
if lastModifiedText != '':
lines = almoner.getTextLines(lastModifiedText)
if len(lines) > 0:
words = lines[0].split(',')
if len(words) > 1:
lastModifiedDatetime = datetime.datetime.strptime(words[1], globalDateTimeFormat)
print('Last modified: %s' % lastModifiedDatetime)
nowMinusLast = nowDatetime - lastModifiedDatetime
paidNameSet = getPaidNameSet(previousDevtomeName)
print('Now minus last: %s' % nowMinusLast)
twentySixHours = 26 * 3600
startChangesAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&'
recentPageAddress = startChangesAddress + 'first[0]'
lineDatetime = None
dateTitle = 'class="date">'
linkTitle = 'class="wikilink1" title="'
nameTitle = 'name="'
names = []
while True:
print('Parsing: %s' % recentPageAddress)
lines = almoner.getTextLines(almoner.getInternetText(recentPageAddress))
for lineIndex, line in enumerate(lines):
if dateTitle in line:
dateLine = lines[lineIndex + 1]
dateString = dateLine[: dateLine.find('<')]
if dateString.startswith('20'):
dateString = dateString[len('20') :]
lineDatetime = datetime.datetime.strptime(dateString, globalDateTimeFormat)
if linkTitle in line:
line = line[line.find(linkTitle) + len(linkTitle) :]
name = line[: line.find('"')]
if name != 'start':
lastMinusLine = lastModifiedDatetime - lineDatetime
if getSeconds(lastMinusLine) > twentySixHours:
names.sort()
return names
if name.startswith('wiki:user:'******'wiki:user:'******'<input') and 'value="less recent' in line and nameTitle in line:
line = line[line.find(nameTitle) + len(nameTitle) :]
name = line[: line.find('"')]
recentPageAddress = startChangesAddress + name
time.sleep(1)
return None
def getPeerLines(arguments):
'Get the inner peer text according to the arguments.'
peerFileName = almoner.getParameter(arguments, 'peer.csv', 'inputpeer')
peerLines = almoner.getTextLines(almoner.getLocationText(peerFileName))
print('Number of peers: %s' % len(peerLines))
print('')
return peerLines
def __init__(self, name):
'Initialize.'
self.articles = []
self.name = name
sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name
print('Loading user page from %s' % self.name)
sourceText = devtome.getSourceText(sourceAddress)
isCollated = False
isOriginal = False
for line in almoner.getTextLines(sourceText):
lineStrippedLower = line.strip().lower()
if '==' in lineStrippedLower:
isCollated = False
isOriginal = False
if isCollated:
lowerLinkName = devtome.getLinkName(line).lower()
self.articles.append(lowerLinkName)
if isOriginal:
lowerLinkName = devtome.getLinkName(line).lower()
self.articles.append(lowerLinkName)
if '==' in lineStrippedLower:
if 'collated' in lineStrippedLower:
isCollated = True
elif 'original' in lineStrippedLower:
isOriginal = True
def writeOutput(arguments):
'Write output.'
if '-h' in arguments or '-help' in arguments:
print(__doc__)
return
round = int(almoner.getParameter(arguments, '23', 'round'))
rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
currentFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % round, 'current')
previousFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % (round - 1), 'previous')
viewFileName = almoner.getParameter(arguments, 'devtome_analytics_%s.csv' % round, 'view')
lines = almoner.getTextLines(almoner.getFileText(previousFileName))
titleLine = lines[0]
titles = titleLine.split(',')
backupFolder = rootFileName + '_articles'
viewDictionary = getViewDictionary(viewFileName)
authors = getAuthors(backupFolder, lines, titles, viewDictionary)
totalTomecount = getTotalTomecount(authors)
tomecountText = getTomecountText(authors, totalTomecount)
earningsText = getEarningsText(authors)
outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt', 'summary')
almoner.writeFileText(currentFileName, tomecountText)
outputEarningsTo = almoner.getParameter(arguments, 'devtome_earnings_%s.csv' % round, 'earnings')
if almoner.sendOutputTo(outputEarningsTo, earningsText):
print('The devtome earnings file has been written to:\n%s\n' % outputEarningsTo)
if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, round, totalTomecount)):
print('The summary file has been written to:\n%s\n' % outputSummaryTo)
def getPeerLines(arguments):
'Get the inner peer text according to the arguments.'
peerFileName = almoner.getParameter(arguments, 'peer.csv', 'inputpeer')
peerLines = almoner.getTextLines(almoner.getLocationText(peerFileName))
print('Number of peers: %s' % len(peerLines))
print('')
return peerLines
def writeOutput(arguments):
'Write output.'
if '-h' in arguments or '-help' in arguments:
print(__doc__)
return
round = int(almoner.getParameter(arguments, '23', 'round'))
rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
currentFileName = almoner.getParameter(arguments,
rootFileName + '_%s.csv' % round,
'current')
previousFileName = almoner.getParameter(
arguments, rootFileName + '_%s.csv' % (round - 1), 'previous')
lines = almoner.getTextLines(almoner.getFileText(previousFileName))
titleLine = lines[0]
titles = titleLine.split(',')
backupFolder = rootFileName + '_articles'
authors = getAuthors(backupFolder, lines, titles)
totalTomecount = getTotalTomecount(authors)
tomecountText = getTomecountText(authors, totalTomecount)
earningsText = getEarningsText(authors)
outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt',
'summary')
almoner.writeFileText(currentFileName, tomecountText)
outputEarningsTo = almoner.getParameter(arguments,
'devtome_earnings_%s.csv' % round,
'earnings')
if almoner.sendOutputTo(outputEarningsTo, earningsText):
print('The devtome earnings file has been written to:\n%s\n' %
outputEarningsTo)
if almoner.sendOutputTo(
outputSummaryTo, getSummaryText(earningsText, round,
totalTomecount)):
print('The summary file has been written to:\n%s\n' % outputSummaryTo)
def getPaymentDictionary(round):
'Get the payment dictionary.'
addressDictionary = account.getAddressDictionary(round)
paymentDictionary = {}
lines = almoner.getTextLines(almoner.getFileText('receiver_%s.csv' % round))
isAddressSection = False
addressLines = []
for line in lines[: 4000]:
firstWord = ''
line = line.strip()
words = line.split(',')
if len(words) > 0:
firstWord = words[0].strip()
if firstWord == '_endcoins':
isAddressSection = False
if isAddressSection and len(line) > 0:
addressLines.append(line)
if firstWord == '_begincoins':
isAddressSection = True
addressLines += addressLines * (4000 / len(addressLines))
for addressLine in addressLines[: 4000]:
payment = 45000.0 / float(len(words))
words = addressLine.split(',')
for word in words:
name = addressDictionary[word].lower()
if name in paymentDictionary:
paymentDictionary[name] += payment
else:
paymentDictionary[name] = payment
return paymentDictionary
def getAccountLines(arguments, suffixNumberString):
'Get the lines according to the arguments.'
linkFileName = almoner.getParameter(arguments, 'account_location.csv',
'location')
linkLines = almoner.getTextLines(almoner.getLocationText(linkFileName))[1:]
accountLines = ['']
nameSet = set([])
for linkLine in linkLines:
linkLineSplit = linkLine.split(',')
name = linkLineSplit[0]
if 'Devcoin Share List' == name or 'Bitcoin Share List' == name:
continue
location = linkLineSplit[1]
extraLines = []
if '_xx' in location:
location = location.replace('_xx', '_' + suffixNumberString)
locationText = almoner.getLocationText(location)
if '<title>404 Not Found</title>' in locationText:
print('Warning, could not download page: %s' % location)
else:
extraLines = almoner.getTextLines(
locationText.replace('coinzen.org/index.php/topic,',
'coinzen.org/index.php/topic='))
else:
extraLines = getNameAddressLines(location, nameSet)
for extraLineIndex in xrange(len(extraLines) - 1, -1, -1):
extraWords = extraLines[extraLineIndex].split(',')
if len(extraWords) < 3:
print('Warning, less than 3 words in:')
print(linkLine)
print(extraWords)
print('')
del extraLines[extraLineIndex]
else:
secondWord = extraWords[1]
if '-' in secondWord or '(' in secondWord or ':' in secondWord:
print('Coin address is invalid in:')
print(extraWords)
del extraLines[extraLineIndex]
numberOfShares = len(getReceiverLinesByAccountLines(extraLines))
accountLines.append(name + ': %s Shares' % numberOfShares)
accountLines += extraLines
accountLines.append('')
# ~ addAdministratorBonus(accountLines)
return accountLines
def getViewDictionary(viewFileName):
'Get the page view dictionary.'
viewDictionary = {}
lines = almoner.getTextLines(almoner.getFileText(viewFileName))
for line in lines[1 :]:
words = line.split(',')
if len(words) > 1:
viewDictionary[words[0]] = words[1]
return viewDictionary
def getViewDictionary(viewFileName):
'Get the page view dictionary.'
viewDictionary = {}
lines = almoner.getTextLines(almoner.getFileText(viewFileName))
for line in lines[1:]:
words = line.split(',')
if len(words) > 1:
viewDictionary[words[0]] = int(words[1])
return viewDictionary
def getRaters():
raters = []
lines = almoner.getTextLines(almoner.getFileText('rater.csv'))
for line in lines[1 :]:
words = line.split(',')
if len(words) > 0:
rater = words[0].strip()
if rater != '':
raters.append(rater)
return raters
def getPaidNameSet(previousDevtomeName):
'Get the names of the paid writers.'
lines = almoner.getTextLines(almoner.getFileText(previousDevtomeName))
paidNameSet = set([])
for line in lines[1:]:
words = line.split(',')
if len(words) > 0:
firstWord = words[0].strip().lower()
if firstWord != '':
paidNameSet.add(firstWord)
return paidNameSet
def getPaidNameSet(previousDevtomeName):
'Get the names of the paid writers.'
lines = almoner.getTextLines(almoner.getFileText(previousDevtomeName))
paidNameSet = set([])
for line in lines[1 :]:
words = line.split(',')
if len(words) > 0:
firstWord = words[0].strip().lower()
if firstWord != '':
paidNameSet.add(firstWord)
return paidNameSet
def getCountDictionary(round):
'Get the weighted word count dictionary.'
countDictionary = {}
devtomeFileName = 'devtome_%s.csv' % round
lines = almoner.getTextLines(almoner.getFileText(devtomeFileName))
for line in lines[1 :]:
words = line.split(',')
if len(words) > 7:
name = words[0].strip()
if name != '':
countDictionary[name] = int(words[7].strip())
return countDictionary
def getWriters(round):
writers = []
lines = almoner.getTextLines(almoner.getFileText('devtome_%s.csv' % (round - 1)))
for line in lines[1 :]:
words = line.split(',')
if len(words) > 1:
name = words[0].strip()
if name != '':
writer = Writer(name)
if len(writer.articles) > 0:
writers.append(writer)
return writers
def __init__(self, backupFolder, titles, words):
'Initialize.'
self.tomecount = Tomecount()
self.parameterDictionary = {}
for wordIndex, word in enumerate(words):
self.parameterDictionary[titles[wordIndex]] = word
if 'Cumulative Payout' in self.parameterDictionary:
self.tomecount.previousPayout = int(self.parameterDictionary['Cumulative Payout'])
name = self.parameterDictionary['Name']
self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % name
print('Loading articles from %s' % name)
sourceText = getSourceText(self.sourceAddress)
almoner.writeFileText(os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower:
isCollated = False
isOriginal = False
if 'collated' in lineStrippedLower:
isCollated = True
elif 'original' in lineStrippedLower:
isOriginal = True
if isCollated:
linkText = getLinkText(lineStrippedLower)
self.tomecount.imageCount += getImageCount(linkText)
wordCount = getWordCount(linkText)
self.tomecount.collatedWordCount += wordCount
if wordCount > 0:
print('Collated article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount)))
almoner.writeFileText(os.path.join(backupFolder, getLinkName(lineStrippedLower)[1 :]), linkText)
if isOriginal:
linkText = getLinkText(lineStrippedLower)
self.tomecount.imageCount += getImageCount(linkText)
wordCount = getWordCount(linkText)
self.tomecount.originalWordCount += wordCount
if wordCount > 0:
print('Original article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount)))
almoner.writeFileText(os.path.join(backupFolder, getLinkName(lineStrippedLower)[1 :]), linkText)
self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10
self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount
self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount
self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount
if self.tomecount.weightedWordCount >= 1000:
self.tomecount.cumulativePayout = int(round(float(self.tomecount.weightedWordCount) * 0.001))
print('Weighted Word Count: %s' % almoner.getCommaNumberString(self.tomecount.weightedWordCount))
self.tomecount.payout = max(self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0)
maximumPayout = 80
if self.tomecount.payout > maximumPayout:
self.tomecount.payout = maximumPayout
self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout
def getRaterNames():
'Get the rater names.'
raterNames = []
lines = almoner.getTextLines(almoner.getFileText('rater.csv'))
for line in lines[1 :]:
words = line.split(',')
if len(words) > 0:
raterName = words[0].strip().lower()
if raterName != '':
if raterName[0] == '*':
raterName = raterName[1 :]
raterNames.append(raterName)
return raterNames
def addToAuthorDictionary(authorDictionary, name, text):
"Add author name to invoiced articles."
isArticle = False
for line in almoner.getTextLines(text):
lineStrippedLower = line.strip().lower()
if "==" in lineStrippedLower:
isArticle = False
if "collated" in lineStrippedLower or "original" in lineStrippedLower:
isArticle = True
if isArticle:
title = devtome.getLinkName(lineStrippedLower, name)
if title != "":
authorDictionary[title] = name
def getPreviousRecipentSet(round, start):
'Get the set of the previous recipient names.'
previousRecipentSet = set([])
for accountIndex in xrange(start, round):
accountFileName = 'account_%s.csv' % accountIndex
lines = almoner.getTextLines(almoner.getFileText(accountFileName))
for line in lines[1:]:
splitLine = line.split(',')
if len(splitLine) > 1:
name = splitLine[0].strip()
if name != '':
previousRecipentSet.add(name.lower())
return previousRecipentSet
def addToAuthorDictionary(authorDictionary, name, text):
'Add author name to invoiced articles.'
isArticle = False
for line in almoner.getTextLines(text):
lineStrippedLower = line.strip().lower()
if '==' in lineStrippedLower:
isArticle = False
if 'collated' in lineStrippedLower or 'original' in lineStrippedLower:
isArticle = True
if isArticle:
title = devtome.getLinkName(lineStrippedLower)
if title != '':
authorDictionary[title] = name[len('wiki:user:') :]
def getPreviousRecipentSet(round, start):
'Get the set of the previous recipient names.'
previousRecipentSet = set([])
for accountIndex in xrange(start, round):
accountFileName = 'account_%s.csv' % accountIndex
lines = almoner.getTextLines(almoner.getFileText(accountFileName))
for line in lines[1 :]:
splitLine = line.split(',')
if len(splitLine) > 1:
name = splitLine[0].strip()
if name != '':
previousRecipentSet.add(name)
return previousRecipentSet
def __init__(self, coinAddress, isShareName, name):
'Initialize.'
self.coinAddress = coinAddress
self.domainPayoutSet = set([])
self.name = name
self.payoutFifth = 0
self.postPayoutSet = set([])
self.postWords = 0
self.signaturePageSet = set([])
self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name
self.subdomainPayout = 0
print('\nLoading pages from %s' % self.name.capitalize())
sourceText = almoner.getSourceText(self.sourceAddress)
isLink = False
isPost = False
isSignature = False
for line in almoner.getTextLines(sourceText):
lineStrippedLower = line.strip().lower()
if '==' in lineStrippedLower:
isLink = False
isPost = False
isSignature = False
if 'link' in lineStrippedLower:
isLink = True
if 'post' in lineStrippedLower:
isPost = True
if 'signature' in lineStrippedLower:
isSignature = True
if isLink:
self.addLinkPayout(lineStrippedLower)
if isPost:
self.addPostPayout(lineStrippedLower)
if isSignature:
self.addSignaturePayout(lineStrippedLower)
if len(self.domainPayoutSet) == 0:
if self.subdomainPayout == 1:
self.payoutFifth += 1
print('Subdomain payout: 1')
if self.postWords > 100:
if self.postWords > 1000:
self.payoutFifth += 2
print('Big post payout: 2')
else:
self.payoutFifth += 1
print('Small post payout: 1')
if self.payoutFifth > 0:
if isShareName:
print('%s is on a share list, so the payout is doubled.' %
self.name)
self.payoutFifth += self.payoutFifth
print('Total payout fifths: %s' % self.payoutFifth)
def getAccountLines(arguments, suffixNumberString):
'Get the lines according to the arguments.'
linkFileName = almoner.getParameter(arguments, 'account_location.csv', 'location')
linkLines = almoner.getTextLines(almoner.getLocationText(linkFileName))[1 :]
accountLines = ['']
nameSet = set([])
for linkLine in linkLines:
linkLineSplit = linkLine.split(',')
name = linkLineSplit[0]
location = linkLineSplit[1]
extraLines = []
if '_xx' in location:
location = location.replace('_xx', '_' + suffixNumberString)
locationText = almoner.getLocationText(location)
if '<title>404 Not Found</title>' in locationText:
print('Warning, could not download page: %s' % location)
else:
extraLines = almoner.getTextLines(locationText.replace('coinzen.org/index.php/topic,', 'coinzen.org/index.php/topic='))
else:
extraLines = getNameAddressLines(location, nameSet)
for extraLineIndex in xrange(len(extraLines) - 1, -1, -1):
extraWords = extraLines[extraLineIndex].split(',')
if len(extraWords) < 3:
print('Warning, less than 3 words in:')
print(extraWords)
del extraLines[extraLineIndex]
else:
secondWord = extraWords[1]
if '-' in secondWord or '(' in secondWord or ':' in secondWord:
print('Coin address is invalid in:')
print(extraWords)
del extraLines[extraLineIndex]
numberOfShares = len(getReceiverLinesByAccountLines(extraLines))
accountLines.append(name + ': %s Shares' % numberOfShares)
accountLines += extraLines
accountLines.append('')
addAdministratorBonus(accountLines)
return accountLines
def __init__(self, coinAddress, isShareName, name):
'Initialize.'
self.coinAddress = coinAddress
self.domainPayoutSet = set([])
self.name = name
self.payoutFifth = 0
self.postPayoutSet = set([])
self.postWords = 0
self.signaturePageSet = set([])
self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name
self.subdomainPayout = 0
print('\nLoading pages from %s' % self.name)
sourceText = devtome.getSourceText(self.sourceAddress)
isLink = False
isPost = False
isSignature = False
for line in almoner.getTextLines(sourceText):
lineStrippedLower = line.strip().lower()
if '==' in lineStrippedLower:
isLink = False
isPost = False
isSignature = False
if 'link' in lineStrippedLower:
isLink = True
if 'post' in lineStrippedLower:
isPost = True
if 'signature' in lineStrippedLower:
isSignature = True
if isLink:
self.addLinkPayout(lineStrippedLower)
if isPost:
self.addPostPayout(lineStrippedLower)
if isSignature:
self.addSignaturePayout(lineStrippedLower)
if len(self.domainPayoutSet) == 0:
if self.subdomainPayout == 1:
self.payoutFifth += 1
print('Subdomain payout: 1')
if self.postWords > 100:
if self.postWords > 1000:
self.payoutFifth += 2
print('Big post payout: 2')
else:
self.payoutFifth += 1
print('Small post payout: 1')
if self.payoutFifth > 0:
if isShareName:
print('%s is on a share list, so the payout is doubled.' % self.name)
self.payoutFifth += self.payoutFifth
print('Total payout fifths: %s' % self.payoutFifth)
def getAccountLines(arguments, suffixNumberString):
'Get the lines according to the arguments.'
linkFileName = almoner.getParameter(arguments, 'account_location.csv', 'location')
linkLines = almoner.getTextLines(almoner.getLocationText(linkFileName))[1 :]
accountLines = ['']
for linkLine in linkLines:
linkLineSplit = linkLine.split(',')
name = linkLineSplit[0]
location = linkLineSplit[1]
extraLines = []
if '_xx' in location:
location = location.replace('_xx', '_' + suffixNumberString)
extraLines = almoner.getTextLines(almoner.getLocationText(location))
else:
extraLines = almoner.getNameAddressLines(location)
numberOfShares = len(getReceiverLinesByAccountLines(extraLines))
print('Name: %s, Location: %s, Shares: %s' % (name, location, numberOfShares))
accountLines.append(name + ': %s Shares' % numberOfShares)
accountLines += extraLines
accountLines.append('')
addAdministratorBonus(accountLines)
print('')
return accountLines
def writeOutput(arguments):
'Write output.'
if '-h' in arguments or '-help' in arguments:
print(__doc__)
return
advertisingRevenue = int(almoner.getParameter(arguments, '0', 'revenue'))
round = int(almoner.getParameter(arguments, '23', 'round'))
advertisingFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
currentFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % round, 'current')
previousFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % (round - 1), 'previous')
ratingFileName = almoner.getParameter(arguments, 'rating_%s.csv' % round, 'rating')
viewFileName = almoner.getParameter(arguments, 'devtome_analytics_%s.csv' % round, 'view')
categoryDictionary = {}
nolines = almoner.getTextLines(almoner.getFileText(previousFileName))
lines = [nolines[0]]
titleLine = lines[0]
titles = titleLine.split(',')
backupFolder = rootFileName + '_articles'
ratingDictionary = getRatingDictionary(ratingFileName)
viewDictionary = getViewDictionary(viewFileName)
authors = getAuthors(backupFolder, categoryDictionary, lines, ratingDictionary, titles, viewDictionary)
totalTomecount = getTotalTomecount(advertisingRevenue, authors)
tomecountText = getTomecountText(authors, totalTomecount)
advertisingRevenueText = getAdvertisingRevenueText(authors)
earningsText = getEarningsText(authors)
activeWritersText = getActiveWritersText(authors, round)
newArticlesText = getNewArticlesText(authors, round)
warningsText = getWarningsText(authors)
outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt', 'summary')
almoner.writeFileText(currentFileName, tomecountText)
outputActiveWritersTo = almoner.getParameter(arguments, 'devtome_active_writers.txt', 'writers')
outputAdvertisingRevenueTo = almoner.getParameter(arguments, 'devtome_advertising_revenue.csv', 'advertising')
outputEarningsTo = almoner.getParameter(arguments, 'devtome_earnings_%s.csv' % round, 'earnings')
outputNewArticlesTo = almoner.getParameter(arguments, 'devtome_new_articles.txt', 'articles')
outputWarningsTo = almoner.getParameter(arguments, 'devtome_warnings.txt', 'warnings')
writeCategoryFiles(categoryDictionary, rootFileName)
if advertisingRevenue > 0:
if almoner.sendOutputTo(outputAdvertisingRevenueTo, advertisingRevenueText):
print('The devtome advertising revenue file has been written to:\n%s\n' % outputAdvertisingRevenueTo)
if almoner.sendOutputTo(outputEarningsTo, earningsText):
print('The devtome earnings file has been written to:\n%s\n' % outputEarningsTo)
if almoner.sendOutputTo(outputNewArticlesTo, newArticlesText):
print('The devtome new articles file has been written to:\n%s\n' % outputNewArticlesTo)
if almoner.sendOutputTo(outputActiveWritersTo, activeWritersText):
print('The devtome active writers file has been written to:\n%s\n' % outputActiveWritersTo)
if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, round, totalTomecount)):
print('The summary file has been written to:\n%s\n' % outputSummaryTo)
if almoner.sendOutputTo(outputWarningsTo, warningsText):
print('The devtome warnings file has been written to:\n%s\n' % outputWarningsTo)
def getRatingDictionary(ratingFileName):
'Get the rating median vote dictionary.'
lines = almoner.getTextLines(almoner.getFileText(ratingFileName))
if len(lines) < 2:
return {}
ratingDictionary = {}
ratingMedianIndex = getRatingMedianIndex(lines[0])
for line in lines[1 :]:
words = line.split(',')
if len(words) > ratingMedianIndex:
name = words[0].strip().lower()
if len(name) > 0:
ratingDictionary[name] = float(words[ratingMedianIndex].strip().lower())
return ratingDictionary
def getAccountLines(arguments, suffixNumberString):
'Get the lines according to the arguments.'
linkFileName = almoner.getParameter(arguments, 'account_location.csv', 'location')
linkLines = almoner.getTextLines(almoner.getLocationText(linkFileName))[1 :]
accountLines = ['']
for linkLine in linkLines:
linkLineSplit = linkLine.split(',')
name = linkLineSplit[0]
location = linkLineSplit[1]
extraLines = []
if '_xx' in location:
location = location.replace('_xx', '_' + suffixNumberString)
extraLines = almoner.getTextLines(almoner.getLocationText(location))
else:
extraLines = almoner.getNameAddressLines(location)
numberOfShares = len(getReceiverLinesByAccountLines(extraLines))
print('Name: %s, Location: %s, Shares: %s' % (name, location, numberOfShares))
accountLines.append(name + ': %s Shares' % numberOfShares)
accountLines += extraLines
accountLines.append('')
addAdministratorBonus(accountLines)
print('')
return accountLines
def getRatingDictionary(ratingFileName):
'Get the rating median vote dictionary.'
lines = almoner.getTextLines(almoner.getFileText(ratingFileName))
if len(lines) < 2:
return {}
ratingDictionary = {}
ratingMedianIndex = getRatingMedianIndex(lines[0])
for line in lines[1 :]:
words = line.split(',')
if len(words) > ratingMedianIndex:
name = words[0].strip().lower()
if len(name) > 0:
ratingDictionary[name] = float(words[ratingMedianIndex].strip().lower())
return ratingDictionary
def writeOutput(arguments):
'Write output.'
if '-h' in arguments or '-help' in arguments:
print(__doc__)
return
advertisingRevenue = int(almoner.getParameter(arguments, '0', 'revenue'))
round = int(almoner.getParameter(arguments, '23', 'round'))
advertisingFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
currentFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % round, 'current')
previousFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % (round - 1), 'previous')
ratingFileName = almoner.getParameter(arguments, 'rating_%s.csv' % round, 'rating')
viewFileName = almoner.getParameter(arguments, 'devtome_analytics_%s.csv' % round, 'view')
categoryDictionary = {}
lines = almoner.getTextLines(almoner.getFileText(previousFileName))
titleLine = lines[0]
titles = titleLine.split(',')
backupFolder = rootFileName + '_articles'
ratingDictionary = getRatingDictionary(ratingFileName)
viewDictionary = getViewDictionary(viewFileName)
authors = getAuthors(backupFolder, categoryDictionary, lines, ratingDictionary, titles, viewDictionary)
totalTomecount = getTotalTomecount(advertisingRevenue, authors)
tomecountText = getTomecountText(authors, totalTomecount)
advertisingRevenueText = getAdvertisingRevenueText(authors)
earningsText = getEarningsText(authors)
activeWritersText = getActiveWritersText(authors, round)
newArticlesText = getNewArticlesText(authors, round)
warningsText = getWarningsText(authors)
outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt', 'summary')
almoner.writeFileText(currentFileName, tomecountText)
outputActiveWritersTo = almoner.getParameter(arguments, 'devtome_active_writers.txt', 'writers')
outputAdvertisingRevenueTo = almoner.getParameter(arguments, 'devtome_advertising_revenue.csv', 'advertising')
outputEarningsTo = almoner.getParameter(arguments, 'devtome_earnings_%s.csv' % round, 'earnings')
outputNewArticlesTo = almoner.getParameter(arguments, 'devtome_new_articles.txt', 'articles')
outputWarningsTo = almoner.getParameter(arguments, 'devtome_warnings.txt', 'warnings')
writeCategoryFiles(categoryDictionary, rootFileName)
if advertisingRevenue > 0:
if almoner.sendOutputTo(outputAdvertisingRevenueTo, advertisingRevenueText):
print('The devtome advertising revenue file has been written to:\n%s\n' % outputAdvertisingRevenueTo)
if almoner.sendOutputTo(outputEarningsTo, earningsText):
print('The devtome earnings file has been written to:\n%s\n' % outputEarningsTo)
if almoner.sendOutputTo(outputNewArticlesTo, newArticlesText):
print('The devtome new articles file has been written to:\n%s\n' % outputNewArticlesTo)
if almoner.sendOutputTo(outputActiveWritersTo, activeWritersText):
print('The devtome active writers file has been written to:\n%s\n' % outputActiveWritersTo)
if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, round, totalTomecount)):
print('The summary file has been written to:\n%s\n' % outputSummaryTo)
if almoner.sendOutputTo(outputWarningsTo, warningsText):
print('The devtome warnings file has been written to:\n%s\n' % outputWarningsTo)
def writeOutput(arguments):
'Write output.'
if '-h' in arguments or '-help' in arguments:
print(__doc__)
return
round = int(almoner.getParameter(arguments, '23', 'round'))
publishersFileName = almoner.getParameter(arguments, 'devtome_%s.csv' % round, 'publishers')
lines = almoner.getTextLines(almoner.getFileText(publishersFileName))
outputEarningsTo = almoner.getParameter(arguments, 'marketing_earnings_%s.csv' % round, 'earnings')
outputSummaryTo = almoner.getParameter(arguments, 'marketing_summary.txt', 'summary')
publishers = getPublishers(lines, round)
earningsText = getEarningsText(publishers)
if almoner.sendOutputTo(outputEarningsTo, earningsText):
print('The marketing earnings bounty file has been written to:\n%s\n' % outputEarningsTo)
if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, publishers, round)):
print('The summary file has been written to:\n%s\n' % outputSummaryTo)
def writeOutput(arguments):
'Write output.'
if '-h' in arguments or '-help' in arguments:
print(__doc__)
return
round = int(almoner.getParameter(arguments, '23', 'round'))
publishersFileName = almoner.getParameter(arguments, 'devtome_%s.csv' % round, 'publishers')
lines = almoner.getTextLines(almoner.getFileText(publishersFileName))
outputEarningsTo = almoner.getParameter(arguments, 'marketing_earnings_%s.csv' % round, 'earnings')
outputSummaryTo = almoner.getParameter(arguments, 'marketing_summary.txt', 'summary')
workerNameSet = set(account.getRecipientDictionary(round).keys())
publishers = getPublishers(lines, workerNameSet)
earningsText = getEarningsText(publishers)
if almoner.sendOutputTo(outputEarningsTo, earningsText):
print('The marketing earnings bounty file has been written to:\n%s\n' % outputEarningsTo)
if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, publishers, round)):
print('The summary file has been written to:\n%s\n' % outputSummaryTo)
def getRatingsByAddress(address):
'Get the ratings by address.'
ratings = []
firstUnderscore = address.find('_')
if firstUnderscore == -1:
print('Warning, no underscore in address.')
return []
lastUnderscore = address.rfind('_')
if firstUnderscore == lastUnderscore:
print('Warning, firstUnderscore same as lastUnderscore.')
return []
rater = address[firstUnderscore + 1 : lastUnderscore].lower()
lines = almoner.getTextLines(almoner.getSourceText(address))
for line in lines:
rating = Rating(address, line, rater)
if rating.article != '':
ratings.append(rating)
return ratings
def getWriterNames(writerFileName):
'Get the writer names.'
writerNames = []
lines = almoner.getTextLines(almoner.getFileText(writerFileName))
if lines[0].startswith('Create:'):
isWriterName = False
for line in lines[1 :]:
if isWriterName:
writerNames.append(line.strip().lower())
if line.startswith('Writer names:'):
isWriterName = True
return writerNames
for line in lines[1 :]:
words = line.split(',')
if len(words) > 0:
name = words[0].strip().lower()
if name != '':
writerNames.append(name)
return writerNames
def getWeightedWordCounts(lastRound, start):
'Get the weighted word counts.'
weightedWordCounts = []
for round in xrange(start, lastRound):
devtomeFileName = 'devtome_%s.csv' % round
if not os.path.exists(devtomeFileName):
return weightedWordCounts
devtomeText = almoner.getFileText(devtomeFileName)
if devtomeText == '':
return weightedWordCounts
weightedWordCountIndex = -1
for line in almoner.getTextLines(devtomeText):
words = line.strip().split(',')
if len(words) > 1:
if weightedWordCountIndex != -1:
weightedWordCounts.append(int(words[weightedWordCountIndex]))
weightedWordCountIndex = -1
if words[0].strip() == '' and words[1].strip() == 'Totals':
weightedWordCountIndex = words.index('Weighted Word Count')
return weightedWordCounts
def getArticles(name):
'Get invoiced articles by name.'
articles = []
sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % name
print('Loading user page from %s' % name)
sourceText = almoner.getSourceText(sourceAddress)
isArticle = False
for line in almoner.getTextLines(sourceText):
lineStrippedLower = line.strip().lower()
if '==' in lineStrippedLower:
if '===' not in lineStrippedLower:
isArticle = False
if isArticle:
lowerLinkName = devtome.getLinkName(line, name).lower()
if lowerLinkName != '':
articles.append(lowerLinkName)
if '==' in lineStrippedLower:
if 'collated' in lineStrippedLower or 'original' in lineStrippedLower:
isArticle = True
return articles
def getTitles(wikiAddress):
'Get all titles of the dokuwiki.'
indexDepth = 0
popularPageAddress = wikiAddress + '/doku.php?id=start&idx=wiki%3Auser'
lines = almoner.getTextLines(almoner.getInternetText(popularPageAddress))
prefix = '?id='
prefixLength = len(prefix)
titles = []
for line in lines:
if line.startswith('</ul>'):
if indexDepth > 0:
indexDepth -= 1
if indexDepth > 0 and 'class="wikilink1"' in line:
prefixIndex = line.find(prefix) + prefixLength
title = line[prefixIndex:]
quoteIndex = title.find('"')
if len(title) > 0:
titles.append(title[:quoteIndex])
if line == '<ul class="idx">':
indexDepth += 1
return titles
def getWeightedWordCounts(lastRound, start):
'Get the weighted word counts.'
weightedWordCounts = []
for round in xrange(start, lastRound):
devtomeFileName = 'devtome_%s.csv' % round
if not os.path.exists(devtomeFileName):
return weightedWordCounts
devtomeText = almoner.getFileText(devtomeFileName)
if devtomeText == '':
return weightedWordCounts
weightedWordCountIndex = -1
for line in almoner.getTextLines(devtomeText):
words = line.strip().split(',')
if len(words) > 1:
if weightedWordCountIndex != -1:
weightedWordCounts.append(
int(words[weightedWordCountIndex]))
weightedWordCountIndex = -1
if words[0].strip() == '' and words[1].strip() == 'Totals':
weightedWordCountIndex = words.index('Weighted Word Count')
return weightedWordCounts
def getTitles(wikiAddress):
'Get all titles of the dokuwiki.'
indexDepth = 0
popularPageAddress = wikiAddress + '/doku.php?id=start&idx=wiki%3Auser'
lines = almoner.getTextLines(almoner.getInternetText(popularPageAddress))
prefix = '?id='
prefixLength = len(prefix)
titles = []
for line in lines:
if line.startswith('</ul>'):
if indexDepth > 0:
indexDepth -= 1
if indexDepth > 0 and 'class="wikilink1"' in line:
prefixIndex = line.find(prefix) + prefixLength
title = line[prefixIndex :]
quoteIndex = title.find('"')
if len(title) > 0:
titles.append(title[: quoteIndex])
if line == '<ul class="idx">':
indexDepth += 1
return titles
def main():
'Replace peers.'
beginPeerString = '_beginpeers\n'
endPeerString = '_endpeers\n'
receiverIndex = 0
receiverFileName = getReceiverFileName(receiverIndex)
receiverText = almoner.getFileText(receiverFileName)
peerLines = almoner.getTextLines(almoner.getFileText('peer.csv'))
replacementString = beginPeerString + almoner.getTextByLines(
peerLines) + endPeerString
while receiverText != '':
beginPeerIndex = receiverText.find(beginPeerString)
endPeerEndIndex = receiverText.find(endPeerString) + len(endPeerString)
betweenString = receiverText[beginPeerIndex:endPeerEndIndex]
almoner.writeFileText('backup_receiver_%s.csv' % receiverIndex,
receiverText)
replacedText = receiverText.replace(betweenString, replacementString)
almoner.writeFileText(receiverFileName, replacedText)
print('Replaced peers in file: ' + receiverFileName)
receiverIndex += 1
receiverFileName = getReceiverFileName(receiverIndex)
receiverText = almoner.getFileText(receiverFileName, False)
def __init__(self, backupFolder, titles, words):
'Initialize.'
self.tomecount = Tomecount()
self.parameterDictionary = {}
for wordIndex, word in enumerate(words):
self.parameterDictionary[titles[wordIndex]] = word
if 'Cumulative Payout' in self.parameterDictionary:
self.tomecount.previousPayout = int(
self.parameterDictionary['Cumulative Payout'])
name = self.parameterDictionary['Name']
self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % name
print('Loading articles from %s' % name)
sourceText = getSourceText(self.sourceAddress)
almoner.writeFileText(os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower:
isCollated = False
isOriginal = False
if 'collated' in lineStrippedLower:
isCollated = True
elif 'original' in lineStrippedLower:
isOriginal = True
if isCollated:
linkText = getLinkText(lineStrippedLower)
self.tomecount.imageCount += getImageCount(linkText)
wordCount = getWordCount(linkText)
self.tomecount.collatedWordCount += wordCount
if wordCount > 0:
print('Collated article: %s, Word Count: %s' %
(lineStrippedLower,
almoner.getCommaNumberString(wordCount)))
almoner.writeFileText(
os.path.join(backupFolder,
getLinkName(lineStrippedLower)[1:]),
linkText)
if isOriginal:
linkText = getLinkText(lineStrippedLower)
self.tomecount.imageCount += getImageCount(linkText)
wordCount = getWordCount(linkText)
self.tomecount.originalWordCount += wordCount
if wordCount > 0:
print('Original article: %s, Word Count: %s' %
(lineStrippedLower,
almoner.getCommaNumberString(wordCount)))
almoner.writeFileText(
os.path.join(backupFolder,
getLinkName(lineStrippedLower)[1:]),
linkText)
self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10
self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount
self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount
self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount
if self.tomecount.weightedWordCount >= 1000:
self.tomecount.cumulativePayout = int(
round(float(self.tomecount.weightedWordCount) * 0.001))
print('Weighted Word Count: %s' %
almoner.getCommaNumberString(self.tomecount.weightedWordCount))
self.tomecount.payout = max(
self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0)
maximumPayout = 80
if self.tomecount.payout > maximumPayout:
self.tomecount.payout = maximumPayout
self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout
def __init__(self, averageRating, backupFolder, backupFileSet,
categoryDictionary, ratingDictionary, titles, viewDictionary,
words):
'Initialize.'
self.backupFolder = backupFolder
self.backupFileSet = backupFileSet
identicalCollatedCount = 0
identicalOriginalCount = 0
self.newArticles = []
self.tomecount = Tomecount()
self.parameterDictionary = {}
self.sentenceSet = set([])
self.warnings = []
for wordIndex, word in enumerate(words):
self.parameterDictionary[titles[wordIndex]] = word
if 'Cumulative Payout' in self.parameterDictionary:
self.tomecount.previousPayout = int(
self.parameterDictionary['Cumulative Payout'])
self.name = self.parameterDictionary['Name']
self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name
tipAddress = ''
print('Loading articles from %s' % self.name)
sourceText = almoner.getSourceText(self.sourceAddress)
almoner.writeFileText(
os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower:
if '===' not in lineStrippedLower:
isCollated = False
isOriginal = False
isTip = False
if isCollated:
linkName = getLinkName(line, self.name)
underscoredLinkName = linkName.lower().replace(' ', '_')
linkText = getSourceTextIfByAuthor(self, linkName)
if linkName != '' and linkText == '':
self.printWarning(
'Warning, could not invoice article link: %s' %
linkName)
if linkText not in linkTexts:
linkTexts.add(linkText)
self.tomecount.imageCount += getImageCount(linkText)
wordCount = getWordCount(linkText)
if underscoredLinkName in viewDictionary:
self.tomecount.pageViews += viewDictionary[
underscoredLinkName]
if wordCount > 0:
print('Collated article: %s, Word Count: %s' %
(lineStrippedLower,
almoner.getCommaNumberString(wordCount)))
self.saveArticle(categoryDictionary, linkName,
linkText, underscoredLinkName)
identicalCollatedCount += self.getIdenticalWordCount(
linkText)
self.tomecount.collatedWordCount += wordCount
if isOriginal:
linkName = getLinkName(line, self.name)
underscoredLinkName = linkName.lower().replace(' ', '_')
linkText = getSourceTextIfByAuthor(self, linkName)
if linkName != '' and linkText == '':
self.printWarning(
'Warning, could not invoice article link: %s' %
linkName)
if linkText not in linkTexts:
linkTexts.add(linkText)
self.tomecount.imageCount += getImageCount(linkText)
wordCount = getWordCount(linkText)
if underscoredLinkName in viewDictionary:
self.tomecount.pageViews += viewDictionary[
underscoredLinkName]
if wordCount > 0:
print('Original article: %s, Word Count: %s' %
(lineStrippedLower,
almoner.getCommaNumberString(wordCount)))
self.saveArticle(categoryDictionary, linkName,
linkText, underscoredLinkName)
identicalOriginalCount += self.getIdenticalWordCount(
linkText)
self.tomecount.originalWordCount += wordCount
if isTip:
tipLine = line.strip().replace("'", '')
colonIndex = tipLine.find(':')
if colonIndex >= 0:
addressName = tipLine[:colonIndex].strip().lower()
if 'dvc' in addressName or 'devcoin' in addressName or 'coin address' in addressName:
tipAddress = tipLine[colonIndex + 1:].strip()
if '==' in lineStrippedLower:
if 'collated' in lineStrippedLower:
isCollated = True
elif 'original' in lineStrippedLower:
isOriginal = True
elif 'tip' in lineStrippedLower:
isTip = True
if identicalCollatedCount > 0:
self.tomecount.collatedWeightedWordCount -= identicalCollatedCount
print('Identical Collated Word Count: %s' %
almoner.getCommaNumberString(identicalCollatedCount))
if identicalOriginalCount > 0:
self.tomecount.originalWordCount -= identicalOriginalCount
print('Identical Original Word Count: %s' %
almoner.getCommaNumberString(identicalOriginalCount))
self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10
self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount
self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount
self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount
if self.tomecount.weightedWordCount >= 1000:
self.tomecount.cumulativePayout = int(
round(float(self.tomecount.weightedWordCount) * 0.001))
print('Weighted Word Count: %s' %
almoner.getCommaNumberString(self.tomecount.weightedWordCount))
self.tomecount.payout = max(
self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0)
maximumPayout = 50
if tipAddress != self.parameterDictionary[
'Coin Address'] and self.name != 'Mosinnagant':
self.printWarning(
'Warning, the coin address is not the same as the tip address, so nothing will be paid.'
)
maximumPayout = 0
if self.tomecount.payout > maximumPayout:
self.tomecount.payout = maximumPayout
self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout
if self.tomecount.cumulativePayout > 0:
self.tomecount.categorization = float(
self.tomecount.categorizedArticleCount) / float(
self.tomecount.articleCount)
self.tomecount.ratingMedian = averageRating
lowerName = self.name.lower()
if lowerName in ratingDictionary:
self.tomecount.ratingMedian = ratingDictionary[lowerName]
weightedPageViews = self.tomecount.pageViews
if self.tomecount.previousPayout == 0:
weightedPageViews += weightedPageViews
self.tomecount.viewsPerThousandWords = 1000.0 * float(
weightedPageViews) / float(self.tomecount.weightedWordCount)
self.tomecount.normalizedCategorization = self.tomecount.categorization
self.tomecount.normalizedPopularity = self.tomecount.viewsPerThousandWords
self.tomecount.normalizedRatingMedian = self.tomecount.ratingMedian
self.tomecount.popularityTimesRating = int(
round(self.tomecount.pageViews *
float(self.tomecount.ratingMedian) / 99.0))
def writeCategoryFile(categoryDictionary, categoryFolder, categoryKey,
rootFileName):
'Write category file to a folder.'
categorySuffix = 'category:' + categoryKey
categoryFileName = os.path.join(categoryFolder, categorySuffix)
sourceText = almoner.getSourceText(
'http://devtome.com/doku.php?id=%s&do=edit' % categorySuffix).replace(
'"', '"')
scriptToken = '{{script}}'
scriptIndex = sourceText.find(scriptToken)
if scriptIndex == -1:
return
scriptIndex += len(scriptToken)
categoryText = sourceText[:scriptIndex] + '\n'
afterScriptText = sourceText[scriptIndex:]
lastLetter = None
lines = almoner.getTextLines(afterScriptText)
isAlphabeticallyGrouped = False
scriptEndToken = None
titleDictionary = {}
for line in lines:
if scriptEndToken == None:
lineStripped = line.strip()
if lineStripped != '':
if lineStripped.startswith('=') and lineStripped.endswith('='):
heading = lineStripped.replace('=', '').strip()
if len(heading) > 1:
scriptEndToken = lineStripped
elif len(heading) == 1:
isAlphabeticallyGrouped = True
else:
if lineStripped.startswith('*'):
lineStripped = lineStripped[1:]
if lineStripped.startswith('[['):
lineStripped = lineStripped[2:]
if lineStripped.startswith(':'):
lineStripped = lineStripped[1:]
if lineStripped.endswith(']]'):
lineStripped = lineStripped[:-2]
titleKey = lineStripped.lower().replace('_', ' ')
barIndex = titleKey.find('|')
if barIndex != -1:
titleKey = titleKey[:barIndex]
titleDictionary[titleKey] = lineStripped
fromTokenText = ''
if scriptEndToken != None:
fromTokenText = afterScriptText[afterScriptText.find(scriptEndToken):]
articleTitles = categoryDictionary[categoryKey]
for articleTitle in articleTitles:
articleTitleLower = articleTitle.lower().replace('_', ' ')
if articleTitleLower not in titleDictionary:
titleDictionary[articleTitleLower] = articleTitle
titleKeys = titleDictionary.keys()
titleKeys.sort()
for titleKey in titleKeys:
if isAlphabeticallyGrouped:
firstLetter = titleKey[0]
if firstLetter != lastLetter:
categoryText += '===%s===\n' % firstLetter.capitalize()
lastLetter = firstLetter
title = titleDictionary[titleKey]
if not ']]' in title:
title += ']]'
categoryText += '[[:%s\n\n' % title
categoryText += fromTokenText
almoner.writeFileText(os.path.join(categoryFolder, categorySuffix),
categoryText)
def getRecentTitles(archiveType, fileNameRoot, wikiAddress):
'Get all titles of the dokuwiki.'
archiveFileName = fileNameRoot + '.' + archiveType
if not os.path.exists(archiveFileName):
return getTitles(wikiAddress)
if archiveType == 'zip':
zipArchive = zipfile.ZipFile(archiveFileName, 'r')
zipArchive.extractall(fileNameRoot)
zipArchive.close()
else:
mode = 'r'
if archiveType == 'bz2':
mode = 'r:bz2'
tarArchive = tarfile.open(archiveFileName, mode)
tarArchive.extractall(fileNameRoot)
tarArchive.close()
lastModifiedText = almoner.getFileText(
os.path.join(fileNameRoot, 'last_modified.txt'))
lastModifiedDatetime = datetime.datetime.strptime(lastModifiedText,
globalDateTimeFormat)
print('Last modified: %s' % lastModifiedDatetime)
nowDatetime = datetime.datetime.today()
nowMinusLast = nowDatetime - lastModifiedDatetime
print('Now minus last: %s' % nowMinusLast)
twentySixHours = 26 * 3600
if getSeconds(nowMinusLast) > (24 * 5 + 22) * 3600:
return getTitles(wikiAddress)
recentPageAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&first[0]'
lines = almoner.getTextLines(almoner.getInternetText(recentPageAddress))
lineDatetime = None
dateTitle = 'class="date">'
linkTitle = 'class="wikilink1" title="'
nameTitle = 'name="'
start = 0
titleSet = set([])
while True:
for lineIndex, line in enumerate(lines):
if dateTitle in line:
dateLine = lines[lineIndex + 1]
dateString = dateLine[:dateLine.find('<')]
if dateString.startswith('20'):
dateString = dateString[2:]
lineDatetime = datetime.datetime.strptime(
dateString, globalDateTimeFormat)
if linkTitle in line:
line = line[line.find(linkTitle) + len(linkTitle):]
title = line[:line.find('"')]
if title != 'start':
lastMinusLine = lastModifiedDatetime - lineDatetime
if title in titleSet or getSeconds(
lastMinusLine) > twentySixHours:
titles = list(titleSet)
titles.sort()
return titles
titleSet.add(title)
if line.startswith(
'<input'
) and 'value="less recent' in line and nameTitle in line:
line = line[line.find(nameTitle) + len(nameTitle):]
name = line[:line.find('"')]
recentPageAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&' + name
lines = almoner.getTextLines(
almoner.getInternetText(recentPageAddress))
return getTitles(wikiAddress)
def __init__(self, backupFolder, titles, viewDictionary, words):
'Initialize.'
self.tomecount = Tomecount()
self.parameterDictionary = {}
self.warnings = []
for wordIndex, word in enumerate(words):
self.parameterDictionary[titles[wordIndex]] = word
if 'Cumulative Payout' in self.parameterDictionary:
self.tomecount.previousPayout = int(self.parameterDictionary['Cumulative Payout'])
self.name = self.parameterDictionary['Name']
self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name
tipAddress = ''
print('Loading articles from %s' % self.name)
sourceText = getSourceText(self.sourceAddress)
almoner.writeFileText(os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower:
isCollated = False
isOriginal = False
isTip = False
if isCollated:
lowerLinkName = getLinkName(line).lower()
linkText = getLinkText(lowerLinkName, self.name)
if linkText not in linkTexts:
linkTexts.add(linkText)
self.tomecount.imageCount += getImageCount(linkText)
wordCount = getWordCount(linkText)
self.tomecount.collatedWordCount += wordCount
if lowerLinkName in viewDictionary:
self.tomecount.pageViews += int(viewDictionary[lowerLinkName])
if wordCount > 0:
print('Collated article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount)))
almoner.writeFileText(os.path.join(backupFolder, lowerLinkName), linkText)
if isOriginal:
lowerLinkName = getLinkName(line).lower()
linkText = getLinkText(lowerLinkName, self.name)
if lowerLinkName != '' and linkText == '':
self.printWarning('Warning, could not invoice article link: %s' % lowerLinkName)
if linkText not in linkTexts:
linkTexts.add(linkText)
self.tomecount.imageCount += getImageCount(linkText)
wordCount = getWordCount(linkText)
self.tomecount.originalWordCount += wordCount
if lowerLinkName in viewDictionary:
self.tomecount.pageViews += int(viewDictionary[lowerLinkName])
if wordCount > 0:
print('Original article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount)))
almoner.writeFileText(os.path.join(backupFolder, lowerLinkName), linkText)
if isTip:
tipAddress = line.strip()
if ':' in tipAddress:
tipAddress = tipAddress[tipAddress.find(':') + 1 :].strip()
if '==' in lineStrippedLower:
if 'collated' in lineStrippedLower:
isCollated = True
elif 'original' in lineStrippedLower:
isOriginal = True
elif 'tip' in lineStrippedLower:
isTip = True
self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10
self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount
self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount
self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount
if self.tomecount.weightedWordCount >= 1000:
self.tomecount.cumulativePayout = int(round(float(self.tomecount.weightedWordCount) * 0.001))
print('Weighted Word Count: %s' % almoner.getCommaNumberString(self.tomecount.weightedWordCount))
self.tomecount.payout = max(self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0)
maximumPayout = 80
if tipAddress != self.parameterDictionary['Coin Address'] and self.name != 'Mosinnagant':
self.printWarning('Warning, the coin address is not the same as the tip address, so nothing will be paid.')
maximumPayout = 0
if self.tomecount.payout > maximumPayout:
self.tomecount.payout = maximumPayout
self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout
if self.tomecount.cumulativePayout > 0:
worthRatio = float(self.tomecount.pageViews) / float(self.tomecount.weightedWordCount)
self.tomecount.normalizedRootWorth = math.sqrt(math.sqrt(worthRatio))
def getPreviousLines(round):
'Get the lines from the rating text of the previous round.'
return almoner.getTextLines(almoner.getFileText('rating_%s.csv' % (round - 1)))