コード例 #1
0
ファイル: wikicopy.py プロジェクト: jdbtracker/charity
def getRecentTitles(archiveType, fileNameRoot, wikiAddress):
	'Get all titles of the dokuwiki.'
	archiveFileName = fileNameRoot + '.' + archiveType
	if not os.path.exists(archiveFileName):
		return getTitles(wikiAddress)
	if archiveType == 'zip':
		zipArchive = zipfile.ZipFile(archiveFileName, 'r')
		zipArchive.extractall(fileNameRoot)
		zipArchive.close()
	else:
		mode = 'r'
		if archiveType == 'bz2':
			mode = 'r:bz2'
		tarArchive = tarfile.open(archiveFileName, mode)
		tarArchive.extractall(fileNameRoot)
		tarArchive.close()
	lastModifiedText = almoner.getFileText(os.path.join(fileNameRoot, 'last_modified.txt'))
	lastModifiedDatetime = datetime.datetime.strptime(lastModifiedText, globalDateTimeFormat)
	print('Last modified: %s' % lastModifiedDatetime)
	nowDatetime = datetime.datetime.today()
	nowMinusLast = nowDatetime - lastModifiedDatetime
	print('Now minus last: %s' % nowMinusLast)
	twentySixHours = 26 * 3600
	if getSeconds(nowMinusLast) > (24 * 5 + 22) * 3600:
		return getTitles(wikiAddress)
	recentPageAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&first[0]'
	lines = almoner.getTextLines(almoner.getInternetText(recentPageAddress))
	lineDatetime = None
	dateTitle = 'class="date">'
	linkTitle = 'class="wikilink1" title="'
	nameTitle = 'name="'
	start = 0
	titleSet = set([])
	while True:
		for lineIndex, line in enumerate(lines):
			if dateTitle in line:
				dateLine = lines[lineIndex + 1]
				dateString = dateLine[: dateLine.find('<')]
				if dateString.startswith('20'):
					dateString = dateString[2 :]
				lineDatetime = datetime.datetime.strptime(dateString, globalDateTimeFormat)
			if linkTitle in line:
				line = line[line.find(linkTitle) + len(linkTitle) :]
				title = line[: line.find('"')]
				if title != 'start':
					lastMinusLine = lastModifiedDatetime - lineDatetime
					if title in titleSet or getSeconds(lastMinusLine) > twentySixHours:
						titles = list(titleSet)
						titles.sort()
						return titles
					titleSet.add(title)
			if line.startswith('<input') and 'value="less recent' in line and nameTitle in line:
				line = line[line.find(nameTitle) + len(nameTitle) :]
				name = line[: line.find('"')]
				recentPageAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&' + name
		lines = almoner.getTextLines(almoner.getInternetText(recentPageAddress))
	return getTitles(wikiAddress)
コード例 #2
0
ファイル: writer.py プロジェクト: FuzzyBearBTC/charity
def getRecentNames(fileName, nowDatetime, previousDevtomeName, wikiAddress):
    'Get the recent user names.'
    lastModifiedText = almoner.getFileText(fileName)
    lastModifiedDatetime = nowDatetime - timedelta(30)
    if lastModifiedText != '':
        lines = almoner.getTextLines(lastModifiedText)
        if len(lines) > 0:
            words = lines[0].split(',')
            if len(words) > 1:
                lastModifiedDatetime = datetime.datetime.strptime(
                    words[1], globalDateTimeFormat)
    print('Last modified: %s' % lastModifiedDatetime)
    nowMinusLast = nowDatetime - lastModifiedDatetime
    paidNameSet = getPaidNameSet(previousDevtomeName)
    print('Now minus last: %s' % nowMinusLast)
    twentySixHours = 26 * 3600
    startChangesAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&'
    recentPageAddress = startChangesAddress + 'first[0]'
    lineDatetime = None
    dateTitle = 'class="date">'
    linkTitle = 'class="wikilink1" title="'
    nameTitle = 'name="'
    names = []
    while True:
        print('Parsing: %s' % recentPageAddress)
        lines = almoner.getTextLines(
            almoner.getInternetText(recentPageAddress))
        for lineIndex, line in enumerate(lines):
            if dateTitle in line:
                dateLine = lines[lineIndex + 1]
                dateString = dateLine[:dateLine.find('<')]
                if dateString.startswith('20'):
                    dateString = dateString[len('20'):]
                lineDatetime = datetime.datetime.strptime(
                    dateString, globalDateTimeFormat)
            if linkTitle in line:
                line = line[line.find(linkTitle) + len(linkTitle):]
                name = line[:line.find('"')]
                if name != 'start':
                    lastMinusLine = lastModifiedDatetime - lineDatetime
                    if getSeconds(lastMinusLine) > twentySixHours:
                        names.sort()
                        return names
                    if name.startswith('wiki:user:'******'wiki:user:'******'<input'
            ) and 'value="less recent' in line and nameTitle in line:
                line = line[line.find(nameTitle) + len(nameTitle):]
                name = line[:line.find('"')]
                recentPageAddress = startChangesAddress + name
        time.sleep(1)
    return None
コード例 #3
0
def getOutput(arguments):
	'Get the output according to the arguments.'
	bitcoinFileName = almoner.getParameter(arguments, 'bitcoinshare.html', 'inputbitcoin')
	devcoinFileName = almoner.getParameter(arguments, '', 'inputdevcoin')
	bitcoinOutput = almoner.getAddressText(bitcoinFileName)
	devcoinOutput = almoner.getAddressText(devcoinFileName)
	output = bitcoinOutput + devcoinOutput
	print('Number of bitcoin lines: %s' % len(almoner.getTextLines(bitcoinOutput)))
	print('Number of devcoin lines: %s' % len(almoner.getTextLines(devcoinOutput)))
	print('Number of address lines: %s' % len(almoner.getTextLines(output)))
	return output
コード例 #4
0
ファイル: devcoinalmoner.py プロジェクト: devcoin/accounting
def getOutput(arguments):
	'Get the output according to the arguments.'
	bitcoinFileName = almoner.getParameter(arguments, 'bitcoinshare.html', 'inputbitcoin')
	devcoinFileName = almoner.getParameter(arguments, '', 'inputdevcoin')
	bitcoinOutput = almoner.getAddressText(bitcoinFileName)
	devcoinOutput = almoner.getAddressText(devcoinFileName)
	output = bitcoinOutput + devcoinOutput
	print('Number of bitcoin lines: %s' % len(almoner.getTextLines(bitcoinOutput)))
	print('Number of devcoin lines: %s' % len(almoner.getTextLines(devcoinOutput)))
	print('Number of address lines: %s' % len(almoner.getTextLines(output)))
	return output
コード例 #5
0
ファイル: writer.py プロジェクト: Unthinkingbit/charity
def getRecentNames(fileName, nowDatetime, previousDevtomeName, wikiAddress):
	'Get the recent user names.'
	lastModifiedText = almoner.getFileText(fileName)
	lastModifiedDatetime = nowDatetime - timedelta(30)
	if lastModifiedText != '':
		lines = almoner.getTextLines(lastModifiedText)
		if len(lines) > 0:
			words = lines[0].split(',')
			if len(words) > 1:
				lastModifiedDatetime = datetime.datetime.strptime(words[1], globalDateTimeFormat)
	print('Last modified: %s' % lastModifiedDatetime)
	nowMinusLast = nowDatetime - lastModifiedDatetime
	paidNameSet = getPaidNameSet(previousDevtomeName)
	print('Now minus last: %s' % nowMinusLast)
	twentySixHours = 26 * 3600
	startChangesAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&'
	recentPageAddress = startChangesAddress + 'first[0]'
	lineDatetime = None
	dateTitle = 'class="date">'
	linkTitle = 'class="wikilink1" title="'
	nameTitle = 'name="'
	names = []
	while True:
		print('Parsing: %s' % recentPageAddress)
		lines = almoner.getTextLines(almoner.getInternetText(recentPageAddress))
		for lineIndex, line in enumerate(lines):
			if dateTitle in line:
				dateLine = lines[lineIndex + 1]
				dateString = dateLine[: dateLine.find('<')]
				if dateString.startswith('20'):
					dateString = dateString[len('20') :]
				lineDatetime = datetime.datetime.strptime(dateString, globalDateTimeFormat)
			if linkTitle in line:
				line = line[line.find(linkTitle) + len(linkTitle) :]
				name = line[: line.find('"')]
				if name != 'start':
					lastMinusLine = lastModifiedDatetime - lineDatetime
					if getSeconds(lastMinusLine) > twentySixHours:
						names.sort()
						return names
					if name.startswith('wiki:user:'******'wiki:user:'******'<input') and 'value="less recent' in line and nameTitle in line:
				line = line[line.find(nameTitle) + len(nameTitle) :]
				name = line[: line.find('"')]
				recentPageAddress = startChangesAddress + name
		time.sleep(1)
	return None
コード例 #6
0
ファイル: account.py プロジェクト: Unthinkingbit/charity
def getPeerLines(arguments):
	'Get the inner peer text according to the arguments.'
	peerFileName = almoner.getParameter(arguments, 'peer.csv', 'inputpeer')
	peerLines = almoner.getTextLines(almoner.getLocationText(peerFileName))
	print('Number of peers: %s' % len(peerLines))
	print('')
	return peerLines
コード例 #7
0
ファイル: rater.py プロジェクト: jdbtracker/charity
	def __init__(self, name):
		'Initialize.'
		self.articles = []
		self.name = name
		sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name
		print('Loading user page from %s' % self.name)
		sourceText = devtome.getSourceText(sourceAddress)
		isCollated = False
		isOriginal = False
		for line in almoner.getTextLines(sourceText):
			lineStrippedLower = line.strip().lower()
			if '==' in lineStrippedLower:
				isCollated = False
				isOriginal = False
			if isCollated:
				lowerLinkName = devtome.getLinkName(line).lower()
				self.articles.append(lowerLinkName)
			if isOriginal:
				lowerLinkName = devtome.getLinkName(line).lower()
				self.articles.append(lowerLinkName)
			if '==' in lineStrippedLower:
				if 'collated' in lineStrippedLower:
					isCollated = True
				elif 'original' in lineStrippedLower:
					isOriginal = True
コード例 #8
0
ファイル: devtome.py プロジェクト: jdbtracker/charity
def writeOutput(arguments):
	'Write output.'
	if '-h' in arguments or '-help' in arguments:
		print(__doc__)
		return
	round = int(almoner.getParameter(arguments, '23', 'round'))
	rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
	currentFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % round, 'current')
	previousFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % (round - 1), 'previous')
	viewFileName = almoner.getParameter(arguments, 'devtome_analytics_%s.csv' % round, 'view')
	lines = almoner.getTextLines(almoner.getFileText(previousFileName))
	titleLine = lines[0]
	titles = titleLine.split(',')
	backupFolder = rootFileName + '_articles'
	viewDictionary = getViewDictionary(viewFileName)
	authors = getAuthors(backupFolder, lines, titles, viewDictionary)
	totalTomecount = getTotalTomecount(authors)
	tomecountText = getTomecountText(authors, totalTomecount)
	earningsText = getEarningsText(authors)
	outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt', 'summary')
	almoner.writeFileText(currentFileName, tomecountText)
	outputEarningsTo = almoner.getParameter(arguments, 'devtome_earnings_%s.csv' % round, 'earnings')
	if almoner.sendOutputTo(outputEarningsTo, earningsText):
		print('The devtome earnings file has been written to:\n%s\n' % outputEarningsTo)
	if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, round, totalTomecount)):
		print('The summary file has been written to:\n%s\n' % outputSummaryTo)
コード例 #9
0
ファイル: account.py プロジェクト: moozilla/charity
def getPeerLines(arguments):
	'Get the inner peer text according to the arguments.'
	peerFileName = almoner.getParameter(arguments, 'peer.csv', 'inputpeer')
	peerLines = almoner.getTextLines(almoner.getLocationText(peerFileName))
	print('Number of peers: %s' % len(peerLines))
	print('')
	return peerLines
コード例 #10
0
def writeOutput(arguments):
    'Write output.'
    if '-h' in arguments or '-help' in arguments:
        print(__doc__)
        return
    round = int(almoner.getParameter(arguments, '23', 'round'))
    rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
    currentFileName = almoner.getParameter(arguments,
                                           rootFileName + '_%s.csv' % round,
                                           'current')
    previousFileName = almoner.getParameter(
        arguments, rootFileName + '_%s.csv' % (round - 1), 'previous')
    lines = almoner.getTextLines(almoner.getFileText(previousFileName))
    titleLine = lines[0]
    titles = titleLine.split(',')
    backupFolder = rootFileName + '_articles'
    authors = getAuthors(backupFolder, lines, titles)
    totalTomecount = getTotalTomecount(authors)
    tomecountText = getTomecountText(authors, totalTomecount)
    earningsText = getEarningsText(authors)
    outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt',
                                           'summary')
    almoner.writeFileText(currentFileName, tomecountText)
    outputEarningsTo = almoner.getParameter(arguments,
                                            'devtome_earnings_%s.csv' % round,
                                            'earnings')
    if almoner.sendOutputTo(outputEarningsTo, earningsText):
        print('The devtome earnings file has been written to:\n%s\n' %
              outputEarningsTo)
    if almoner.sendOutputTo(
            outputSummaryTo, getSummaryText(earningsText, round,
                                            totalTomecount)):
        print('The summary file has been written to:\n%s\n' % outputSummaryTo)
コード例 #11
0
ファイル: payment.py プロジェクト: FuzzyBearBTC/charity
def getPaymentDictionary(round):
	'Get the payment dictionary.'
	addressDictionary = account.getAddressDictionary(round)
	paymentDictionary = {}
	lines = almoner.getTextLines(almoner.getFileText('receiver_%s.csv' % round))
	isAddressSection = False
	addressLines = []
	for line in lines[: 4000]:
		firstWord = ''
		line = line.strip()
		words = line.split(',')
		if len(words) > 0:
			firstWord = words[0].strip()
		if firstWord == '_endcoins':
			isAddressSection = False
		if isAddressSection and len(line) > 0:
				addressLines.append(line)
		if firstWord == '_begincoins':
			isAddressSection = True
	addressLines += addressLines * (4000 / len(addressLines))
	for addressLine in addressLines[: 4000]:
		payment = 45000.0 / float(len(words))
		words = addressLine.split(',')
		for word in words:
			name = addressDictionary[word].lower()
			if name in paymentDictionary:
				paymentDictionary[name] += payment
			else:
				paymentDictionary[name] = payment
	return paymentDictionary
コード例 #12
0
ファイル: account.py プロジェクト: devcoin/accounting
def getAccountLines(arguments, suffixNumberString):
    'Get the lines according to the arguments.'
    linkFileName = almoner.getParameter(arguments, 'account_location.csv',
                                        'location')
    linkLines = almoner.getTextLines(almoner.getLocationText(linkFileName))[1:]
    accountLines = ['']
    nameSet = set([])
    for linkLine in linkLines:
        linkLineSplit = linkLine.split(',')
        name = linkLineSplit[0]
        if 'Devcoin Share List' == name or 'Bitcoin Share List' == name:
            continue
        location = linkLineSplit[1]
        extraLines = []
        if '_xx' in location:
            location = location.replace('_xx', '_' + suffixNumberString)
            locationText = almoner.getLocationText(location)
            if '<title>404 Not Found</title>' in locationText:
                print('Warning, could not download page: %s' % location)
            else:
                extraLines = almoner.getTextLines(
                    locationText.replace('coinzen.org/index.php/topic,',
                                         'coinzen.org/index.php/topic='))
        else:
            extraLines = getNameAddressLines(location, nameSet)
        for extraLineIndex in xrange(len(extraLines) - 1, -1, -1):
            extraWords = extraLines[extraLineIndex].split(',')
            if len(extraWords) < 3:
                print('Warning, less than 3 words in:')
                print(linkLine)
                print(extraWords)
                print('')
                del extraLines[extraLineIndex]
            else:
                secondWord = extraWords[1]
                if '-' in secondWord or '(' in secondWord or ':' in secondWord:
                    print('Coin address is invalid in:')
                    print(extraWords)
                    del extraLines[extraLineIndex]
        numberOfShares = len(getReceiverLinesByAccountLines(extraLines))
        accountLines.append(name + ': %s Shares' % numberOfShares)
        accountLines += extraLines
        accountLines.append('')
    # ~ addAdministratorBonus(accountLines)
    return accountLines
コード例 #13
0
ファイル: devtome.py プロジェクト: jdbtracker/charity
def getViewDictionary(viewFileName):
	'Get the page view dictionary.'
	viewDictionary = {}
	lines = almoner.getTextLines(almoner.getFileText(viewFileName))
	for line in lines[1 :]:
		words = line.split(',')
		if len(words) > 1:
			viewDictionary[words[0]] = words[1]
	return viewDictionary
コード例 #14
0
ファイル: devtome.py プロジェクト: chefatef22/charity
def getViewDictionary(viewFileName):
    'Get the page view dictionary.'
    viewDictionary = {}
    lines = almoner.getTextLines(almoner.getFileText(viewFileName))
    for line in lines[1:]:
        words = line.split(',')
        if len(words) > 1:
            viewDictionary[words[0]] = int(words[1])
    return viewDictionary
コード例 #15
0
ファイル: rater.py プロジェクト: jdbtracker/charity
def getRaters():
	raters = []
	lines = almoner.getTextLines(almoner.getFileText('rater.csv'))
	for line in lines[1 :]:
		words = line.split(',')
		if len(words) > 0:
			rater = words[0].strip()
			if rater != '':
				raters.append(rater)
	return raters
コード例 #16
0
ファイル: writer.py プロジェクト: FuzzyBearBTC/charity
def getPaidNameSet(previousDevtomeName):
    'Get the names of the paid writers.'
    lines = almoner.getTextLines(almoner.getFileText(previousDevtomeName))
    paidNameSet = set([])
    for line in lines[1:]:
        words = line.split(',')
        if len(words) > 0:
            firstWord = words[0].strip().lower()
            if firstWord != '':
                paidNameSet.add(firstWord)
    return paidNameSet
コード例 #17
0
ファイル: writer.py プロジェクト: Unthinkingbit/charity
def getPaidNameSet(previousDevtomeName):
	'Get the names of the paid writers.'
	lines = almoner.getTextLines(almoner.getFileText(previousDevtomeName))
	paidNameSet = set([])
	for line in lines[1 :]:
		words = line.split(',')
		if len(words) > 0:
			firstWord = words[0].strip().lower()
			if firstWord != '':
				paidNameSet.add(firstWord)
	return paidNameSet
コード例 #18
0
def getCountDictionary(round):
	'Get the weighted word count dictionary.'
	countDictionary = {}
	devtomeFileName = 'devtome_%s.csv' % round
	lines = almoner.getTextLines(almoner.getFileText(devtomeFileName))
	for line in lines[1 :]:
		words = line.split(',')
		if len(words) > 7:
			name = words[0].strip()
			if name != '':
				countDictionary[name] = int(words[7].strip())
	return countDictionary
コード例 #19
0
ファイル: rater.py プロジェクト: jdbtracker/charity
def getWriters(round):
	writers = []
	lines = almoner.getTextLines(almoner.getFileText('devtome_%s.csv' % (round - 1)))
	for line in lines[1 :]:
		words = line.split(',')
		if len(words) > 1:
			name = words[0].strip()
			if name != '':
				writer = Writer(name)
				if len(writer.articles) > 0:
					writers.append(writer)
	return writers
コード例 #20
0
ファイル: devtome.py プロジェクト: moozilla/charity
	def __init__(self, backupFolder, titles, words):
		'Initialize.'
		self.tomecount = Tomecount()
		self.parameterDictionary = {}
		for wordIndex, word in enumerate(words):
			self.parameterDictionary[titles[wordIndex]] = word
		if 'Cumulative Payout' in self.parameterDictionary:
			self.tomecount.previousPayout = int(self.parameterDictionary['Cumulative Payout'])
		name = self.parameterDictionary['Name']
		self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % name
		print('Loading articles from %s' % name)
		sourceText = getSourceText(self.sourceAddress)
		almoner.writeFileText(os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower:
				isCollated = False
				isOriginal = False
				if 'collated' in lineStrippedLower:
					isCollated = True
				elif 'original' in lineStrippedLower:
					isOriginal = True
			if isCollated:
				linkText = getLinkText(lineStrippedLower)
				self.tomecount.imageCount += getImageCount(linkText)
				wordCount = getWordCount(linkText)
				self.tomecount.collatedWordCount += wordCount
				if wordCount > 0:
					print('Collated article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount)))
					almoner.writeFileText(os.path.join(backupFolder, getLinkName(lineStrippedLower)[1 :]), linkText)
			if isOriginal:
				linkText = getLinkText(lineStrippedLower)
				self.tomecount.imageCount += getImageCount(linkText)
				wordCount = getWordCount(linkText)
				self.tomecount.originalWordCount += wordCount
				if wordCount > 0:
					print('Original article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount)))
					almoner.writeFileText(os.path.join(backupFolder, getLinkName(lineStrippedLower)[1 :]), linkText)
		self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10
		self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount
		self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount
		self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount
		if self.tomecount.weightedWordCount >= 1000:
			self.tomecount.cumulativePayout = int(round(float(self.tomecount.weightedWordCount) * 0.001))
		print('Weighted Word Count: %s' % almoner.getCommaNumberString(self.tomecount.weightedWordCount))
		self.tomecount.payout = max(self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0)
		maximumPayout = 80
		if self.tomecount.payout > maximumPayout:
			self.tomecount.payout = maximumPayout
			self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout
コード例 #21
0
def getRaterNames():
	'Get the rater names.'
	raterNames = []
	lines = almoner.getTextLines(almoner.getFileText('rater.csv'))
	for line in lines[1 :]:
		words = line.split(',')
		if len(words) > 0:
			raterName = words[0].strip().lower()
			if raterName != '':
				if raterName[0] == '*':
					raterName = raterName[1 :]
				raterNames.append(raterName)
	return raterNames
コード例 #22
0
ファイル: similarity.py プロジェクト: Unthinkingbit/charity
def addToAuthorDictionary(authorDictionary, name, text):
    "Add author name to invoiced articles."
    isArticle = False
    for line in almoner.getTextLines(text):
        lineStrippedLower = line.strip().lower()
        if "==" in lineStrippedLower:
            isArticle = False
            if "collated" in lineStrippedLower or "original" in lineStrippedLower:
                isArticle = True
        if isArticle:
            title = devtome.getLinkName(lineStrippedLower, name)
            if title != "":
                authorDictionary[title] = name
コード例 #23
0
ファイル: newbie.py プロジェクト: FuzzyBearBTC/charity
def getPreviousRecipentSet(round, start):
    'Get the set of the previous recipient names.'
    previousRecipentSet = set([])
    for accountIndex in xrange(start, round):
        accountFileName = 'account_%s.csv' % accountIndex
        lines = almoner.getTextLines(almoner.getFileText(accountFileName))
        for line in lines[1:]:
            splitLine = line.split(',')
            if len(splitLine) > 1:
                name = splitLine[0].strip()
                if name != '':
                    previousRecipentSet.add(name.lower())
    return previousRecipentSet
コード例 #24
0
ファイル: similarity.py プロジェクト: jdbtracker/charity
def addToAuthorDictionary(authorDictionary, name, text):
	'Add author name to invoiced articles.'
	isArticle = False
	for line in almoner.getTextLines(text):
		lineStrippedLower = line.strip().lower()
		if '==' in lineStrippedLower:
			isArticle = False
			if 'collated' in lineStrippedLower or 'original' in lineStrippedLower:
				isArticle = True
		if isArticle:
			title = devtome.getLinkName(lineStrippedLower)
			if title != '':
				authorDictionary[title] = name[len('wiki:user:') :]
コード例 #25
0
ファイル: newbie.py プロジェクト: moozilla/charity
def getPreviousRecipentSet(round, start):
	'Get the set of the previous recipient names.'
	previousRecipentSet = set([])
	for accountIndex in xrange(start, round):
		accountFileName = 'account_%s.csv' % accountIndex
		lines = almoner.getTextLines(almoner.getFileText(accountFileName))
		for line in lines[1 :]:
			splitLine = line.split(',')
			if len(splitLine) > 1:
				name = splitLine[0].strip()
				if name != '':
					previousRecipentSet.add(name)
	return previousRecipentSet
コード例 #26
0
ファイル: marketing.py プロジェクト: FuzzyBearBTC/charity
 def __init__(self, coinAddress, isShareName, name):
     'Initialize.'
     self.coinAddress = coinAddress
     self.domainPayoutSet = set([])
     self.name = name
     self.payoutFifth = 0
     self.postPayoutSet = set([])
     self.postWords = 0
     self.signaturePageSet = set([])
     self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name
     self.subdomainPayout = 0
     print('\nLoading pages from %s' % self.name.capitalize())
     sourceText = almoner.getSourceText(self.sourceAddress)
     isLink = False
     isPost = False
     isSignature = False
     for line in almoner.getTextLines(sourceText):
         lineStrippedLower = line.strip().lower()
         if '==' in lineStrippedLower:
             isLink = False
             isPost = False
             isSignature = False
             if 'link' in lineStrippedLower:
                 isLink = True
             if 'post' in lineStrippedLower:
                 isPost = True
             if 'signature' in lineStrippedLower:
                 isSignature = True
         if isLink:
             self.addLinkPayout(lineStrippedLower)
         if isPost:
             self.addPostPayout(lineStrippedLower)
         if isSignature:
             self.addSignaturePayout(lineStrippedLower)
     if len(self.domainPayoutSet) == 0:
         if self.subdomainPayout == 1:
             self.payoutFifth += 1
             print('Subdomain payout: 1')
     if self.postWords > 100:
         if self.postWords > 1000:
             self.payoutFifth += 2
             print('Big post payout: 2')
         else:
             self.payoutFifth += 1
             print('Small post payout: 1')
     if self.payoutFifth > 0:
         if isShareName:
             print('%s is on a share list, so the payout is doubled.' %
                   self.name)
             self.payoutFifth += self.payoutFifth
         print('Total payout fifths: %s' % self.payoutFifth)
コード例 #27
0
ファイル: account.py プロジェクト: Unthinkingbit/charity
def getAccountLines(arguments, suffixNumberString):
	'Get the lines according to the arguments.'
	linkFileName = almoner.getParameter(arguments, 'account_location.csv', 'location')
	linkLines = almoner.getTextLines(almoner.getLocationText(linkFileName))[1 :]
	accountLines = ['']
	nameSet = set([])
	for linkLine in linkLines:
		linkLineSplit = linkLine.split(',')
		name = linkLineSplit[0]
		location = linkLineSplit[1]
		extraLines = []
		if '_xx' in location:
			location = location.replace('_xx', '_' + suffixNumberString)
			locationText = almoner.getLocationText(location)
			if '<title>404 Not Found</title>' in locationText:
				print('Warning, could not download page: %s' % location)
			else:
				extraLines = almoner.getTextLines(locationText.replace('coinzen.org/index.php/topic,', 'coinzen.org/index.php/topic='))
		else:
			extraLines = getNameAddressLines(location, nameSet)
		for extraLineIndex in xrange(len(extraLines) - 1, -1, -1):
			extraWords = extraLines[extraLineIndex].split(',')
			if len(extraWords) < 3:
				print('Warning, less than 3 words in:')
				print(extraWords)
				del extraLines[extraLineIndex]
			else:
				secondWord = extraWords[1]
				if '-' in secondWord or '(' in secondWord or ':' in secondWord:
					print('Coin address is invalid in:')
					print(extraWords)
					del extraLines[extraLineIndex]
		numberOfShares = len(getReceiverLinesByAccountLines(extraLines))
		accountLines.append(name + ': %s Shares' % numberOfShares)
		accountLines += extraLines
		accountLines.append('')
	addAdministratorBonus(accountLines)
	return accountLines
コード例 #28
0
ファイル: marketing.py プロジェクト: jdbtracker/charity
	def __init__(self, coinAddress, isShareName, name):
		'Initialize.'
		self.coinAddress = coinAddress
		self.domainPayoutSet = set([])
		self.name = name
		self.payoutFifth = 0
		self.postPayoutSet = set([])
		self.postWords = 0
		self.signaturePageSet = set([])
		self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name
		self.subdomainPayout = 0
		print('\nLoading pages from %s' % self.name)
		sourceText = devtome.getSourceText(self.sourceAddress)
		isLink = False
		isPost = False
		isSignature = False
		for line in almoner.getTextLines(sourceText):
			lineStrippedLower = line.strip().lower()
			if '==' in lineStrippedLower:
				isLink = False
				isPost = False
				isSignature = False
				if 'link' in lineStrippedLower:
					isLink = True
				if 'post' in lineStrippedLower:
					isPost = True
				if 'signature' in lineStrippedLower:
					isSignature = True
			if isLink:
				self.addLinkPayout(lineStrippedLower)
			if isPost:
				self.addPostPayout(lineStrippedLower)
			if isSignature:
				self.addSignaturePayout(lineStrippedLower)
		if len(self.domainPayoutSet) == 0:
			if self.subdomainPayout == 1:
				self.payoutFifth += 1
				print('Subdomain payout: 1')
		if self.postWords > 100:
			if self.postWords > 1000:
				self.payoutFifth += 2
				print('Big post payout: 2')
			else:
				self.payoutFifth += 1
				print('Small post payout: 1')
		if self.payoutFifth > 0:
			if isShareName:
				print('%s is on a share list, so the payout is doubled.' % self.name)
				self.payoutFifth += self.payoutFifth
			print('Total payout fifths: %s' % self.payoutFifth)
コード例 #29
0
ファイル: account.py プロジェクト: jdbtracker/charity
def getAccountLines(arguments, suffixNumberString):
	'Get the lines according to the arguments.'
	linkFileName = almoner.getParameter(arguments, 'account_location.csv', 'location')
	linkLines = almoner.getTextLines(almoner.getLocationText(linkFileName))[1 :]
	accountLines = ['']
	for linkLine in linkLines:
		linkLineSplit = linkLine.split(',')
		name = linkLineSplit[0]
		location = linkLineSplit[1]
		extraLines = []
		if '_xx' in location:
			location = location.replace('_xx', '_' + suffixNumberString)
			extraLines = almoner.getTextLines(almoner.getLocationText(location))
		else:
			extraLines = almoner.getNameAddressLines(location)
		numberOfShares = len(getReceiverLinesByAccountLines(extraLines))
		print('Name: %s, Location: %s, Shares: %s' % (name, location, numberOfShares))
		accountLines.append(name + ': %s Shares' % numberOfShares)
		accountLines += extraLines
		accountLines.append('')
	addAdministratorBonus(accountLines)
	print('')
	return accountLines
コード例 #30
0
ファイル: devtome.py プロジェクト: devcoin/accounting
def writeOutput(arguments):
	'Write output.'
	if '-h' in arguments or '-help' in arguments:
		print(__doc__)
		return
	advertisingRevenue = int(almoner.getParameter(arguments, '0', 'revenue'))
	round = int(almoner.getParameter(arguments, '23', 'round'))
	advertisingFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
	rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
	currentFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % round, 'current')
	previousFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % (round - 1), 'previous')
	ratingFileName = almoner.getParameter(arguments, 'rating_%s.csv' % round, 'rating')
	viewFileName = almoner.getParameter(arguments, 'devtome_analytics_%s.csv' % round, 'view')
	categoryDictionary = {}
	nolines = almoner.getTextLines(almoner.getFileText(previousFileName))
	lines = [nolines[0]]
	titleLine = lines[0]
	titles = titleLine.split(',')
	backupFolder = rootFileName + '_articles'
	ratingDictionary = getRatingDictionary(ratingFileName)
	viewDictionary = getViewDictionary(viewFileName)
	authors = getAuthors(backupFolder, categoryDictionary, lines, ratingDictionary, titles, viewDictionary)
	totalTomecount = getTotalTomecount(advertisingRevenue, authors)
	tomecountText = getTomecountText(authors, totalTomecount)
	advertisingRevenueText = getAdvertisingRevenueText(authors)
	earningsText = getEarningsText(authors)
	activeWritersText = getActiveWritersText(authors, round)
	newArticlesText = getNewArticlesText(authors, round)
	warningsText = getWarningsText(authors)
	outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt', 'summary')
	almoner.writeFileText(currentFileName, tomecountText)
	outputActiveWritersTo = almoner.getParameter(arguments, 'devtome_active_writers.txt', 'writers')
	outputAdvertisingRevenueTo = almoner.getParameter(arguments, 'devtome_advertising_revenue.csv', 'advertising')
	outputEarningsTo = almoner.getParameter(arguments, 'devtome_earnings_%s.csv' % round, 'earnings')
	outputNewArticlesTo = almoner.getParameter(arguments, 'devtome_new_articles.txt', 'articles')
	outputWarningsTo = almoner.getParameter(arguments, 'devtome_warnings.txt', 'warnings')
	writeCategoryFiles(categoryDictionary, rootFileName)
	if advertisingRevenue > 0:
		if almoner.sendOutputTo(outputAdvertisingRevenueTo, advertisingRevenueText):
			print('The devtome advertising revenue file has been written to:\n%s\n' % outputAdvertisingRevenueTo)
	if almoner.sendOutputTo(outputEarningsTo, earningsText):
		print('The devtome earnings file has been written to:\n%s\n' % outputEarningsTo)
	if almoner.sendOutputTo(outputNewArticlesTo, newArticlesText):
		print('The devtome new articles file has been written to:\n%s\n' % outputNewArticlesTo)
	if almoner.sendOutputTo(outputActiveWritersTo, activeWritersText):
		print('The devtome active writers file has been written to:\n%s\n' % outputActiveWritersTo)
	if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, round, totalTomecount)):
		print('The summary file has been written to:\n%s\n' % outputSummaryTo)
	if almoner.sendOutputTo(outputWarningsTo, warningsText):
		print('The devtome warnings file has been written to:\n%s\n' % outputWarningsTo)
コード例 #31
0
ファイル: devtome.py プロジェクト: devcoin/accounting
def getRatingDictionary(ratingFileName):
	'Get the rating median vote dictionary.'
	lines = almoner.getTextLines(almoner.getFileText(ratingFileName))
	if len(lines) < 2:
		return {}
	ratingDictionary = {}
	ratingMedianIndex = getRatingMedianIndex(lines[0])
	for line in lines[1 :]:
		words = line.split(',')
		if len(words) > ratingMedianIndex:
			name = words[0].strip().lower()
			if len(name) > 0:
				ratingDictionary[name] = float(words[ratingMedianIndex].strip().lower())
	return ratingDictionary
コード例 #32
0
ファイル: account.py プロジェクト: moozilla/charity
def getAccountLines(arguments, suffixNumberString):
	'Get the lines according to the arguments.'
	linkFileName = almoner.getParameter(arguments, 'account_location.csv', 'location')
	linkLines = almoner.getTextLines(almoner.getLocationText(linkFileName))[1 :]
	accountLines = ['']
	for linkLine in linkLines:
		linkLineSplit = linkLine.split(',')
		name = linkLineSplit[0]
		location = linkLineSplit[1]
		extraLines = []
		if '_xx' in location:
			location = location.replace('_xx', '_' + suffixNumberString)
			extraLines = almoner.getTextLines(almoner.getLocationText(location))
		else:
			extraLines = almoner.getNameAddressLines(location)
		numberOfShares = len(getReceiverLinesByAccountLines(extraLines))
		print('Name: %s, Location: %s, Shares: %s' % (name, location, numberOfShares))
		accountLines.append(name + ': %s Shares' % numberOfShares)
		accountLines += extraLines
		accountLines.append('')
	addAdministratorBonus(accountLines)
	print('')
	return accountLines
コード例 #33
0
ファイル: devtome.py プロジェクト: Unthinkingbit/charity
def getRatingDictionary(ratingFileName):
	'Get the rating median vote dictionary.'
	lines = almoner.getTextLines(almoner.getFileText(ratingFileName))
	if len(lines) < 2:
		return {}
	ratingDictionary = {}
	ratingMedianIndex = getRatingMedianIndex(lines[0])
	for line in lines[1 :]:
		words = line.split(',')
		if len(words) > ratingMedianIndex:
			name = words[0].strip().lower()
			if len(name) > 0:
				ratingDictionary[name] = float(words[ratingMedianIndex].strip().lower())
	return ratingDictionary
コード例 #34
0
ファイル: devtome.py プロジェクト: Unthinkingbit/charity
def writeOutput(arguments):
	'Write output.'
	if '-h' in arguments or '-help' in arguments:
		print(__doc__)
		return
	advertisingRevenue = int(almoner.getParameter(arguments, '0', 'revenue'))
	round = int(almoner.getParameter(arguments, '23', 'round'))
	advertisingFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
	rootFileName = almoner.getParameter(arguments, 'devtome', 'wiki')
	currentFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % round, 'current')
	previousFileName = almoner.getParameter(arguments, rootFileName + '_%s.csv' % (round - 1), 'previous')
	ratingFileName = almoner.getParameter(arguments, 'rating_%s.csv' % round, 'rating')
	viewFileName = almoner.getParameter(arguments, 'devtome_analytics_%s.csv' % round, 'view')
	categoryDictionary = {}
	lines = almoner.getTextLines(almoner.getFileText(previousFileName))
	titleLine = lines[0]
	titles = titleLine.split(',')
	backupFolder = rootFileName + '_articles'
	ratingDictionary = getRatingDictionary(ratingFileName)
	viewDictionary = getViewDictionary(viewFileName)
	authors = getAuthors(backupFolder, categoryDictionary, lines, ratingDictionary, titles, viewDictionary)
	totalTomecount = getTotalTomecount(advertisingRevenue, authors)
	tomecountText = getTomecountText(authors, totalTomecount)
	advertisingRevenueText = getAdvertisingRevenueText(authors)
	earningsText = getEarningsText(authors)
	activeWritersText = getActiveWritersText(authors, round)
	newArticlesText = getNewArticlesText(authors, round)
	warningsText = getWarningsText(authors)
	outputSummaryTo = almoner.getParameter(arguments, 'devtome_summary.txt', 'summary')
	almoner.writeFileText(currentFileName, tomecountText)
	outputActiveWritersTo = almoner.getParameter(arguments, 'devtome_active_writers.txt', 'writers')
	outputAdvertisingRevenueTo = almoner.getParameter(arguments, 'devtome_advertising_revenue.csv', 'advertising')
	outputEarningsTo = almoner.getParameter(arguments, 'devtome_earnings_%s.csv' % round, 'earnings')
	outputNewArticlesTo = almoner.getParameter(arguments, 'devtome_new_articles.txt', 'articles')
	outputWarningsTo = almoner.getParameter(arguments, 'devtome_warnings.txt', 'warnings')
	writeCategoryFiles(categoryDictionary, rootFileName)
	if advertisingRevenue > 0:
		if almoner.sendOutputTo(outputAdvertisingRevenueTo, advertisingRevenueText):
			print('The devtome advertising revenue file has been written to:\n%s\n' % outputAdvertisingRevenueTo)
	if almoner.sendOutputTo(outputEarningsTo, earningsText):
		print('The devtome earnings file has been written to:\n%s\n' % outputEarningsTo)
	if almoner.sendOutputTo(outputNewArticlesTo, newArticlesText):
		print('The devtome new articles file has been written to:\n%s\n' % outputNewArticlesTo)
	if almoner.sendOutputTo(outputActiveWritersTo, activeWritersText):
		print('The devtome active writers file has been written to:\n%s\n' % outputActiveWritersTo)
	if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, round, totalTomecount)):
		print('The summary file has been written to:\n%s\n' % outputSummaryTo)
	if almoner.sendOutputTo(outputWarningsTo, warningsText):
		print('The devtome warnings file has been written to:\n%s\n' % outputWarningsTo)
コード例 #35
0
ファイル: marketing.py プロジェクト: jdbtracker/charity
def writeOutput(arguments):
	'Write output.'
	if '-h' in arguments or '-help' in arguments:
		print(__doc__)
		return
	round = int(almoner.getParameter(arguments, '23', 'round'))
	publishersFileName = almoner.getParameter(arguments, 'devtome_%s.csv' % round, 'publishers')
	lines = almoner.getTextLines(almoner.getFileText(publishersFileName))
	outputEarningsTo = almoner.getParameter(arguments, 'marketing_earnings_%s.csv' % round, 'earnings')
	outputSummaryTo = almoner.getParameter(arguments, 'marketing_summary.txt', 'summary')
	publishers = getPublishers(lines, round)
	earningsText = getEarningsText(publishers)
	if almoner.sendOutputTo(outputEarningsTo, earningsText):
		print('The marketing earnings bounty file has been written to:\n%s\n' % outputEarningsTo)
	if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, publishers, round)):
		print('The summary file has been written to:\n%s\n' % outputSummaryTo)
コード例 #36
0
def writeOutput(arguments):
	'Write output.'
	if '-h' in arguments or '-help' in arguments:
		print(__doc__)
		return
	round = int(almoner.getParameter(arguments, '23', 'round'))
	publishersFileName = almoner.getParameter(arguments, 'devtome_%s.csv' % round, 'publishers')
	lines = almoner.getTextLines(almoner.getFileText(publishersFileName))
	outputEarningsTo = almoner.getParameter(arguments, 'marketing_earnings_%s.csv' % round, 'earnings')
	outputSummaryTo = almoner.getParameter(arguments, 'marketing_summary.txt', 'summary')
	workerNameSet = set(account.getRecipientDictionary(round).keys())
	publishers = getPublishers(lines, workerNameSet)
	earningsText = getEarningsText(publishers)
	if almoner.sendOutputTo(outputEarningsTo, earningsText):
		print('The marketing earnings bounty file has been written to:\n%s\n' % outputEarningsTo)
	if almoner.sendOutputTo(outputSummaryTo, getSummaryText(earningsText, publishers, round)):
		print('The summary file has been written to:\n%s\n' % outputSummaryTo)
コード例 #37
0
def getRatingsByAddress(address):
	'Get the ratings by address.'
	ratings = []
	firstUnderscore = address.find('_')
	if firstUnderscore == -1:
		print('Warning, no underscore in address.')
		return []
	lastUnderscore = address.rfind('_')
	if firstUnderscore == lastUnderscore:
		print('Warning, firstUnderscore same as lastUnderscore.')
		return []
	rater = address[firstUnderscore + 1 : lastUnderscore].lower()
	lines = almoner.getTextLines(almoner.getSourceText(address))
	for line in lines:
		rating = Rating(address, line, rater)
		if rating.article != '':
			ratings.append(rating)
	return ratings
コード例 #38
0
ファイル: rater.py プロジェクト: FuzzyBearBTC/charity
def getWriterNames(writerFileName):
	'Get the writer names.'
	writerNames = []
	lines = almoner.getTextLines(almoner.getFileText(writerFileName))
	if lines[0].startswith('Create:'):
		isWriterName = False
		for line in lines[1 :]:
			if isWriterName:
				writerNames.append(line.strip().lower())
			if line.startswith('Writer names:'):
				isWriterName = True
		return writerNames
	for line in lines[1 :]:
		words = line.split(',')
		if len(words) > 0:
			name = words[0].strip().lower()
			if name != '':
				writerNames.append(name)
	return writerNames
コード例 #39
0
def getWeightedWordCounts(lastRound, start):
	'Get the weighted word counts.'
	weightedWordCounts = []
	for round in xrange(start, lastRound):
		devtomeFileName = 'devtome_%s.csv' % round
		if not os.path.exists(devtomeFileName):
			return weightedWordCounts
		devtomeText = almoner.getFileText(devtomeFileName)
		if devtomeText == '':
			return weightedWordCounts
		weightedWordCountIndex = -1
		for line in almoner.getTextLines(devtomeText):
			words = line.strip().split(',')
			if len(words) > 1:
				if weightedWordCountIndex != -1:
					weightedWordCounts.append(int(words[weightedWordCountIndex]))
					weightedWordCountIndex = -1
				if words[0].strip() == '' and words[1].strip() == 'Totals':
					weightedWordCountIndex = words.index('Weighted Word Count')
	return weightedWordCounts
コード例 #40
0
ファイル: rater.py プロジェクト: FuzzyBearBTC/charity
def getArticles(name):
	'Get invoiced articles by name.'
	articles = []
	sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % name
	print('Loading user page from %s' % name)
	sourceText = almoner.getSourceText(sourceAddress)
	isArticle = False
	for line in almoner.getTextLines(sourceText):
		lineStrippedLower = line.strip().lower()
		if '==' in lineStrippedLower:
			if '===' not in lineStrippedLower:
				isArticle = False
		if isArticle:
			lowerLinkName = devtome.getLinkName(line, name).lower()
			if lowerLinkName != '':
				articles.append(lowerLinkName)
		if '==' in lineStrippedLower:
			if 'collated' in lineStrippedLower or 'original' in lineStrippedLower:
				isArticle = True
	return articles
コード例 #41
0
ファイル: wikicopy.py プロジェクト: moozilla/charity
def getTitles(wikiAddress):
    'Get all titles of the dokuwiki.'
    indexDepth = 0
    popularPageAddress = wikiAddress + '/doku.php?id=start&idx=wiki%3Auser'
    lines = almoner.getTextLines(almoner.getInternetText(popularPageAddress))
    prefix = '?id='
    prefixLength = len(prefix)
    titles = []
    for line in lines:
        if line.startswith('</ul>'):
            if indexDepth > 0:
                indexDepth -= 1
        if indexDepth > 0 and 'class="wikilink1"' in line:
            prefixIndex = line.find(prefix) + prefixLength
            title = line[prefixIndex:]
            quoteIndex = title.find('"')
            if len(title) > 0:
                titles.append(title[:quoteIndex])
        if line == '<ul class="idx">':
            indexDepth += 1
    return titles
コード例 #42
0
def getWeightedWordCounts(lastRound, start):
    'Get the weighted word counts.'
    weightedWordCounts = []
    for round in xrange(start, lastRound):
        devtomeFileName = 'devtome_%s.csv' % round
        if not os.path.exists(devtomeFileName):
            return weightedWordCounts
        devtomeText = almoner.getFileText(devtomeFileName)
        if devtomeText == '':
            return weightedWordCounts
        weightedWordCountIndex = -1
        for line in almoner.getTextLines(devtomeText):
            words = line.strip().split(',')
            if len(words) > 1:
                if weightedWordCountIndex != -1:
                    weightedWordCounts.append(
                        int(words[weightedWordCountIndex]))
                    weightedWordCountIndex = -1
                if words[0].strip() == '' and words[1].strip() == 'Totals':
                    weightedWordCountIndex = words.index('Weighted Word Count')
    return weightedWordCounts
コード例 #43
0
ファイル: wikicopy.py プロジェクト: jdbtracker/charity
def getTitles(wikiAddress):
	'Get all titles of the dokuwiki.'
	indexDepth = 0
	popularPageAddress = wikiAddress + '/doku.php?id=start&idx=wiki%3Auser'
	lines = almoner.getTextLines(almoner.getInternetText(popularPageAddress))
	prefix = '?id='
	prefixLength = len(prefix)
	titles = []
	for line in lines:
		if line.startswith('</ul>'):
			if indexDepth > 0:
				indexDepth -= 1
		if indexDepth > 0 and 'class="wikilink1"' in line:
			prefixIndex = line.find(prefix) + prefixLength
			title = line[prefixIndex :]
			quoteIndex = title.find('"')
			if len(title) > 0:
				titles.append(title[: quoteIndex])
		if line == '<ul class="idx">':
			indexDepth += 1
	return titles
コード例 #44
0
ファイル: peer.py プロジェクト: FuzzyBearBTC/charity
def main():
    'Replace peers.'
    beginPeerString = '_beginpeers\n'
    endPeerString = '_endpeers\n'
    receiverIndex = 0
    receiverFileName = getReceiverFileName(receiverIndex)
    receiverText = almoner.getFileText(receiverFileName)
    peerLines = almoner.getTextLines(almoner.getFileText('peer.csv'))
    replacementString = beginPeerString + almoner.getTextByLines(
        peerLines) + endPeerString
    while receiverText != '':
        beginPeerIndex = receiverText.find(beginPeerString)
        endPeerEndIndex = receiverText.find(endPeerString) + len(endPeerString)
        betweenString = receiverText[beginPeerIndex:endPeerEndIndex]
        almoner.writeFileText('backup_receiver_%s.csv' % receiverIndex,
                              receiverText)
        replacedText = receiverText.replace(betweenString, replacementString)
        almoner.writeFileText(receiverFileName, replacedText)
        print('Replaced peers in file: ' + receiverFileName)
        receiverIndex += 1
        receiverFileName = getReceiverFileName(receiverIndex)
        receiverText = almoner.getFileText(receiverFileName, False)
コード例 #45
0
 def __init__(self, backupFolder, titles, words):
     'Initialize.'
     self.tomecount = Tomecount()
     self.parameterDictionary = {}
     for wordIndex, word in enumerate(words):
         self.parameterDictionary[titles[wordIndex]] = word
     if 'Cumulative Payout' in self.parameterDictionary:
         self.tomecount.previousPayout = int(
             self.parameterDictionary['Cumulative Payout'])
     name = self.parameterDictionary['Name']
     self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % name
     print('Loading articles from %s' % name)
     sourceText = getSourceText(self.sourceAddress)
     almoner.writeFileText(os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower:
             isCollated = False
             isOriginal = False
             if 'collated' in lineStrippedLower:
                 isCollated = True
             elif 'original' in lineStrippedLower:
                 isOriginal = True
         if isCollated:
             linkText = getLinkText(lineStrippedLower)
             self.tomecount.imageCount += getImageCount(linkText)
             wordCount = getWordCount(linkText)
             self.tomecount.collatedWordCount += wordCount
             if wordCount > 0:
                 print('Collated article: %s, Word Count: %s' %
                       (lineStrippedLower,
                        almoner.getCommaNumberString(wordCount)))
                 almoner.writeFileText(
                     os.path.join(backupFolder,
                                  getLinkName(lineStrippedLower)[1:]),
                     linkText)
         if isOriginal:
             linkText = getLinkText(lineStrippedLower)
             self.tomecount.imageCount += getImageCount(linkText)
             wordCount = getWordCount(linkText)
             self.tomecount.originalWordCount += wordCount
             if wordCount > 0:
                 print('Original article: %s, Word Count: %s' %
                       (lineStrippedLower,
                        almoner.getCommaNumberString(wordCount)))
                 almoner.writeFileText(
                     os.path.join(backupFolder,
                                  getLinkName(lineStrippedLower)[1:]),
                     linkText)
     self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10
     self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount
     self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount
     self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount
     if self.tomecount.weightedWordCount >= 1000:
         self.tomecount.cumulativePayout = int(
             round(float(self.tomecount.weightedWordCount) * 0.001))
     print('Weighted Word Count: %s' %
           almoner.getCommaNumberString(self.tomecount.weightedWordCount))
     self.tomecount.payout = max(
         self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0)
     maximumPayout = 80
     if self.tomecount.payout > maximumPayout:
         self.tomecount.payout = maximumPayout
         self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout
コード例 #46
0
ファイル: devtome.py プロジェクト: chefatef22/charity
 def __init__(self, averageRating, backupFolder, backupFileSet,
              categoryDictionary, ratingDictionary, titles, viewDictionary,
              words):
     'Initialize.'
     self.backupFolder = backupFolder
     self.backupFileSet = backupFileSet
     identicalCollatedCount = 0
     identicalOriginalCount = 0
     self.newArticles = []
     self.tomecount = Tomecount()
     self.parameterDictionary = {}
     self.sentenceSet = set([])
     self.warnings = []
     for wordIndex, word in enumerate(words):
         self.parameterDictionary[titles[wordIndex]] = word
     if 'Cumulative Payout' in self.parameterDictionary:
         self.tomecount.previousPayout = int(
             self.parameterDictionary['Cumulative Payout'])
     self.name = self.parameterDictionary['Name']
     self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name
     tipAddress = ''
     print('Loading articles from %s' % self.name)
     sourceText = almoner.getSourceText(self.sourceAddress)
     almoner.writeFileText(
         os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower:
             if '===' not in lineStrippedLower:
                 isCollated = False
                 isOriginal = False
                 isTip = False
         if isCollated:
             linkName = getLinkName(line, self.name)
             underscoredLinkName = linkName.lower().replace(' ', '_')
             linkText = getSourceTextIfByAuthor(self, linkName)
             if linkName != '' and linkText == '':
                 self.printWarning(
                     'Warning, could not invoice article link: %s' %
                     linkName)
             if linkText not in linkTexts:
                 linkTexts.add(linkText)
                 self.tomecount.imageCount += getImageCount(linkText)
                 wordCount = getWordCount(linkText)
                 if underscoredLinkName in viewDictionary:
                     self.tomecount.pageViews += viewDictionary[
                         underscoredLinkName]
                 if wordCount > 0:
                     print('Collated article: %s, Word Count: %s' %
                           (lineStrippedLower,
                            almoner.getCommaNumberString(wordCount)))
                     self.saveArticle(categoryDictionary, linkName,
                                      linkText, underscoredLinkName)
                     identicalCollatedCount += self.getIdenticalWordCount(
                         linkText)
                     self.tomecount.collatedWordCount += wordCount
         if isOriginal:
             linkName = getLinkName(line, self.name)
             underscoredLinkName = linkName.lower().replace(' ', '_')
             linkText = getSourceTextIfByAuthor(self, linkName)
             if linkName != '' and linkText == '':
                 self.printWarning(
                     'Warning, could not invoice article link: %s' %
                     linkName)
             if linkText not in linkTexts:
                 linkTexts.add(linkText)
                 self.tomecount.imageCount += getImageCount(linkText)
                 wordCount = getWordCount(linkText)
                 if underscoredLinkName in viewDictionary:
                     self.tomecount.pageViews += viewDictionary[
                         underscoredLinkName]
                 if wordCount > 0:
                     print('Original article: %s, Word Count: %s' %
                           (lineStrippedLower,
                            almoner.getCommaNumberString(wordCount)))
                     self.saveArticle(categoryDictionary, linkName,
                                      linkText, underscoredLinkName)
                     identicalOriginalCount += self.getIdenticalWordCount(
                         linkText)
                     self.tomecount.originalWordCount += wordCount
         if isTip:
             tipLine = line.strip().replace("'", '')
             colonIndex = tipLine.find(':')
             if colonIndex >= 0:
                 addressName = tipLine[:colonIndex].strip().lower()
                 if 'dvc' in addressName or 'devcoin' in addressName or 'coin address' in addressName:
                     tipAddress = tipLine[colonIndex + 1:].strip()
         if '==' in lineStrippedLower:
             if 'collated' in lineStrippedLower:
                 isCollated = True
             elif 'original' in lineStrippedLower:
                 isOriginal = True
             elif 'tip' in lineStrippedLower:
                 isTip = True
     if identicalCollatedCount > 0:
         self.tomecount.collatedWeightedWordCount -= identicalCollatedCount
         print('Identical Collated Word Count: %s' %
               almoner.getCommaNumberString(identicalCollatedCount))
     if identicalOriginalCount > 0:
         self.tomecount.originalWordCount -= identicalOriginalCount
         print('Identical Original Word Count: %s' %
               almoner.getCommaNumberString(identicalOriginalCount))
     self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10
     self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount
     self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount
     self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount
     if self.tomecount.weightedWordCount >= 1000:
         self.tomecount.cumulativePayout = int(
             round(float(self.tomecount.weightedWordCount) * 0.001))
     print('Weighted Word Count: %s' %
           almoner.getCommaNumberString(self.tomecount.weightedWordCount))
     self.tomecount.payout = max(
         self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0)
     maximumPayout = 50
     if tipAddress != self.parameterDictionary[
             'Coin Address'] and self.name != 'Mosinnagant':
         self.printWarning(
             'Warning, the coin address is not the same as the tip address, so nothing will be paid.'
         )
         maximumPayout = 0
     if self.tomecount.payout > maximumPayout:
         self.tomecount.payout = maximumPayout
         self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout
     if self.tomecount.cumulativePayout > 0:
         self.tomecount.categorization = float(
             self.tomecount.categorizedArticleCount) / float(
                 self.tomecount.articleCount)
         self.tomecount.ratingMedian = averageRating
         lowerName = self.name.lower()
         if lowerName in ratingDictionary:
             self.tomecount.ratingMedian = ratingDictionary[lowerName]
         weightedPageViews = self.tomecount.pageViews
         if self.tomecount.previousPayout == 0:
             weightedPageViews += weightedPageViews
         self.tomecount.viewsPerThousandWords = 1000.0 * float(
             weightedPageViews) / float(self.tomecount.weightedWordCount)
         self.tomecount.normalizedCategorization = self.tomecount.categorization
         self.tomecount.normalizedPopularity = self.tomecount.viewsPerThousandWords
         self.tomecount.normalizedRatingMedian = self.tomecount.ratingMedian
         self.tomecount.popularityTimesRating = int(
             round(self.tomecount.pageViews *
                   float(self.tomecount.ratingMedian) / 99.0))
コード例 #47
0
ファイル: devtome.py プロジェクト: chefatef22/charity
def writeCategoryFile(categoryDictionary, categoryFolder, categoryKey,
                      rootFileName):
    'Write category file to a folder.'
    categorySuffix = 'category:' + categoryKey
    categoryFileName = os.path.join(categoryFolder, categorySuffix)
    sourceText = almoner.getSourceText(
        'http://devtome.com/doku.php?id=%s&do=edit' % categorySuffix).replace(
            '&quot;', '"')
    scriptToken = '{{script}}'
    scriptIndex = sourceText.find(scriptToken)
    if scriptIndex == -1:
        return
    scriptIndex += len(scriptToken)
    categoryText = sourceText[:scriptIndex] + '\n'
    afterScriptText = sourceText[scriptIndex:]
    lastLetter = None
    lines = almoner.getTextLines(afterScriptText)
    isAlphabeticallyGrouped = False
    scriptEndToken = None
    titleDictionary = {}
    for line in lines:
        if scriptEndToken == None:
            lineStripped = line.strip()
            if lineStripped != '':
                if lineStripped.startswith('=') and lineStripped.endswith('='):
                    heading = lineStripped.replace('=', '').strip()
                    if len(heading) > 1:
                        scriptEndToken = lineStripped
                    elif len(heading) == 1:
                        isAlphabeticallyGrouped = True
                else:
                    if lineStripped.startswith('*'):
                        lineStripped = lineStripped[1:]
                    if lineStripped.startswith('[['):
                        lineStripped = lineStripped[2:]
                    if lineStripped.startswith(':'):
                        lineStripped = lineStripped[1:]
                    if lineStripped.endswith(']]'):
                        lineStripped = lineStripped[:-2]
                    titleKey = lineStripped.lower().replace('_', ' ')
                    barIndex = titleKey.find('|')
                    if barIndex != -1:
                        titleKey = titleKey[:barIndex]
                    titleDictionary[titleKey] = lineStripped
    fromTokenText = ''
    if scriptEndToken != None:
        fromTokenText = afterScriptText[afterScriptText.find(scriptEndToken):]
    articleTitles = categoryDictionary[categoryKey]
    for articleTitle in articleTitles:
        articleTitleLower = articleTitle.lower().replace('_', ' ')
        if articleTitleLower not in titleDictionary:
            titleDictionary[articleTitleLower] = articleTitle
    titleKeys = titleDictionary.keys()
    titleKeys.sort()
    for titleKey in titleKeys:
        if isAlphabeticallyGrouped:
            firstLetter = titleKey[0]
            if firstLetter != lastLetter:
                categoryText += '===%s===\n' % firstLetter.capitalize()
                lastLetter = firstLetter
        title = titleDictionary[titleKey]
        if not ']]' in title:
            title += ']]'
        categoryText += '[[:%s\n\n' % title
    categoryText += fromTokenText
    almoner.writeFileText(os.path.join(categoryFolder, categorySuffix),
                          categoryText)
コード例 #48
0
ファイル: wikicopy.py プロジェクト: moozilla/charity
def getRecentTitles(archiveType, fileNameRoot, wikiAddress):
    'Get all titles of the dokuwiki.'
    archiveFileName = fileNameRoot + '.' + archiveType
    if not os.path.exists(archiveFileName):
        return getTitles(wikiAddress)
    if archiveType == 'zip':
        zipArchive = zipfile.ZipFile(archiveFileName, 'r')
        zipArchive.extractall(fileNameRoot)
        zipArchive.close()
    else:
        mode = 'r'
        if archiveType == 'bz2':
            mode = 'r:bz2'
        tarArchive = tarfile.open(archiveFileName, mode)
        tarArchive.extractall(fileNameRoot)
        tarArchive.close()
    lastModifiedText = almoner.getFileText(
        os.path.join(fileNameRoot, 'last_modified.txt'))
    lastModifiedDatetime = datetime.datetime.strptime(lastModifiedText,
                                                      globalDateTimeFormat)
    print('Last modified: %s' % lastModifiedDatetime)
    nowDatetime = datetime.datetime.today()
    nowMinusLast = nowDatetime - lastModifiedDatetime
    print('Now minus last: %s' % nowMinusLast)
    twentySixHours = 26 * 3600
    if getSeconds(nowMinusLast) > (24 * 5 + 22) * 3600:
        return getTitles(wikiAddress)
    recentPageAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&first[0]'
    lines = almoner.getTextLines(almoner.getInternetText(recentPageAddress))
    lineDatetime = None
    dateTitle = 'class="date">'
    linkTitle = 'class="wikilink1" title="'
    nameTitle = 'name="'
    start = 0
    titleSet = set([])
    while True:
        for lineIndex, line in enumerate(lines):
            if dateTitle in line:
                dateLine = lines[lineIndex + 1]
                dateString = dateLine[:dateLine.find('<')]
                if dateString.startswith('20'):
                    dateString = dateString[2:]
                lineDatetime = datetime.datetime.strptime(
                    dateString, globalDateTimeFormat)
            if linkTitle in line:
                line = line[line.find(linkTitle) + len(linkTitle):]
                title = line[:line.find('"')]
                if title != 'start':
                    lastMinusLine = lastModifiedDatetime - lineDatetime
                    if title in titleSet or getSeconds(
                            lastMinusLine) > twentySixHours:
                        titles = list(titleSet)
                        titles.sort()
                        return titles
                    titleSet.add(title)
            if line.startswith(
                    '<input'
            ) and 'value="less recent' in line and nameTitle in line:
                line = line[line.find(nameTitle) + len(nameTitle):]
                name = line[:line.find('"')]
                recentPageAddress = wikiAddress + '/doku.php?do=recent&id=start&show_changes=pages&' + name
        lines = almoner.getTextLines(
            almoner.getInternetText(recentPageAddress))
    return getTitles(wikiAddress)
コード例 #49
0
ファイル: devtome.py プロジェクト: jdbtracker/charity
	def __init__(self, backupFolder, titles, viewDictionary, words):
		'Initialize.'
		self.tomecount = Tomecount()
		self.parameterDictionary = {}
		self.warnings = []
		for wordIndex, word in enumerate(words):
			self.parameterDictionary[titles[wordIndex]] = word
		if 'Cumulative Payout' in self.parameterDictionary:
			self.tomecount.previousPayout = int(self.parameterDictionary['Cumulative Payout'])
		self.name = self.parameterDictionary['Name']
		self.sourceAddress = 'http://devtome.com/doku.php?id=wiki:user:%s&do=edit' % self.name
		tipAddress = ''
		print('Loading articles from %s' % self.name)
		sourceText = getSourceText(self.sourceAddress)
		almoner.writeFileText(os.path.join(backupFolder, 'wiki:user:'******'==' in lineStrippedLower:
				isCollated = False
				isOriginal = False
				isTip = False
			if isCollated:
				lowerLinkName = getLinkName(line).lower()
				linkText = getLinkText(lowerLinkName, self.name)
				if linkText not in linkTexts:
					linkTexts.add(linkText)
					self.tomecount.imageCount += getImageCount(linkText)
					wordCount = getWordCount(linkText)
					self.tomecount.collatedWordCount += wordCount
					if lowerLinkName in viewDictionary:
						self.tomecount.pageViews += int(viewDictionary[lowerLinkName])
					if wordCount > 0:
						print('Collated article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount)))
						almoner.writeFileText(os.path.join(backupFolder, lowerLinkName), linkText)
			if isOriginal:
				lowerLinkName = getLinkName(line).lower()
				linkText = getLinkText(lowerLinkName, self.name)
				if lowerLinkName != '' and linkText == '':
					self.printWarning('Warning, could not invoice article link: %s' % lowerLinkName)
				if linkText not in linkTexts:
					linkTexts.add(linkText)
					self.tomecount.imageCount += getImageCount(linkText)
					wordCount = getWordCount(linkText)
					self.tomecount.originalWordCount += wordCount
					if lowerLinkName in viewDictionary:
						self.tomecount.pageViews += int(viewDictionary[lowerLinkName])
					if wordCount > 0:
						print('Original article: %s, Word Count: %s' % (lineStrippedLower, almoner.getCommaNumberString(wordCount)))
						almoner.writeFileText(os.path.join(backupFolder, lowerLinkName), linkText)
			if isTip:
				tipAddress = line.strip()
				if ':' in tipAddress:
					tipAddress = tipAddress[tipAddress.find(':') + 1 :].strip()
			if '==' in lineStrippedLower:
				if 'collated' in lineStrippedLower:
					isCollated = True
				elif 'original' in lineStrippedLower:
					isOriginal = True
				elif 'tip' in lineStrippedLower:
					isTip = True
		self.tomecount.collatedWeightedWordCount = self.tomecount.collatedWordCount * 3 / 10
		self.tomecount.wordCount = self.tomecount.collatedWordCount + self.tomecount.originalWordCount
		self.tomecount.weightedWordCount = self.tomecount.collatedWeightedWordCount + self.tomecount.originalWordCount
		self.tomecount.weightedWordCount += 10 * self.tomecount.imageCount
		if self.tomecount.weightedWordCount >= 1000:
			self.tomecount.cumulativePayout = int(round(float(self.tomecount.weightedWordCount) * 0.001))
		print('Weighted Word Count: %s' % almoner.getCommaNumberString(self.tomecount.weightedWordCount))
		self.tomecount.payout = max(self.tomecount.cumulativePayout - self.tomecount.previousPayout, 0)
		maximumPayout = 80
		if tipAddress != self.parameterDictionary['Coin Address'] and self.name != 'Mosinnagant':
			self.printWarning('Warning, the coin address is not the same as the tip address, so nothing will be paid.')
			maximumPayout = 0
		if self.tomecount.payout > maximumPayout:
			self.tomecount.payout = maximumPayout
			self.tomecount.cumulativePayout = self.tomecount.previousPayout + maximumPayout
		if self.tomecount.cumulativePayout > 0:
			worthRatio = float(self.tomecount.pageViews) / float(self.tomecount.weightedWordCount)
			self.tomecount.normalizedRootWorth = math.sqrt(math.sqrt(worthRatio))
コード例 #50
0
def getPreviousLines(round):
	'Get the lines from the rating text of the previous round.'
	return almoner.getTextLines(almoner.getFileText('rating_%s.csv' % (round - 1)))