def run(self, revisionTextA, revisionTextB, revisionDict):
gdiff = diff_match_patch()
revisionDiffs = gdiff.diff_main(revisionTextA,
revisionTextB, False)
gdiff.diff_cleanupSemantic(revisionDiffs)
revisionDiffs = filter(self.isRemoveOrAdd, revisionDiffs)
diffWordCount = map(self.countWords, revisionDiffs)
addedWordCount = self.getAddWordCount(diffWordCount)
deletedWordCount = self.getDeletedWordCount(diffWordCount)
revision = Revision(**revisionDict)
revision.wordsAdded = addedWordCount
revision.wordsDeleted = deletedWordCount
revision.wordCount = self.getWordCount(revisionTextB)
return revision.to_dict()
def is_new(self):
# today = datetime.date.today()
# delta = datetime.timedelta(days=settings.BOT_CONSIDERED_NEW)
# result = today - self.date_added <= delta
# return result
return self.revision >= Revision.get_instance(
).nr - settings.BOT_CONSIDERED_NEW + 1
def many_by_usernames(names: List):
results = Bot.select(
).where((fn.lower(Bot.username) << [n.lower() for n in names]) & (
Bot.revision <= Revision.get_instance().nr & Bot.approved == True))
if len(results) > 0:
return results
else:
raise Bot.DoesNotExist()
def main(cli_args):
client = TreeherderClient(cli_args.repo, cli_args.csrf_token, cli_args.session_id)
revision = Revision(client, cli_args.revision)
results = revision.results
jobs = [result.get_jobs_for(cli_args.job_type, job_number=cli_args.job_number, job_platform=cli_args.job_platform) for result in results]
jobs = _flatten_list(jobs)
for job in jobs:
for i in range(1, cli_args.repeat + 1):
print('Repeat #{}'.format(i))
job.__getattribute__(cli_args.command)()
def update_categories(self, categories: List):
self.notify_admin(
"Updating BotList categories to Revision {}...".format(
Revision.get_instance().nr))
for cat in categories:
text = _format_category_bots(cat)
msg = self.send_or_edit(text, cat.current_message_id)
if msg:
cat.current_message_id = msg.message_id
self.sent['category'].append("{} {}".format(
'Resent' if self.resend else 'Updated', cat))
cat.save()
self._save_channel()
# Add "share", "up", and "down" buttons
for i in range(0, len(categories)):
buttons = list()
if i > 0:
# Not first category
# Add "Up" button
buttons.append(
InlineKeyboardButton(
"🔺",
url=BotList.create_hyperlink(
categories[i - 1].current_message_id)))
buttons.append(
InlineKeyboardButton("Share",
url="https://t.me/{}?start={}".format(
settings.SELF_BOT_NAME,
categories[i].id)))
if i < len(categories) - 1:
# Not last category
buttons.append(
InlineKeyboardButton(
"🔻",
url=BotList.create_hyperlink(
categories[i + 1].current_message_id)))
reply_markup = InlineKeyboardMarkup([buttons])
self.bot.edit_message_reply_markup(
self.channel.chat_id,
categories[i].current_message_id,
reply_markup=reply_markup,
timeout=60)
def search_bots(query):
query = query.lower().strip()
split = query.split(' ')
# easter egg
if query in ('awesome bot', 'great bot', 'superb bot', 'best bot', 'best bot ever'):
return [Bot.by_username('@botlistbot')]
# exact results
where_query = (
(fn.lower(Bot.username).contains(query) |
fn.lower(Bot.name) << split |
fn.lower(Bot.extra) ** query) &
(Bot.revision <= Revision.get_instance().nr &
Bot.approved == True)
)
results = set(Bot.select().distinct().where(where_query))
# keyword results
keyword_results = Bot.select(Bot).join(Keyword).where(
(fn.lower(Keyword.name) << split) &
(Bot.revision <= Revision.get_instance().nr) &
(Bot.approved == True)
)
results.update(keyword_results)
# many @usernames
usernames = re.findall(settings.REGEX_BOT_ONLY, query)
if usernames:
try:
bots = Bot.many_by_usernames(usernames)
results.update(bots)
except Bot.DoesNotExist:
pass
return list(results)
def run(self, toEmail, documentId, credentialsAsJson):
credentials = OAuth2Credentials.from_json(credentialsAsJson)
http = credentials.authorize(httplib2.Http())
request = config.getService().revisions().list(fileId=documentId)
response = request.execute(http=http)
revisions = []
for item in response.get('items'):
revision = Revision()
revision.documentId = documentId
revision.revisionNumber = item.get('id')
revision.author = item.get('lastModifyingUserName')
revision.exportLink = item.get('exportLinks').get('text/plain')
revision.dateTime = item.get('modifiedDate')
revisions.append(revision.to_dict())
return revisions
def testRun(self):
pipeline = RevisionsDiffPipeline('Hello', 'Hello there', {})
pipeline.start_test()
result = pipeline.outputs.default.value
expected = Revision()
expected.wordsAdded = 1
expected.wordsDeleted = 0
expected.wordCount = 2
self.assertEqual(result, expected.to_dict())
def run(self, toEmail, documentId, credentialsAsJson):
credentials = OAuth2Credentials.from_json(credentialsAsJson)
http = credentials.authorize(httplib2.Http())
request = config.getService().revisions().list(fileId=documentId)
response = request.execute(http=http)
revisions = []
for item in response.get('items'):
revision = Revision()
revision.documentId = documentId
revision.revisionNumber = item.get('id')
revision.author = item.get('lastModifyingUserName')
revision.exportLink = item.get('exportLinks').get('text/plain')
revision.dateTime = item.get('modifiedDate')
revisions.append(revision.to_dict())
return revisions
def run(self, revisionTextA, revisionTextB, revisionDict):
gdiff = diff_match_patch()
revisionDiffs = gdiff.diff_main(revisionTextA, revisionTextB, False)
gdiff.diff_cleanupSemantic(revisionDiffs)
revisionDiffs = filter(self.isRemoveOrAdd, revisionDiffs)
diffWordCount = map(self.countWords, revisionDiffs)
addedWordCount = self.getAddWordCount(diffWordCount)
deletedWordCount = self.getDeletedWordCount(diffWordCount)
revision = Revision(**revisionDict)
revision.wordsAdded = addedWordCount
revision.wordsDeleted = deletedWordCount
revision.wordCount = self.getWordCount(revisionTextB)
return revision.to_dict()
def send_botlist(bot, update, resend=False, silent=False):
log.info("Re-sending BotList..." if resend else "Updating BotList...")
channel = helpers.get_channel()
revision = Revision.get_instance()
revision.nr += 1
revision.save()
all_categories = Category.select_all()
botlist = BotList(bot, update, channel, resend, silent)
if resend:
botlist.delete_full_botlist()
botlist.update_intro()
botlist.update_categories(all_categories)
botlist.update_new_bots_list()
botlist.update_category_list()
botlist.send_footer()
botlist.finish()
channel.save()
Statistic.of(update, 'send', 'botlist (resend: {})'.format(str(resend)),
Statistic.IMPORTANT)
def run(self, toEmail, subject, body, documentsAnalyses):
# TODO(michaelcupino): Move csv transforms into a service.
csvFile = tempfile.TemporaryFile()
writer = csv.writer(csvFile)
values = [['Date', 'Time', 'Who in doc', 'Word count', 'Words added',
'Words deleted', 'Punct. cap', 'Words moved', 'Document ID',
'Document Name']]
writer.writerows(values)
for documentAnalyses in documentsAnalyses:
for documentAnalysis in documentAnalyses:
revision = Revision(**documentAnalysis)
values = [[
revision.dateTime, # Date
'', # Time
revision.author,
revision.wordCount,
revision.wordsAdded,
revision.wordsDeleted,
'', # Punct. cap
'', # Words moved
revision.documentId, # Document ID
'', # Document Name
]]
writer.writerows(values)
csvFile.seek(0)
csvFileBytes = csvFile.read()
csvFile.close()
message = mail.EmailMessage()
message.sender = '*****@*****.**'
message.to = toEmail
message.subject = subject
message.body = str(body)
message.attachments = [('export.csv', csvFileBytes)]
message.send()
def select_pending_update():
return Bot.select().where(Bot.approved == True,
Bot.revision == Revision.get_instance().next)
def select_approved():
return Bot.select().where(Bot.approved == True,
Bot.revision <= Revision.get_instance().nr)
def select_new_bots():
return Bot.select().where(
(Bot.revision >= Revision.get_instance().nr -
settings.BOT_CONSIDERED_NEW + 1)
& (Bot.revision < Revision.get_instance().next)
& Bot.approved == True)
def of_category_without_new(category):
return Bot.select().where(
Bot.category == category, Bot.approved == True,
Bot.revision <= Revision.get_instance().nr).order_by(
fn.Lower(Bot.username))
def explorable_bots():
results = Bot.select().where(
~(Bot.description.is_null()), Bot.approved == True,
Bot.revision <= Revision.get_instance().nr)
return list(results)
def new_bot_submission(bot, update, chat_data, args=None):
tg_user = update.message.from_user
user = User.from_telegram_object(tg_user)
if util.stop_banned(update, user):
return
reply_to = util.original_reply_id(update)
if args:
text = ' '.join(args)
else:
text = update.message.text
command_no_args = len(
re.findall(r'^/new\s*$',
text)) > 0 or text.lower().strip() == '/new@botlistbot'
if command_no_args:
update.message.reply_text(util.action_hint(
"Please use this command with an argument. For example:\n/new @mybot 🔎"
),
reply_to_message_id=reply_to)
return
# `#new` is already checked by handler
try:
username = re.match(settings.REGEX_BOT_IN_TEXT, text).groups()[0]
if username.lower() == '@' + settings.SELF_BOT_NAME.lower():
log.info("Ignoring {}".format(text))
return
except AttributeError:
if args:
update.message.reply_text(util.failure(
"Sorry, but you didn't send me a bot `@username`."),
quote=True,
parse_mode=ParseMode.MARKDOWN,
reply_to_message_id=reply_to)
log.info("Ignoring {}".format(text))
# no bot username, ignore update
return
try:
new_bot = Bot.by_username(username)
if new_bot.approved:
update.message.reply_text(util.action_hint(
"Sorry fool, but {} is already in the @BotList 😉".format(
new_bot.username)),
reply_to_message_id=reply_to)
else:
update.message.reply_text(util.action_hint(
"{} has already been submitted. Please have patience...".
format(new_bot.username)),
reply_to_message_id=reply_to)
return
except Bot.DoesNotExist:
new_bot = Bot(revision=Revision.get_instance().next,
approved=False,
username=username,
submitted_by=user)
new_bot.inlinequeries = "🔎" in text
new_bot.official = "🔹" in text
# find language
languages = Country.select().execute()
for lang in languages:
if lang.emoji in text:
new_bot.country = lang
new_bot.date_added = datetime.date.today()
description_reg = re.match(settings.REGEX_BOT_IN_TEXT + ' -\s?(.*)', text)
description_notify = ''
if description_reg:
description = description_reg.group(2)
new_bot.description = description
description_notify = ' Your description was included.'
new_bot.save()
if util.is_private_message(update) and util.uid_from_update(
update) in settings.MODERATORS:
from components.explore import send_bot_details
send_bot_details(bot, update, chat_data, new_bot)
else:
msg = update.message.reply_text(util.success(
"You submitted {} for approval.{}".format(new_bot,
description_notify)),
parse_mode=ParseMode.MARKDOWN,
reply_to_message_id=reply_to)
return ConversationHandler.END
def testRun(self, mockUrlOpenMethod):
exportLinks = [
'docs.google.com/123',
'docs.google.com/456',
'docs.google.com/789'
]
mockRevisions = [
Revision(exportLink=exportLinks[0]).to_dict(),
Revision(exportLink=exportLinks[1]).to_dict(),
Revision(exportLink=exportLinks[2]).to_dict()
]
exportedText = {
exportLinks[0]: StringIO('Hello'),
exportLinks[1]: StringIO('Hello world.'),
exportLinks[2]: StringIO('Hello.'),
}
mockUrlOpenMethod.side_effect = lambda url: exportedText[url]
# TODO(michaelcupino): Fix tests
pipeline = RevisionsAnalysisPipeline(mockRevisions, None)
pipeline.start_test()
result = pipeline.outputs.default.value
revision1 = Revision()
revision1.wordsAdded = 1
revision1.wordsDeleted = 0
revision1.wordCount = 1
revision1.exportLink = 'docs.google.com/123'
revision2 = Revision()
revision2.wordsAdded = 1
revision2.wordsDeleted = 0
revision2.wordCount = 2
revision2.exportLink = 'docs.google.com/456'
revision3 = Revision()
revision3.wordsAdded = 0
revision3.wordsDeleted = 1
revision3.wordCount = 1
revision3.exportLink = 'docs.google.com/789'
self.assertEqual([revision1.to_dict(), revision2.to_dict(),
revision3.to_dict()], result)
import os
from decouple import config
from playhouse.migrate import SqliteMigrator, IntegerField, migrate
from playhouse.sqlite_ext import SqliteExtDatabase
from model.revision import Revision
db_path = config('DATABASE_URI',
default=os.path.expanduser('~/botlistbot.sqlite3'))
db = SqliteExtDatabase(db_path)
migrator = SqliteMigrator(db)
revision = IntegerField(default=100)
with db.transaction():
migrate(migrator.add_column('bot', 'revision', revision), )
Revision.create_table(fail_silently=True)
Revision.insert({'nr': 101}).execute()
def testRun(self, mockOAuth2CredentialsFromJsonMethod, mockUrlOpenMethod):
http = HttpMockSequence([
({
'status': '200'
}, open(datafile('test-drive-revisions-list-0.json'),
'rb').read()),
])
mockCredentials = MagicMock(name='mockCredentials')
mockCredentials.authorize = MagicMock(return_value=http)
mockOAuth2CredentialsFromJsonMethod.return_value = mockCredentials
exportLinks = [
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=697&exportFormat=tx'
't'),
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=698&exportFormat=tx'
't'),
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=868&exportFormat=tx'
't'),
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=869&exportFormat=tx'
't'),
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=937&exportFormat=tx'
't'),
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=939&exportFormat=tx'
't'),
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=1035&exportFormat=t'
'xt'),
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=1036&exportFormat=t'
'xt'),
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=1048&exportFormat=t'
'xt'),
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=1049&exportFormat=t'
'xt'),
('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2'
'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=1057&exportFormat=t'
'xt')
]
exportedText = {
exportLinks[0]:
StringIO('Hello'),
exportLinks[1]:
StringIO('Hello world.'),
exportLinks[2]:
StringIO('Hello'),
exportLinks[3]:
StringIO(''),
exportLinks[4]:
StringIO('Hello again, this is a longer sentence'),
exportLinks[5]:
StringIO(('Hello again, this is a longer sentence with '
'another word')),
exportLinks[6]:
StringIO(('Hello again, this is a longer sentence with '
'another word. Here is another sentence')),
exportLinks[7]:
StringIO(
('Hello again, this is a longer sentence with '
'another word. Here is another sentence with another word')),
exportLinks[8]:
StringIO(
('Hello again, this is a longer sentence with '
'another word. Here is another sentence with another word')),
exportLinks[9]:
StringIO(('Hello again, this is a longer sentence with '
'another word. Here is another sentence.')),
exportLinks[10]:
StringIO(('Hello again, this is a longer sentence with '
'another word. Here is another sentence. The end.'))
}
mockUrlOpenMethod.side_effect = lambda url: exportedText[url]
# TODO(michaelcupino): Fix tests
pipeline = DocumentAnalysisPipeline('*****@*****.**', 'abc123', None)
pipeline.start_test()
result = pipeline.outputs.default.value
# TODO(michaelcupino): Updated expected data to match actual result.
expectedResult = []
expectedResult.append(
Revision(
wordsAdded=1,
wordsDeleted=0,
wordCount=1,
author='Jonathan H',
dateTime='2014-03-15T17:49:32.768Z',
exportLink=
('https://docs.google.com/feeds/download/documents/export/Ex'
'port?id=1Duz2yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=697&'
'exportFormat=txt'),
revisionNumber='697',
documentId='abc123').to_dict())
expectedResult.append(Revision(wordsAdded=1, wordsDeleted=0).to_dict())
expectedResult.append(Revision(wordsAdded=0, wordsDeleted=1).to_dict())
expectedResult.append(Revision(wordsAdded=0, wordsDeleted=1).to_dict())
expectedResult.append(Revision(wordsAdded=7, wordsDeleted=0).to_dict())
expectedResult.append(Revision(wordsAdded=3, wordsDeleted=0).to_dict())
expectedResult.append(Revision(wordsAdded=5, wordsDeleted=0).to_dict())
expectedResult.append(Revision(wordsAdded=3, wordsDeleted=0).to_dict())
expectedResult.append(Revision(wordsAdded=0, wordsDeleted=0).to_dict())
# TODO(michaelcupino): I would have expected there to be no words added.
# Figure out why this behavior occurrs.
expectedResult.append(Revision(wordsAdded=1, wordsDeleted=3).to_dict())
expectedResult.append(Revision(wordsAdded=2, wordsDeleted=0).to_dict())
self.assertEqual(expectedResult, result)