def run(self, revisionTextA, revisionTextB, revisionDict): gdiff = diff_match_patch() revisionDiffs = gdiff.diff_main(revisionTextA, revisionTextB, False) gdiff.diff_cleanupSemantic(revisionDiffs) revisionDiffs = filter(self.isRemoveOrAdd, revisionDiffs) diffWordCount = map(self.countWords, revisionDiffs) addedWordCount = self.getAddWordCount(diffWordCount) deletedWordCount = self.getDeletedWordCount(diffWordCount) revision = Revision(**revisionDict) revision.wordsAdded = addedWordCount revision.wordsDeleted = deletedWordCount revision.wordCount = self.getWordCount(revisionTextB) return revision.to_dict()
def is_new(self): # today = datetime.date.today() # delta = datetime.timedelta(days=settings.BOT_CONSIDERED_NEW) # result = today - self.date_added <= delta # return result return self.revision >= Revision.get_instance( ).nr - settings.BOT_CONSIDERED_NEW + 1
def many_by_usernames(names: List): results = Bot.select( ).where((fn.lower(Bot.username) << [n.lower() for n in names]) & ( Bot.revision <= Revision.get_instance().nr & Bot.approved == True)) if len(results) > 0: return results else: raise Bot.DoesNotExist()
def main(cli_args): client = TreeherderClient(cli_args.repo, cli_args.csrf_token, cli_args.session_id) revision = Revision(client, cli_args.revision) results = revision.results jobs = [result.get_jobs_for(cli_args.job_type, job_number=cli_args.job_number, job_platform=cli_args.job_platform) for result in results] jobs = _flatten_list(jobs) for job in jobs: for i in range(1, cli_args.repeat + 1): print('Repeat #{}'.format(i)) job.__getattribute__(cli_args.command)()
def update_categories(self, categories: List): self.notify_admin( "Updating BotList categories to Revision {}...".format( Revision.get_instance().nr)) for cat in categories: text = _format_category_bots(cat) msg = self.send_or_edit(text, cat.current_message_id) if msg: cat.current_message_id = msg.message_id self.sent['category'].append("{} {}".format( 'Resent' if self.resend else 'Updated', cat)) cat.save() self._save_channel() # Add "share", "up", and "down" buttons for i in range(0, len(categories)): buttons = list() if i > 0: # Not first category # Add "Up" button buttons.append( InlineKeyboardButton( "🔺", url=BotList.create_hyperlink( categories[i - 1].current_message_id))) buttons.append( InlineKeyboardButton("Share", url="https://t.me/{}?start={}".format( settings.SELF_BOT_NAME, categories[i].id))) if i < len(categories) - 1: # Not last category buttons.append( InlineKeyboardButton( "🔻", url=BotList.create_hyperlink( categories[i + 1].current_message_id))) reply_markup = InlineKeyboardMarkup([buttons]) self.bot.edit_message_reply_markup( self.channel.chat_id, categories[i].current_message_id, reply_markup=reply_markup, timeout=60)
def search_bots(query): query = query.lower().strip() split = query.split(' ') # easter egg if query in ('awesome bot', 'great bot', 'superb bot', 'best bot', 'best bot ever'): return [Bot.by_username('@botlistbot')] # exact results where_query = ( (fn.lower(Bot.username).contains(query) | fn.lower(Bot.name) << split | fn.lower(Bot.extra) ** query) & (Bot.revision <= Revision.get_instance().nr & Bot.approved == True) ) results = set(Bot.select().distinct().where(where_query)) # keyword results keyword_results = Bot.select(Bot).join(Keyword).where( (fn.lower(Keyword.name) << split) & (Bot.revision <= Revision.get_instance().nr) & (Bot.approved == True) ) results.update(keyword_results) # many @usernames usernames = re.findall(settings.REGEX_BOT_ONLY, query) if usernames: try: bots = Bot.many_by_usernames(usernames) results.update(bots) except Bot.DoesNotExist: pass return list(results)
def run(self, toEmail, documentId, credentialsAsJson): credentials = OAuth2Credentials.from_json(credentialsAsJson) http = credentials.authorize(httplib2.Http()) request = config.getService().revisions().list(fileId=documentId) response = request.execute(http=http) revisions = [] for item in response.get('items'): revision = Revision() revision.documentId = documentId revision.revisionNumber = item.get('id') revision.author = item.get('lastModifyingUserName') revision.exportLink = item.get('exportLinks').get('text/plain') revision.dateTime = item.get('modifiedDate') revisions.append(revision.to_dict()) return revisions
def testRun(self): pipeline = RevisionsDiffPipeline('Hello', 'Hello there', {}) pipeline.start_test() result = pipeline.outputs.default.value expected = Revision() expected.wordsAdded = 1 expected.wordsDeleted = 0 expected.wordCount = 2 self.assertEqual(result, expected.to_dict())
def run(self, toEmail, documentId, credentialsAsJson): credentials = OAuth2Credentials.from_json(credentialsAsJson) http = credentials.authorize(httplib2.Http()) request = config.getService().revisions().list(fileId=documentId) response = request.execute(http=http) revisions = [] for item in response.get('items'): revision = Revision() revision.documentId = documentId revision.revisionNumber = item.get('id') revision.author = item.get('lastModifyingUserName') revision.exportLink = item.get('exportLinks').get('text/plain') revision.dateTime = item.get('modifiedDate') revisions.append(revision.to_dict()) return revisions
def run(self, revisionTextA, revisionTextB, revisionDict): gdiff = diff_match_patch() revisionDiffs = gdiff.diff_main(revisionTextA, revisionTextB, False) gdiff.diff_cleanupSemantic(revisionDiffs) revisionDiffs = filter(self.isRemoveOrAdd, revisionDiffs) diffWordCount = map(self.countWords, revisionDiffs) addedWordCount = self.getAddWordCount(diffWordCount) deletedWordCount = self.getDeletedWordCount(diffWordCount) revision = Revision(**revisionDict) revision.wordsAdded = addedWordCount revision.wordsDeleted = deletedWordCount revision.wordCount = self.getWordCount(revisionTextB) return revision.to_dict()
def send_botlist(bot, update, resend=False, silent=False): log.info("Re-sending BotList..." if resend else "Updating BotList...") channel = helpers.get_channel() revision = Revision.get_instance() revision.nr += 1 revision.save() all_categories = Category.select_all() botlist = BotList(bot, update, channel, resend, silent) if resend: botlist.delete_full_botlist() botlist.update_intro() botlist.update_categories(all_categories) botlist.update_new_bots_list() botlist.update_category_list() botlist.send_footer() botlist.finish() channel.save() Statistic.of(update, 'send', 'botlist (resend: {})'.format(str(resend)), Statistic.IMPORTANT)
def run(self, toEmail, subject, body, documentsAnalyses): # TODO(michaelcupino): Move csv transforms into a service. csvFile = tempfile.TemporaryFile() writer = csv.writer(csvFile) values = [['Date', 'Time', 'Who in doc', 'Word count', 'Words added', 'Words deleted', 'Punct. cap', 'Words moved', 'Document ID', 'Document Name']] writer.writerows(values) for documentAnalyses in documentsAnalyses: for documentAnalysis in documentAnalyses: revision = Revision(**documentAnalysis) values = [[ revision.dateTime, # Date '', # Time revision.author, revision.wordCount, revision.wordsAdded, revision.wordsDeleted, '', # Punct. cap '', # Words moved revision.documentId, # Document ID '', # Document Name ]] writer.writerows(values) csvFile.seek(0) csvFileBytes = csvFile.read() csvFile.close() message = mail.EmailMessage() message.sender = '*****@*****.**' message.to = toEmail message.subject = subject message.body = str(body) message.attachments = [('export.csv', csvFileBytes)] message.send()
def select_pending_update(): return Bot.select().where(Bot.approved == True, Bot.revision == Revision.get_instance().next)
def select_approved(): return Bot.select().where(Bot.approved == True, Bot.revision <= Revision.get_instance().nr)
def select_new_bots(): return Bot.select().where( (Bot.revision >= Revision.get_instance().nr - settings.BOT_CONSIDERED_NEW + 1) & (Bot.revision < Revision.get_instance().next) & Bot.approved == True)
def of_category_without_new(category): return Bot.select().where( Bot.category == category, Bot.approved == True, Bot.revision <= Revision.get_instance().nr).order_by( fn.Lower(Bot.username))
def explorable_bots(): results = Bot.select().where( ~(Bot.description.is_null()), Bot.approved == True, Bot.revision <= Revision.get_instance().nr) return list(results)
def new_bot_submission(bot, update, chat_data, args=None): tg_user = update.message.from_user user = User.from_telegram_object(tg_user) if util.stop_banned(update, user): return reply_to = util.original_reply_id(update) if args: text = ' '.join(args) else: text = update.message.text command_no_args = len( re.findall(r'^/new\s*$', text)) > 0 or text.lower().strip() == '/new@botlistbot' if command_no_args: update.message.reply_text(util.action_hint( "Please use this command with an argument. For example:\n/new @mybot 🔎" ), reply_to_message_id=reply_to) return # `#new` is already checked by handler try: username = re.match(settings.REGEX_BOT_IN_TEXT, text).groups()[0] if username.lower() == '@' + settings.SELF_BOT_NAME.lower(): log.info("Ignoring {}".format(text)) return except AttributeError: if args: update.message.reply_text(util.failure( "Sorry, but you didn't send me a bot `@username`."), quote=True, parse_mode=ParseMode.MARKDOWN, reply_to_message_id=reply_to) log.info("Ignoring {}".format(text)) # no bot username, ignore update return try: new_bot = Bot.by_username(username) if new_bot.approved: update.message.reply_text(util.action_hint( "Sorry fool, but {} is already in the @BotList 😉".format( new_bot.username)), reply_to_message_id=reply_to) else: update.message.reply_text(util.action_hint( "{} has already been submitted. Please have patience...". format(new_bot.username)), reply_to_message_id=reply_to) return except Bot.DoesNotExist: new_bot = Bot(revision=Revision.get_instance().next, approved=False, username=username, submitted_by=user) new_bot.inlinequeries = "🔎" in text new_bot.official = "🔹" in text # find language languages = Country.select().execute() for lang in languages: if lang.emoji in text: new_bot.country = lang new_bot.date_added = datetime.date.today() description_reg = re.match(settings.REGEX_BOT_IN_TEXT + ' -\s?(.*)', text) description_notify = '' if description_reg: description = description_reg.group(2) new_bot.description = description description_notify = ' Your description was included.' new_bot.save() if util.is_private_message(update) and util.uid_from_update( update) in settings.MODERATORS: from components.explore import send_bot_details send_bot_details(bot, update, chat_data, new_bot) else: msg = update.message.reply_text(util.success( "You submitted {} for approval.{}".format(new_bot, description_notify)), parse_mode=ParseMode.MARKDOWN, reply_to_message_id=reply_to) return ConversationHandler.END
def testRun(self, mockUrlOpenMethod): exportLinks = [ 'docs.google.com/123', 'docs.google.com/456', 'docs.google.com/789' ] mockRevisions = [ Revision(exportLink=exportLinks[0]).to_dict(), Revision(exportLink=exportLinks[1]).to_dict(), Revision(exportLink=exportLinks[2]).to_dict() ] exportedText = { exportLinks[0]: StringIO('Hello'), exportLinks[1]: StringIO('Hello world.'), exportLinks[2]: StringIO('Hello.'), } mockUrlOpenMethod.side_effect = lambda url: exportedText[url] # TODO(michaelcupino): Fix tests pipeline = RevisionsAnalysisPipeline(mockRevisions, None) pipeline.start_test() result = pipeline.outputs.default.value revision1 = Revision() revision1.wordsAdded = 1 revision1.wordsDeleted = 0 revision1.wordCount = 1 revision1.exportLink = 'docs.google.com/123' revision2 = Revision() revision2.wordsAdded = 1 revision2.wordsDeleted = 0 revision2.wordCount = 2 revision2.exportLink = 'docs.google.com/456' revision3 = Revision() revision3.wordsAdded = 0 revision3.wordsDeleted = 1 revision3.wordCount = 1 revision3.exportLink = 'docs.google.com/789' self.assertEqual([revision1.to_dict(), revision2.to_dict(), revision3.to_dict()], result)
import os from decouple import config from playhouse.migrate import SqliteMigrator, IntegerField, migrate from playhouse.sqlite_ext import SqliteExtDatabase from model.revision import Revision db_path = config('DATABASE_URI', default=os.path.expanduser('~/botlistbot.sqlite3')) db = SqliteExtDatabase(db_path) migrator = SqliteMigrator(db) revision = IntegerField(default=100) with db.transaction(): migrate(migrator.add_column('bot', 'revision', revision), ) Revision.create_table(fail_silently=True) Revision.insert({'nr': 101}).execute()
def testRun(self, mockOAuth2CredentialsFromJsonMethod, mockUrlOpenMethod): http = HttpMockSequence([ ({ 'status': '200' }, open(datafile('test-drive-revisions-list-0.json'), 'rb').read()), ]) mockCredentials = MagicMock(name='mockCredentials') mockCredentials.authorize = MagicMock(return_value=http) mockOAuth2CredentialsFromJsonMethod.return_value = mockCredentials exportLinks = [ ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=697&exportFormat=tx' 't'), ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=698&exportFormat=tx' 't'), ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=868&exportFormat=tx' 't'), ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=869&exportFormat=tx' 't'), ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=937&exportFormat=tx' 't'), ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=939&exportFormat=tx' 't'), ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=1035&exportFormat=t' 'xt'), ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=1036&exportFormat=t' 'xt'), ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=1048&exportFormat=t' 'xt'), ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=1049&exportFormat=t' 'xt'), ('https://docs.google.com/feeds/download/documents/export/Export?id=1Duz2' 'yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=1057&exportFormat=t' 'xt') ] exportedText = { exportLinks[0]: StringIO('Hello'), exportLinks[1]: StringIO('Hello world.'), exportLinks[2]: StringIO('Hello'), exportLinks[3]: StringIO(''), exportLinks[4]: StringIO('Hello again, this is a longer sentence'), exportLinks[5]: StringIO(('Hello again, this is a longer sentence with ' 'another word')), exportLinks[6]: StringIO(('Hello again, this is a longer sentence with ' 'another word. Here is another sentence')), exportLinks[7]: StringIO( ('Hello again, this is a longer sentence with ' 'another word. Here is another sentence with another word')), exportLinks[8]: StringIO( ('Hello again, this is a longer sentence with ' 'another word. Here is another sentence with another word')), exportLinks[9]: StringIO(('Hello again, this is a longer sentence with ' 'another word. Here is another sentence.')), exportLinks[10]: StringIO(('Hello again, this is a longer sentence with ' 'another word. Here is another sentence. The end.')) } mockUrlOpenMethod.side_effect = lambda url: exportedText[url] # TODO(michaelcupino): Fix tests pipeline = DocumentAnalysisPipeline('*****@*****.**', 'abc123', None) pipeline.start_test() result = pipeline.outputs.default.value # TODO(michaelcupino): Updated expected data to match actual result. expectedResult = [] expectedResult.append( Revision( wordsAdded=1, wordsDeleted=0, wordCount=1, author='Jonathan H', dateTime='2014-03-15T17:49:32.768Z', exportLink= ('https://docs.google.com/feeds/download/documents/export/Ex' 'port?id=1Duz2yYSTSgdWQhTgEJ3zPELKTHS3WGhPVLQk1PDJm3Q&revision=697&' 'exportFormat=txt'), revisionNumber='697', documentId='abc123').to_dict()) expectedResult.append(Revision(wordsAdded=1, wordsDeleted=0).to_dict()) expectedResult.append(Revision(wordsAdded=0, wordsDeleted=1).to_dict()) expectedResult.append(Revision(wordsAdded=0, wordsDeleted=1).to_dict()) expectedResult.append(Revision(wordsAdded=7, wordsDeleted=0).to_dict()) expectedResult.append(Revision(wordsAdded=3, wordsDeleted=0).to_dict()) expectedResult.append(Revision(wordsAdded=5, wordsDeleted=0).to_dict()) expectedResult.append(Revision(wordsAdded=3, wordsDeleted=0).to_dict()) expectedResult.append(Revision(wordsAdded=0, wordsDeleted=0).to_dict()) # TODO(michaelcupino): I would have expected there to be no words added. # Figure out why this behavior occurrs. expectedResult.append(Revision(wordsAdded=1, wordsDeleted=3).to_dict()) expectedResult.append(Revision(wordsAdded=2, wordsDeleted=0).to_dict()) self.assertEqual(expectedResult, result)