async def scrapeTopic(self, topic, page=0): posts = [] for start in range(page, 300, 15): content = await getRequest(self.config['URL_ROOT'] + self.config['URL_TOPIC'] + topic['id'] + "&start=" + str(start)) if 'General Error' in content.getText(): break for block in content.findAll('span', {'class': 'name'}): name = block.getText().strip() if name in self.config['members']: post = parsePost(block) log.info('found post from {} {}'.format(name, post['id'])) post['author'] = name post['link'] = (self.config['URL_ROOT'] + self.config['URL_POST'] + post['id'] + "#" + post['id']) posts.append(post) asyncio.sleep(self.waiting) return posts
async def bipTime(self): """ Just a debug function that print the current date and time """ now = datetime.now() msg = '...check: {}...'.format(now) log.info('sending: {}'.format(msg)) await self.send(msg)
async def start(self): """ Start or restart the execution. """ if not self.is_started: self.is_started = True log.info('starting...') # start task to call function periodically self._task = asyncio.ensure_future(self._run())
def updateChecked(self): """ Update the disk file with all the configuration data for the checked list. """ self.lastUpdate = datetime.now() log.info('update {}'.format(self.lastUpdate)) with open(envChecked, 'w+') as f: json.dump(self.checked, f)
async def scrapeIndex(self): log.info(self.config) content = await getRequest(self.config['URL_ROOT'] + self.config['URL_FORUM'] + self.config['FORUM_X4']) topicsList = content.findAll('a', {'class': 'topictitle'}) topics = [topicInfo(t) for t in topicsList] return topics
async def stop(self): """ Stop the execution, if it is running. """ if self.is_started: log.info('stopping...') self.is_started = False # stop task and await self._task.cancel() with suppress(asyncio.CancelledError): await self._task
async def commandLastUpdate(self, msg: Message): """ Return the last time the bot checked the forum for news. :param msg: must contain the keyword 'info', and mention this bot. """ if self.containsMention(msg) and 'info': log.info('last update done: {}'.format(self.lastUpdate)) await self.send('_Last update: {}_'.format(self.lastUpdate)) return True return False
async def commandOrder66(self, msg: Message): """ Start the kill-all-jedi routine. Long live the Emperor! :param msg: must contain the keyword '66', and mention this bot. :return an easter egg message :) """ if self.containsMention(msg) and '66' in msg.content: log.info("killing jedi...") await self.send('_ :skull: KILL ALL THE JEDI! :skull: _') return True return False
async def commandStop(self, msg: Message): """ Stop the current registered routine if it is running. :param msg: must contain the 'stop' keyword, and mention this bot :return: a shutdown message, or an empty string if the command is invalid """ if self.containsMention(msg) and 'stop' in msg.content: log.info('stop!') await self.stop() await self.send('_Sensors shutting down..._') return True return False
async def commandStart(self, msg: Message): """ Start the current registered routine if it is not already started. :param msg: must contain the 'start' keyword, and mention this bot :return: a startup message, or an empty string if the command is invalid """ if self.containsMention(msg) and 'start' in msg.content: log.info('start!') await self.start() await self.send('_booting up sensors..._') return True return False
def topicStartReadFrom(self, topic): """ Compute the page index to start read from (because the old message are already parsed). :param topic: input topic :return: the page index to start from """ if topic['id'] not in self.checked: return 0 n = self.checked[topic['id']]['posts'] log.info('#{:8} +{:<4} {}'.format(topic['id'], (topic['posts'] - n), topic['title'])) return n - (n % 15)
def __init__(self) -> None: super().__init__() self.config = {} self.waiting = randint(10, 30) log.info('current waiting: {}'.format(self.waiting)) with open(resourceDir + 'urls.config') as f: content = [line.split('=', 1) for line in f.readlines()] for tokens in content: self.config[tokens[0].strip()] = tokens[1].strip() with open(resourceDir + 'members.txt') as f: self.config['members'] = [line.strip() for line in f.readlines()]
async def webScraper(self): """ Routine to perform the web scraping of the forum. """ try: ws = WebScraper() topics = await ws.scrapeIndex() topics = sorted(topics, key=lambda x: -int(x['id'])) for topic in topics: if self.checkTopic(topic): # this topic has no news continue page = self.topicStartReadFrom(topic) posts = await ws.scrapeTopic(topic, page) posts = sorted(posts, key=lambda x: -int(x['id'])) for post in posts: if self.checkPost(topic, post): # already done log.info('post from {} #{} already done'.format(post['author'], post['id'])) continue # send all the messages! embed = buildEmbed(topic, post) try: await self.send('BIP!', embed=embed) # register the posts self.updateTopic(topic, post['id']) except Exception as e: log.error(e) # traceback.print_exc() except Exception as e: # if we have any kind of error with the remote site, call for help! log.error(e) # traceback.print_exc() await self.send('@everyone _O-oh..._') await self.stop() return self.updateChecked()
async def commandChangeFrequency(self, msg: Message): """ Change the current update frequency. The input value is in seconds and has 900s (15min) as lower bound. No upper bound is defined. Default value is 3600. The previous interval must expire before the new one will be used. :param msg: must contain the 'frequency' keyword, mention this bot, and have a valid number :return: a message, or an empty string if the command is invalid """ if self.containsMention(msg) and 'frequency' in msg.content: try: tokens = msg.content.split(' ') time = int(tokens[len(tokens) - 1]) self.time = max(time, 900) response = '_New frequency {}_'.format(str(self.time)) log.info('new frequency: {}'.format(self.time)) except ValueError: response = '_Kzzzzzt! Invalid..._' log.info('cannot parse number: {}'.format(msg.content)) await self.send(response) return True return False
def serveDrinks(self, message: Message): if not self.checkCall(message): return None, None mentions = message.mentions content = message.content clean = re.sub('[^A-Za-z0-9 ]+', '', content.lower()) tokens = clean.split(' ') log.debug('tokens: ' + str(tokens)) # search for any kind of drink foundDrinks = [d for d in self.drinks if d['key'] in clean] mention = message.author.mention title = random.choice(titles) if len(tokens) == 1 and len(foundDrinks) == 0: log.info('send generic order request') return random.choice(orders).format(message.author.mention), None if 'grazie' in tokens: # get a random thank string log.info('send thanks') return random.choice(thanks).format(title), None if 'drinks' in tokens: # return the list of drinks log.info('send drink list') return 'Abbiamo ' + ', '.join([d['name'] for d in self.drinks]), None if len(foundDrinks) < 1: # if we haven't found a valid drink log.info('send not found') return random.choice(notFound), None # we have found a drink! foundDrink = foundDrinks[0] if 'tutti' in tokens or message.mention_everyone: # if it is for @everyone or contains a special token log.info('send offer to everyone') return '@everyone il prossimo giro di {2} lo offre {1} {0}!'.format( mention, title, foundDrink['name']), foundDrink['img'] if len(mentions) > 0: # if we have mentions, notify to them log.info('offer to user') intro = ', '.join([str(m.mention) for m in mentions]) \ + ' ' + str(mention) + ' vi offre' else: # just answer the client log.info('answer the client') intro = random.choice(incipits).format(mention) drink = random.choice(drinkings).format(foundDrink['name']) conclusion = random.choice(conclusions) log.info('sending drink') return intro + ' ' + drink + ' ' + conclusion, foundDrink['img']