async def scrapeTopic(self, topic, page=0):
posts = []
for start in range(page, 300, 15):
content = await getRequest(self.config['URL_ROOT'] +
self.config['URL_TOPIC'] + topic['id'] +
"&start=" + str(start))
if 'General Error' in content.getText():
break
for block in content.findAll('span', {'class': 'name'}):
name = block.getText().strip()
if name in self.config['members']:
post = parsePost(block)
log.info('found post from {} {}'.format(name, post['id']))
post['author'] = name
post['link'] = (self.config['URL_ROOT'] +
self.config['URL_POST'] + post['id'] +
"#" + post['id'])
posts.append(post)
asyncio.sleep(self.waiting)
return posts
async def bipTime(self):
"""
Just a debug function that print the current date and time
"""
now = datetime.now()
msg = '...check: {}...'.format(now)
log.info('sending: {}'.format(msg))
await self.send(msg)
async def start(self):
"""
Start or restart the execution.
"""
if not self.is_started:
self.is_started = True
log.info('starting...')
# start task to call function periodically
self._task = asyncio.ensure_future(self._run())
def updateChecked(self):
"""
Update the disk file with all the configuration data for the checked list.
"""
self.lastUpdate = datetime.now()
log.info('update {}'.format(self.lastUpdate))
with open(envChecked, 'w+') as f:
json.dump(self.checked, f)
async def scrapeIndex(self):
log.info(self.config)
content = await getRequest(self.config['URL_ROOT'] +
self.config['URL_FORUM'] +
self.config['FORUM_X4'])
topicsList = content.findAll('a', {'class': 'topictitle'})
topics = [topicInfo(t) for t in topicsList]
return topics
async def stop(self):
"""
Stop the execution, if it is running.
"""
if self.is_started:
log.info('stopping...')
self.is_started = False
# stop task and await
self._task.cancel()
with suppress(asyncio.CancelledError):
await self._task
async def commandLastUpdate(self, msg: Message):
"""
Return the last time the bot checked the forum for news.
:param msg: must contain the keyword 'info', and mention this bot.
"""
if self.containsMention(msg) and 'info':
log.info('last update done: {}'.format(self.lastUpdate))
await self.send('_Last update: {}_'.format(self.lastUpdate))
return True
return False
async def commandOrder66(self, msg: Message):
"""
Start the kill-all-jedi routine. Long live the Emperor!
:param msg: must contain the keyword '66', and mention this bot.
:return an easter egg message :)
"""
if self.containsMention(msg) and '66' in msg.content:
log.info("killing jedi...")
await self.send('_ :skull: KILL ALL THE JEDI! :skull: _')
return True
return False
async def commandStop(self, msg: Message):
"""
Stop the current registered routine if it is running.
:param msg: must contain the 'stop' keyword, and mention this bot
:return: a shutdown message, or an empty string if the command is invalid
"""
if self.containsMention(msg) and 'stop' in msg.content:
log.info('stop!')
await self.stop()
await self.send('_Sensors shutting down..._')
return True
return False
async def commandStart(self, msg: Message):
"""
Start the current registered routine if it is not already started.
:param msg: must contain the 'start' keyword, and mention this bot
:return: a startup message, or an empty string if the command is invalid
"""
if self.containsMention(msg) and 'start' in msg.content:
log.info('start!')
await self.start()
await self.send('_booting up sensors..._')
return True
return False
def topicStartReadFrom(self, topic):
"""
Compute the page index to start read from (because the old message are already parsed).
:param topic: input topic
:return: the page index to start from
"""
if topic['id'] not in self.checked:
return 0
n = self.checked[topic['id']]['posts']
log.info('#{:8} +{:<4} {}'.format(topic['id'], (topic['posts'] - n), topic['title']))
return n - (n % 15)
def __init__(self) -> None:
super().__init__()
self.config = {}
self.waiting = randint(10, 30)
log.info('current waiting: {}'.format(self.waiting))
with open(resourceDir + 'urls.config') as f:
content = [line.split('=', 1) for line in f.readlines()]
for tokens in content:
self.config[tokens[0].strip()] = tokens[1].strip()
with open(resourceDir + 'members.txt') as f:
self.config['members'] = [line.strip() for line in f.readlines()]
async def webScraper(self):
"""
Routine to perform the web scraping of the forum.
"""
try:
ws = WebScraper()
topics = await ws.scrapeIndex()
topics = sorted(topics, key=lambda x: -int(x['id']))
for topic in topics:
if self.checkTopic(topic):
# this topic has no news
continue
page = self.topicStartReadFrom(topic)
posts = await ws.scrapeTopic(topic, page)
posts = sorted(posts, key=lambda x: -int(x['id']))
for post in posts:
if self.checkPost(topic, post):
# already done
log.info('post from {} #{} already done'.format(post['author'], post['id']))
continue
# send all the messages!
embed = buildEmbed(topic, post)
try:
await self.send('BIP!', embed=embed)
# register the posts
self.updateTopic(topic, post['id'])
except Exception as e:
log.error(e)
# traceback.print_exc()
except Exception as e:
# if we have any kind of error with the remote site, call for help!
log.error(e)
# traceback.print_exc()
await self.send('@everyone _O-oh..._')
await self.stop()
return
self.updateChecked()
async def commandChangeFrequency(self, msg: Message):
"""
Change the current update frequency. The input value is in seconds and has 900s (15min) as lower bound.
No upper bound is defined. Default value is 3600.
The previous interval must expire before the new one will be used.
:param msg: must contain the 'frequency' keyword, mention this bot, and have a valid number
:return: a message, or an empty string if the command is invalid
"""
if self.containsMention(msg) and 'frequency' in msg.content:
try:
tokens = msg.content.split(' ')
time = int(tokens[len(tokens) - 1])
self.time = max(time, 900)
response = '_New frequency {}_'.format(str(self.time))
log.info('new frequency: {}'.format(self.time))
except ValueError:
response = '_Kzzzzzt! Invalid..._'
log.info('cannot parse number: {}'.format(msg.content))
await self.send(response)
return True
return False
def serveDrinks(self, message: Message):
if not self.checkCall(message):
return None, None
mentions = message.mentions
content = message.content
clean = re.sub('[^A-Za-z0-9 ]+', '', content.lower())
tokens = clean.split(' ')
log.debug('tokens: ' + str(tokens))
# search for any kind of drink
foundDrinks = [d for d in self.drinks if d['key'] in clean]
mention = message.author.mention
title = random.choice(titles)
if len(tokens) == 1 and len(foundDrinks) == 0:
log.info('send generic order request')
return random.choice(orders).format(message.author.mention), None
if 'grazie' in tokens:
# get a random thank string
log.info('send thanks')
return random.choice(thanks).format(title), None
if 'drinks' in tokens:
# return the list of drinks
log.info('send drink list')
return 'Abbiamo ' + ', '.join([d['name']
for d in self.drinks]), None
if len(foundDrinks) < 1:
# if we haven't found a valid drink
log.info('send not found')
return random.choice(notFound), None
# we have found a drink!
foundDrink = foundDrinks[0]
if 'tutti' in tokens or message.mention_everyone:
# if it is for @everyone or contains a special token
log.info('send offer to everyone')
return '@everyone il prossimo giro di {2} lo offre {1} {0}!'.format(
mention, title, foundDrink['name']), foundDrink['img']
if len(mentions) > 0:
# if we have mentions, notify to them
log.info('offer to user')
intro = ', '.join([str(m.mention) for m in mentions]) \
+ ' ' + str(mention) + ' vi offre'
else:
# just answer the client
log.info('answer the client')
intro = random.choice(incipits).format(mention)
drink = random.choice(drinkings).format(foundDrink['name'])
conclusion = random.choice(conclusions)
log.info('sending drink')
return intro + ' ' + drink + ' ' + conclusion, foundDrink['img']