def update_feed(self, url):
try:
feed = FeedHandler.parse_feed(url[0])
except:
feed = False
traceback.print_exc() # ???
if feed:
print(f'{url[0]}:')
print(f'Longitud de feed: {len(feed)}')
url_items = self.db.get_url_items(url=url[0])
for item in url_items:
url_items[item]['active'] = False
new_items = []
for item in feed:
hash = str(
CityHash64(item['summary'] + item['title'] + item['link']))
if not (hash in url_items):
new_items.append(item)
url_items[hash] = {
'active': True,
'last_date': DateHandler.get_datetime_now()
}
for item, value in url_items.copy().items():
if not value['active']:
print(f'Desactivando {item}')
if not value['active'] and DateHandler.is_older_than_days(
value['last_date'], 5):
print(f'Borrando {item}')
url_items.pop(item)
self.db.update_url_items(url=url[0], items=url_items)
telegram_users = self.db.get_users_for_url(url=url[0])
for user in telegram_users:
if user[6]: # is_active
if not feed:
message = "Something went wrong when I tried to parse the URL: \n\n " + \
url[0] + "\n\nCould you please check that for me? Remove the url from your subscriptions using the /remove command, it seems like it does not work anymore!"
self.bot.send_message(chat_id=user[0],
text=message,
parse_mode=ParseMode.HTML)
return
for post in new_items:
self.send_message(post=post, user=user)
def send_newest_messages(self, url, post, user):
post_update_date = DateHandler.parse_datetime(datetime=post.updated)
url_update_date = DateHandler.parse_datetime(datetime=url[1])
if post_update_date > url_update_date:
message = "[" + user[7] + "] <a href='" + post.link + \
"'>" + post.title + "</a>"
try:
self.bot.send_message(
chat_id=user[0], text=message, parse_mode=ParseMode.HTML)
except Unauthorized:
self.db.update_user(telegram_id=user[0], is_active=0)
except TelegramError:
# handle all other telegram related errors
pass
async def daily_liturgy():
date = DateHandler.get_datetime_now()
date_full = format_date(date.date(), format='full', locale='pt_br')
readings = BuscarLiturgia(dia=date.day, mes=date.month,
ano=date.year).obter_url()
chat_id_activated = db.get_chat_id_activated()
if readings:
for chat_id in chat_id_activated:
await send_daily_liturgy(chat_id, readings)
homily = util.homiliadodia.HomiliadoDia().obter_homilia()
if homily:
for chat_id in chat_id_activated:
await send_daily_liturgy(chat_id, homily)
audio = util.homiliadodia.HomiliadoDia().obter_arquivo_audio()
if audio:
send = await send_daily_liturgy_audio(config('CHANNEL_LD'),
audio['path_audio'],
audio['date'])
path_audio = send.audio.file_id
db.del_names(['audio_liturgy'])
db.set_name_key('audio_liturgy', {date_full: path_audio})
for chat_id in chat_id_activated:
await send_daily_liturgy_audio(chat_id, path_audio, audio['date'])
def add_url(self, url):
conn = sqlite3.connect(self.database_path)
cursor = conn.cursor()
cursor.execute("INSERT OR IGNORE INTO web (url, last_updated) VALUES (?,?)",
(url, dh.get_datetime_now()))
conn.commit()
conn.close()
def send_newest_messages(self, url, post, user):
post_update_date = DateHandler.parse_datetime(datetime=post.updated)
url_update_date = DateHandler.parse_datetime(datetime=url[1])
if post_update_date > url_update_date:
message = "Er is door {0} een nieuw artikel op de website geplaatst!\n" \
"<a href='{1}'>{2}</a>".format(post.author, post.link, post.title)
print(message)
try:
self.bot.send_message(
chat_id=user[0], text=message, parse_mode=ParseMode.HTML)
except Unauthorized:
if not user[0].startswith('@'):
self.db.update_user(telegram_id=user[0], is_active=0)
except TelegramError:
# handle all other telegram related errors
traceback.print_exc()
pass
def update_feed(self, url):
if not self._finished.isSet():
try:
get_url_info = self.db.get_update_url(url)
last_url = get_url_info['last_url']
date_last_url = DateHandler.parse_datetime(
get_url_info['last_update'])
feed = FeedHandler.parse_feed(
url, 4, date_last_url + timedelta(days=-1))
for post in feed:
if not hasattr(post, "published") and not hasattr(
post, "daily_liturgy"):
logger.warning('not published' + url)
continue
# for index, post in enumerate(feed):
date_published = DateHandler.parse_datetime(post.published)
if hasattr(post, "daily_liturgy"):
if date_published > date_last_url and post.link != last_url \
and post.daily_liturgy != '':
message = post.title + '\n' + post.daily_liturgy
result = self.send_newest_messages(message, url)
if post == feed[-1] and result:
self.update_url(url=url,
last_update=date_published,
last_url=post.link)
elif date_published > date_last_url and post.link != last_url:
message = post.title + '\n' + post.link
result = self.send_newest_messages(message, url)
if result:
self.update_url(url=url,
last_update=date_published,
last_url=post.link)
else:
pass
return True, url
except TypeError as e:
logger.error(f"TypeError {url} {str(e)}")
return False, url, 'update_feed'
except TelegramError as e:
logger.error(
f"except update_feed TelegramError {url} {str(e)}")
return False, url, 'update_feed'
def test_update_url(self):
self.db.add_url(url="https://lorem-rss.herokuapp.com/feed")
entry = self.db.get_url(url="https://lorem-rss.herokuapp.com/feed")
timestamp = str(dh.get_datetime_now())
self.db.update_url(
url="https://lorem-rss.herokuapp.com/feed", last_updated=timestamp)
result = self.db.get_url(url="https://lorem-rss.herokuapp.com/feed")
self.assertEqual(result[1], timestamp)
def set_user_daily_liturgy(self, chat_id, chat_name, user_id):
name = 'daily_liturgy:user_id:' + str(user_id) + ':chat_id:' + str(
chat_id)
mapping = {
'chat_id': str(chat_id),
'chat_name': chat_name,
'user_id': str(user_id),
'disable': 'False',
'last_send': str(DateHandler.get_datetime_now())
}
return True if self.set_name_key(name=name, mapping=mapping) else False
def add(self, bot, update, args):
"""
Adds a rss subscription to user
"""
telegram_user = update.message.from_user
if len(args) != 2:
message = "Sorry! I could not add the entry! Please use the the command passing the following arguments:\n\n /add <url> <entryname> \n\n Here is a short example: \n\n /add http://www.feedforall.com/sample.xml ExampleEntry"
update.message.reply_text(message)
return
arg_url = args[0]
arg_entry = args[1]
# Check if argument matches url format
feed=FeedHandler.is_parsable(url=arg_url)
if not feed:
message = f"Sorry! It seems like {arg_url}" + \
"' doesn't provide an RSS news feed... Have you tried another URL from that provider?"
update.message.reply_text(message)
return
# Check if entry does not exists
entries = self.db.get_urls_for_user(telegram_id=telegram_user.id)
if any(arg_url.lower() in entry.lower() for entry,_ in entries):
message = f"Sorry, {telegram_user.first_name}" + \
"! I already have that url stored in your subscriptions."
update.message.reply_text(message)
return
if any(arg_entry in entry for entry in entries):
message = f"Sorry! I already have an entry with name {arg_entry}" + \
" stored in your subscriptions.. Please choose another entry name or delete the entry using '/remove {arg_entry}'"
update.message.reply_text(message)
return
urls = self.db.get_all_urls()
if not (arg_url in urls):
items = {}
for item in feed:
for key in ['summary', 'title', 'link']:
if not(key in item.keys()):
item[key]=''
hash=CityHash64(item['summary']+item['title']+item['link'])
items[hash] = {'active': True, 'last_date': DateHandler.get_datetime_now()}
self.db.add_url(url=arg_url, items=items)
self.db.add_user_bookmark(telegram_id=telegram_user.id, url=arg_url, alias=arg_entry)
message = f"I successfully added {arg_entry} to your subscriptions!"
update.message.reply_text(message)
def send_newest_messages(self, url, post, user):
post_update_date = DateHandler.parse_datetime(datetime=post.updated)
url_update_date = DateHandler.parse_datetime(datetime=url[1])
if post_update_date > url_update_date:
filters = self.db.get_filters(user[0], url[0])
send = len(
filters) == 0 # if there are no filters. send everything
for filter in filters:
send |= self.match_filter(post, filter)
if send:
message = "[" + user[7] + "] <a href='" + post.link + \
"'>" + post.title + "</a>"
try:
self.bot.send_message(chat_id=user[0],
text=message,
parse_mode=ParseMode.HTML)
except Unauthorized:
self.db.update_user(telegram_id=user[0], is_active=0)
except TelegramError:
# handle all other telegram related errors
pass
def update_feed(url):
try:
get_url_info = db.get_update_url(url)
last_url = get_url_info['last_url']
date_last_url = DateHandler.parse_datetime(get_url_info['last_update'])
feed = FeedHandler.parse_feed(url, 4,
date_last_url + timedelta(days=-1))
for post in feed:
if not hasattr(post, "published") and not hasattr(
post, "daily_liturgy"):
logger.warning('not published' + url)
continue
date_published = DateHandler.parse_datetime(post.published)
if hasattr(post, "daily_liturgy"):
if date_published > date_last_url and post.link != last_url \
and post.daily_liturgy != '':
message = post.title + '\n' + post.daily_liturgy
result = send_newest_messages(message=message,
url=url,
disable_page_preview=True)
if post == feed[-1] and result:
update_url(url=url,
last_update=date_published,
last_url=post.link)
elif date_published > date_last_url and post.link != last_url:
message = post.title + '\n' + post.link
result = send_newest_messages(message=message, url=url)
if result:
update_url(url=url,
last_update=date_published,
last_url=post.link)
else:
pass
except TypeError as _:
logger.error(f"TypeError {url} {str(_)}")
def add_url(self, url, items):
"""Add URL to database
Args:
url (string): URL to add
items (dict): A dictionary containing the items from the given feed.
Dictionary in the form: { 'Item_hash': {'active': True/False, 'last_date': Str}, ...}
"""
conn = sqlite3.connect(self.database_path)
cursor = conn.cursor()
cursor.execute(
"INSERT OR IGNORE INTO web (url, last_updated, items) VALUES (?,?,?)",
(url, dh.get_datetime_now(), json.dumps(items)))
conn.commit()
conn.close()
def update_feed(self, url):
telegram_users = self.db.get_users_for_url(url=url[0])
for user in telegram_users:
if user[6]: # is_active
try:
for post in FeedHandler.parse_feed(url[0]):
self.send_newest_messages(
url=url, post=post, user=user)
except:
traceback.print_exc()
message = "Something went wrong when I tried to parse the URL: \n\n " + \
url[0] + "\n\nCould you please check that for me? Remove the url from your subscriptions using the /remove command, it seems like it does not work anymore!"
self.bot.send_message(
chat_id=user[0], text=message, parse_mode=ParseMode.HTML)
self.db.update_url(url=url[0], last_updated=str(
DateHandler.get_datetime_now()))
async def start(client, update):
chat_id = update.chat.id
chat_name = '@' + update.chat.username if update.chat.username else '@' + update.from_user.username \
if update.from_user.username else update.from_user.first_name
chat_title = update.chat.title or update.from_user.first_name
user_id = update.from_user.id
text = 'Vou te enviar a liturgia todos os dias\n\n' \
'Você também pode usar uma das opções abaixo para buscar a leitura do dia desejado\n\n' \
'/ontem - liturgia do dia anterior\n' \
'/hoje - liturgia do dia\n' \
'/amanha - liturgia do dia seguinte\n' \
'/dominical - liturgia dominical\n' \
'/santododia - santo do dia\n' \
'/calendario - escolher um dia específico\n' \
'/help - exibe esta mensagem.'
db.set_user_daily_liturgy(chat_id=chat_id,
chat_name=chat_name,
user_id=user_id)
await update.delete()
await update.reply_text(text=text,
quote=False,
parse_mode='html',
reply_markup=keyboard)
logger.info(
f'Invited by {user_id} to chat {chat_id} ({escape(chat_title)})')
date = DateHandler.get_datetime_now()
date_full = format_date(date.date(), format='full', locale='pt_br')
await client.send_chat_action(chat_id, "typing")
readings = BuscarLiturgia(dia=date.day, mes=date.month,
ano=date.year).obter_url()
if readings:
await send_daily_liturgy(chat_id, readings)
await client.send_chat_action(chat_id, "typing")
audio_telegram = db.get_value_name_key('audio_liturgy', date_full)
if audio_telegram:
homily = util.homiliadodia.HomiliadoDia().obter_homilia()
if homily:
await send_daily_liturgy(chat_id, homily)
await client.send_chat_action(chat_id, "upload_audio")
if audio_telegram:
await send_daily_liturgy_audio(chat_id, audio_telegram, date_full)
def backup(self):
now = DateHandler.get_datetime_now()
last_backup = self.get_value_name_key('backup', 'last_backup')
last_backup = DateHandler.parse_datetime(last_backup)
date_last_backup = DateHandler.date(last_backup)
hour_last_backup = DateHandler.time(last_backup)
if date_last_backup < DateHandler.date(now):
if hour_last_backup <= DateHandler.time(now):
mapping = {'last_backup': str(now)}
self.set_name_key('backup', mapping=mapping)
self.redis.save()
return True
else:
return False
def update_feed(self, url):
telegram_users = self.db.get_users_for_url(url=url[0])
telegram_channels = self.db.get_channels_for_url(url=url[0])
print("Processing url: {0}".format(url[0]))
try:
posts = FeedHandler.parse_feed(url[0])
except ValueError:
traceback.print_exc()
return
for post in posts:
print("Processing post: {0}".format(post.id))
for user in telegram_users:
if user[6]: # is_active
self.send_newest_messages(url=url, post=post, user=user)
for channel in telegram_channels:
self.send_newest_messages(url=url, post=post, user=channel)
self.db.update_url(url=url[0], last_updated=str(
DateHandler.get_datetime_now()))
def admin(_, update):
"""
Send admin commands
"""
chat_id = update.chat.id
if str(chat_id) not in db.list_admins():
return
update.reply_text(text=admin_text, quote=False, parse_mode='html')
# Start Bot
if __name__ == "__main__":
try:
logger.critical('Press Ctrl+%s to exit' % 'C')
print('Press Ctrl+{0} to exit'.format('C'))
scheduler = AsyncIOScheduler()
last_date = DateHandler.get_datetime_now() + timedelta(days=-1)
start_date = DateHandler.combine(last_date.date(),
DateHandler.time('06:00:00-03:00'))
scheduler.add_job(daily_liturgy,
"interval",
days=1,
start_date=start_date)
scheduler.start()
bot.run()
except RPCError as _:
error(_)
logger.critical(f'{_}')
print(_)
from cityhash import CityHash64
db = DatabaseHandler("resources/datastore.db")
arg_url = 'http://yle.fi/uutiset/rss/paauutiset.rss'
feed = FeedHandler.is_parsable(url=arg_url)
items = {}
for item in feed:
hash = CityHash64(item['summary'] + item['title'] + item['link'])
if (hash in items):
print(item['link'], item['summary'], items[hash])
items[hash] = {
'active': True,
'last_date': DateHandler.get_datetime_now(),
'link': item['link']
}
#self.db.add_url(url=arg_url, items=items)
url_items = db.get_url_items(url=arg_url)
for item in url_items:
url_items[item]['active'] = False
new_items = []
for item in feed:
hash = CityHash64(item['summary'] + item['title'] + item['link'])
if not (str(hash) in url_items):
new_items.append(item)
url_items[hash] = {
'active': True,