def processStatus(url, channels):
if not db.existing.add(note):
return
result = web_2_album.get(url)
for channel in channels:
time.sleep(5)
album_sender.send_v2(channel, result)
def postTele(item):
try:
post_link = item.find('span', class_='created_at').find('a')['href']
except:
return
if existing.contain(post_link):
return 'existing'
result = getResult(post_link, item)
if not result:
return
rhash = getHash(result.cap)
if existing.contain(rhash):
return
if 'test' in sys.argv:
print('posting', post_link, list(dataCount(item)))
try:
album_sender.send_v2(douban_channel, result)
except Exception as e:
message = 'douban_read send_failure %s %s' % (post_link, str(e))
debug_group.send_message(message)
print(message)
return
existing.add(post_link)
existing.add(rhash)
return 'sent'
def process(key):
channels = subscription.channels(tele.bot, key)
search_result = weiboo.search(key, force_cache=True)
if isInt(key): # backfill, can remove this part after 7/17
result = weiboo.searchUser(key)
if result:
weibo_name.update(result[0], result[1])
if not search_result:
print('no search result', key)
for url, card in search_result:
result = None
for channel in channels:
if not shouldProcess(channel, card, key):
continue
print(url, channel.id, channel.username)
if not result:
time.sleep(60)
result = weibo_2_album.get(url, card['mblog'])
removeSeeMore(result)
try:
album_sender.send_v2(channel, result)
except Exception as e:
debug_group.send_message(
getLogStr(channel.username, channel.id, url, e))
raise e
def send(status):
thash = str(getHash(status)) + str(channel.id)
if not existing.add(thash):
return
album = twitter_2_album.get(str(status.id),
origin=[str(channel.id), channel.username])
if shouldSendAlbum(channel, album):
try:
album_sender.send_v2(channel, album)
except:
debug_group.send_message('send fail. error: %s, url: %s' %
(e, album.url))
def cut(update, context):
msg = update.effective_message
if msg.chat_id == debug_group.id or msg.media_group_id:
return
file = msg.document or (msg.photo and msg.photo[-1])
file_path = (file and file.get_file().file_path) or msg.text or ''
if not file_path.startswith('http'):
return
result = AlbumResult()
result.cap = msg.caption_markdown or msg.text_markdown or ''
result.imgs = [file_path]
album_sender.send_v2(msg.chat, result, send_all=True, size_factor=2.1)
def exportImp(msg):
soup = BeautifulSoup(msg.text_html_urled, 'html.parser')
for item in soup.find_all('a'):
if 'http' in item.get('href'):
url = item.get('href')
album = getAlbum(msg, url)
if album:
album_sender.send_v2(msg.chat, album)
continue
result = getTelegraph(msg, url)
yield result
if str(msg.chat_id) in no_source_link._db.items:
msg.chat.send_message(result)
else:
msg.chat.send_message('%s | [source](%s)' % (result, url),
parse_mode='Markdown')
def process(key, method=weiboo.search):
channels = subscription.channels(tele.bot, key)
try:
search_result = method(key, force_cache=True)
except Exception as e:
print('search failed', key, str(e))
return
if not search_result:
print('no search result', key)
return
for url, card in search_result:
result = None
for channel in channels:
if not shouldProcess(channel, card, key):
continue
result_posts = []
try:
if not result:
result = getResult(url, card, channels)
result_posts = album_sender.send_v2(channel, result)
except Exception as e:
debug_group.send_message(
getLogStr(channel.username, channel.id, url, e))
finally:
post_len = len(result_posts or [])
time.sleep((post_len**2) / 2 + post_len * 10)
log(url, card, key, channels, result)
def sendLink(site, link, fixed_channel = None):
simplified = None
telegraph = None
album_result = None
additional_info = getAdditionalInfo(site)
channels = list(db.sub.channels(site, tele.bot))
message = link
success = True
for channel, config in channels:
if fixed_channel and channel.id != fixed_channel:
continue
blocked_keys = getMatchedKey(
link + export_to_telegraph.getTitle(link) + export_to_telegraph.getTitle(link, toSimplified=True),
blocklist.get(channel.id, []))
if blocked_keys:
message += ' filtered: ' + ' '.join(blocked_keys)
continue
if not album_result and '.douban.' in link and '/note/' not in link:
album_result = web_2_album.get(link, force_cache = True)
if album_result.imgs:
album_result.cap = cutCaptionHtml(album_result.cap, 800)
else:
album_result.cap = cutCaptionHtml(album_result.cap, 2000)
if not album_result and 'to_album' in config:
album_result = export_to_telegraph.getAlbum(link)
if not simplified and 'to_simplify' in config:
simplified = export_to_telegraph.export(link,
force_cache = True, force=True, toSimplified=True) or link
if not telegraph and not album_result and 'to_telegraph' in config:
telegraph = export_to_telegraph.export(link,
force_cache = True, force=True) or link
message = link
if 'to_simplify' in config:
message = simplified
if 'to_telegraph' in config:
message = telegraph
try:
if album_result:
album_sender.send_v2(channel, album_result)
else:
channel.send_message(message + additional_info, parse_mode='HTML')
except Exception as e:
print(e)
success = False
debug_group.send_message('send fail: %s %d %s' % (link, channel.id, e))
log(message or link, site, [item[0] for item in channels])
return success
def loopImp():
removeOldFiles('tmp')
global processed_channels
processed_channels = set()
channels = list(subscription.getChannels())
for key in subscription.keys():
if key != 'hometimeline' and isinstance(key, str) and random.random() > 0.1:
continue
for status in getStatuses(key):
sent = False
for channel in channels:
if shouldProcess(channel, status, key):
try:
album = twitter_2_album.get(str(status.id),
origin = [str(channel.id), channel.username])
if shouldSendAlbum(channel, album):
album_sender.send_v2(channel, album)
sent = True
except Exception as e:
print('send fail', channel.id, str(e), status.id)
log(key, status, sent)
def toAlbumInternal(update, context):
if update.edited_message or update.edited_channel_post:
return
msg = update.effective_message
url = getUrl(msg)
if not url:
return
result = getResult(url, msg.text, getOrigins(msg))
if not result:
return
if msg.text.endswith(' t'): # text only
result.imgs = []
result.video = ''
rotate = 0
if msg.text.split()[-1].startswith('r'):
try:
rotate = int(msg.text.split()[-1][1:])
except:
...
tmp_msg = None
error = ''
final_result = ''
send_all = (msg.chat_id == -1001367414473)
try:
if str(msg.chat_id) in remove_origin._db.items:
tryDelete(msg)
waitlist_msg = waitlist_log.send_message(msg.text)
else:
tmp_msg = tele.bot.send_message(msg.chat_id, 'sending')
final_result = album_sender.send_v2(msg.chat, result, rotate = rotate, send_all=send_all)[0]
if final_result and str(msg.chat_id) in remove_origin._db.items:
tryDelete(waitlist_msg)
except Exception as e:
error = ' error: ' + str(e)
if final_result:
final_result = final_result.text_html_urled or final_result.caption_html_urled
if final_result:
final_result = ' result: ' + final_result
info_log.send_message(getBasicLog(msg) + error + final_result,
parse_mode='html', disable_web_page_preview=True)
if tmp_msg:
tryDelete(tmp_msg)
def sendLink(site, link, fixed_channel=None):
simplified = None
telegraph = None
album_result = None
sent = False
for channel, config in db.sub.channels(site, tele.bot):
if fixed_channel and channel.id != fixed_channel:
continue
if not simplified and 'to_simplify' in config:
simplified = export_to_telegraph.export(
link, force_cache=True, force=True, toSimplified=True) or link
if '.douban.' in link and '/note/' not in link:
album_result = web_2_album.get(link, force_cache=True)
if not telegraph and not album_result and 'to_telegraph' in config:
telegraph = export_to_telegraph.export(
link, force_cache=True, force=True) or link
message = link
if 'to_simplify' in config:
message = simplified
if 'to_telegraph' in config:
message = telegraph
result = [1] * 10
try:
if album_result:
result = album_sender.send_v2(channel, album_result)
else:
result = [channel.send_message(message)]
except Exception as e:
print(e)
debug_group.send_message('send fail: ' + str(channel.id) + ' ' +
str(e))
finally:
if sent:
post_len = len(result)
time.sleep((post_len**2) / 2 + post_len * 10)
sent = True
def test(url, rotate=False):
r = weibo_2_album.get(url)
r.cap_html_v2 = ''
print(r)
album_sender.send_v2(chat, r, rotate=rotate)
def test(url, rotate=False):
result = web_2_album.get(url)
# print(list(result.cap))
album_sender.send_v2(chat, result, rotate=rotate)
def testAlbum():
album = export_to_telegraph.getAlbum(urls[0])
print(album)
album_sender.send_v2(chat, album)
def testPicBot():
result = Result()
result.cap_html = 'test'
print(result)
r = album_sender.send_v2(chat, result, send_all=True, time_sleep=5)
print(r)
def test(url):
result = weibo_2_album.get(url)
album_sender.send_v2(chat, result)
def testAlbum():
album = title_finder.getAlbum(urls[0])
print(album)
album_sender.send_v2(chat, album)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import yaml
import tweepy
import twitter_2_album
import album_sender
from telegram.ext import Updater
with open('CREDENTIALS') as f:
credential = yaml.load(f, Loader=yaml.FullLoader)
tele = Updater(credential['bot'], use_context=True) # @twitter_send_bot
debug_group = tele.bot.get_chat(420074357)
auth = tweepy.OAuthHandler(credential['twitter_consumer_key'],
credential['twitter_consumer_secret'])
auth.set_access_token(credential['twitter_access_token'],
credential['twitter_access_secret'])
twitterApi = tweepy.API(auth)
# for status in twitterApi.user_timeline(id="bestsapphics"):
# print(status.id)
for status in twitterApi.search('女权'):
r = twitter_2_album.get(str(status.id))
album_sender.send_v2(debug_group, r)