async def run():
removeOldFiles('tmp', day=0.1)
for channel in credential['channels']:
for album, post in getPosts(channel):
if existing.get(album.url):
continue
status_text = getText(album, post) or album.url
if not matchLanguage(channel, status_text):
continue
if len(status_text) > 280:
continue
existing.update(album.url, -1) # place holder
if 'debug' in sys.argv:
print(album.url)
result = await post_twitter(channel, post, album, status_text)
if not result:
continue
existing.update(album.url, result.id)
if 'debug' in sys.argv:
print('https://twitter.com/%s/status/%d' %
(credential['channels'][channel]['twitter_user'],
result.id))
if 'client' in client_cache:
await client_cache['client'].disconnect()
return # only send one item every 10 minute
def loopImp():
removeOldFiles('tmp', day=0.1)
if not scheduled_key:
for key in subscription.subscriptions():
scheduled_key.append(key)
random.shuffle(scheduled_key)
process(scheduled_key.pop())
def loopImp():
removeOldFiles('tmp', day=0.1) # video could be very large
global processed_channels
processed_channels = set()
for key in subscription.subscriptions():
if random.random() > 0.05:
continue
process(key)
def indexClean():
removeOldFiles('tmp', day=2)
os.system('rm tmp/*embed*')
removeNonExistChannel()
cleanupSuspectAndOld()
cleanupRedundant()
save()
rescore()
def sendDailyRead(msg, method=yieldDailyRead):
removeOldFiles('tmp', day=0.1)
if 'force' in msg.text:
removeOldFiles('tmp', day=0)
try:
item_limit = int(msg.text.split()[-1])
except:
item_limit = Limit
try:
offset = int(msg.text.split()[-2])
except:
offset = 0
tele.bot.send_message(msg.chat_id,
getDailyRead(method, item_limit, offset),
disable_web_page_preview=True)
def loopImp():
removeOldFiles('tmp', day=0.1)
sg.reset()
db.reload()
for keyword in db.keywords.items:
print(keyword)
content_id = urllib.request.pathname2url('100103type=1&q=' + keyword)
url = 'https://m.weibo.cn/api/container/getIndex?containerid=%s&page_type=searchall' % content_id
process(url)
for user in db.users.items:
print(user)
url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=%s&containerid=107603%s' \
% (user, user)
process(url)
commitRepo(delay_minute=0)
print('loop finished. commit in thread.')
def loopImp():
removeOldFiles('tmp')
global processed_channels
processed_channels = set()
channels = list(subscription.getChannels())
for key in subscription.keys():
if key != 'hometimeline' and isinstance(key, str) and random.random() > 0.1:
continue
for status in getStatuses(key):
sent = False
for channel in channels:
if shouldProcess(channel, status, key):
try:
album = twitter_2_album.get(str(status.id),
origin = [str(channel.id), channel.username])
if shouldSendAlbum(channel, album):
album_sender.send_v2(channel, album)
sent = True
except Exception as e:
print('send fail', channel.id, str(e), status.id)
log(key, status, sent)
def loopImp():
removeOldFiles('tmp', day=0.1)
for user_id in db.sub.subscriptions():
channels = list(db.sub.channels(user_id, tele.bot))
for note in getNotes(user_id):
processNote(note, channels)