def handle_delete_last_command(update: Update, context: CallbackContext):
chat_id = update.message.chat_id
queue_size = redis.get(f'chat:{chat_id}:queue_size')
if queue_size is None:
queue_size = 0
queue_size = int(queue_size)
if queue_size <= 0:
context.bot.send_message(chat_id=chat_id, text='Queue is empty')
return
queue_size -= 1
tweet_text = redis.get(f'chat:{chat_id}:queue:{queue_size}:text') or ''
tg_attachment_id = redis.get(
f'chat:{chat_id}:queue:{queue_size}:tg_attachment_id')
redis.delete(f'chat:{chat_id}:queue:{queue_size}:text')
redis.delete(f'chat:{chat_id}:queue:{queue_size}:tg_attachment_id')
redis.set(f'chat:{chat_id}:queue_size', queue_size)
context.bot.send_message(
chat_id=chat_id,
text="I've deleted your latest tweet. This was the text: " +
tweet_text)
if tg_attachment_id:
context.bot.send_message(chat_id=chat_id,
text='It also had an attachment')
def get(self, me):
"""Who's pinterested in me?"""
if 'unread' in request.values:
key = 'users:%s:match:unread' % me
unread = list(redis.smembers(key))
redis.delete(key)
return unread
else:
return list(redis.smembers('users:%s:match' % me))
def get(self, me):
"""Who's pinterested in me?"""
if 'unread' in request.values:
key = 'users:%s:match:unread' % me
unread = list(redis.smembers(key))
redis.delete(key)
return unread
else:
return list(redis.smembers('users:%s:match' % me))
def resetRedisQueues():
redis = common.redis.get_redis_queue_conn()
# Since we restarted, clear the url queues.
print("Flushing redis queues")
dropped = 0
for key in redis.scan_iter(match="*"):
redis.delete(key)
dropped += 1
print("Queues flushed. Deleted %s netlocs" % dropped)
def delete(self, user):
"""Delete a single user."""
redis.srem('users', user)
sets = ['sex_am', 'sex_want', 'zip', 'zipshort', 'birthday', 'birthyear']
for key in sets:
value = redis.get('users:%s:%s' % (user, key))
if value is not None:
redis.srem('%s:%s' % (key, value), user)
keys = ['sex_am', 'sex_want', 'ugos', 'zip', 'zipshort', 'birthday', 'birthyears']
for key in keys:
redis.delete('users:%s:%s' % (user, key))
redis.zrem('birthyears', user)
return Response()
def delete(self, user):
"""Delete a single user."""
redis.srem('users', user)
sets = [
'sex_am', 'sex_want', 'zip', 'zipshort', 'birthday', 'birthyear'
]
for key in sets:
value = redis.get('users:%s:%s' % (user, key))
if value is not None:
redis.srem('%s:%s' % (key, value), user)
keys = [
'sex_am', 'sex_want', 'ugos', 'zip', 'zipshort', 'birthday',
'birthyears'
]
for key in keys:
redis.delete('users:%s:%s' % (user, key))
redis.zrem('birthyears', user)
return Response()
def loop():
now = datetime.now()
logging.debug(f'Running with timestamp {now}')
for key in redis.keys('*:settings:tweet_time'):
chat_id = key.split(':')[1]
desired_time = redis.get(key).split(':')
desired_hour = int(desired_time[0])
desired_minute = int(desired_time[1])
local_now = now.astimezone(timezone(redis.get(f'chat:{chat_id}:settings:timezone') or 'UTC'))
if desired_hour != local_now.hour or desired_minute != local_now.minute:
continue
queue_size = redis.get(f'chat:{chat_id}:queue_size') or 0
queue_size = int(queue_size)
if queue_size <= 0:
continue
queue_size -= 1
tweet_text = redis.get(f'chat:{chat_id}:queue:0:text')
tg_attachment_id = redis.get(f'chat:{chat_id}:queue:0:tg_attachment_id')
twitter = get_twitter_api(chat_id)
if not tg_attachment_id:
try:
status = twitter.update_status(tweet_text)
except tweepy.error.TweepError as e:
logging.warning(f'Unable to tweet for chat:{chat_id}:queue:{queue_size} (without attachment) '
f'Reason: {e.reason}')
telegram_updater.bot.send_message(chat_id=chat_id, text=e.reason)
telegram_updater.bot.send_message(chat_id=chat_id, text='Sorry, I was unable to post your daily tweet. '
'This is your tweet:')
telegram_updater.bot.send_message(chat_id=chat_id, text=tweet_text)
telegram_updater.bot.send_message(chat_id=chat_id,
text='You may delete it from the queue: /delete_last')
return
else:
# download telegram photo
logging.debug('Downloading telegram attachment')
file = telegram_updater.bot.getFile(tg_attachment_id)
filename = FILE_STORAGE_PATH / tg_attachment_id
file.download(filename)
try:
status = twitter.update_with_media(filename, tweet_text)
except tweepy.error.TweepError as e:
logging.warning(f'Unable to tweet for chat:{chat_id}:queue:{queue_size} (with attachment). '
f'Reason: {e.reason}')
telegram_updater.bot.send_message(chat_id=chat_id, text=e.reason)
telegram_updater.bot.send_message(chat_id=chat_id,
text='Sorry, I was unable to post your daily tweet. '
'This is your tweet, and it contained one attachment:')
telegram_updater.bot.send_message(chat_id=chat_id, text=tweet_text)
telegram_updater.bot.send_message(chat_id=chat_id,
text='You may delete it from the queue: /delete_last')
return
finally:
filename.unlink(missing_ok=True)
logging.debug('Deleting stored tweet and attachment id')
# Move all elements in the queue one
for i in range(queue_size):
k0 = f'chat:{chat_id}:queue:{i}:text'
k1 = f'chat:{chat_id}:queue:{i+1}:text'
l0 = f'chat:{chat_id}:queue:{i}:tg_attachment_id'
l1 = f'chat:{chat_id}:queue:{i + 1}:tg_attachment_id'
redis.set(k0, redis.get(k1) or '')
redis.set(l0, redis.get(l1) or '')
# The last element is now duplicated; delete it
redis.delete(f'chat:{chat_id}:queue:{queue_size}:text')
redis.delete(f'chat:{chat_id}:queue:{queue_size}:tg_attachment_id')
tweet_url = build_tweet_url(status)
logging.info(f'Tweeted: {tweet_url} for chat_id {chat_id}')
telegram_updater.bot.send_message(chat_id=chat_id, text=f'I just tweeted this: {tweet_url}\n'
f'\n'
f'Tweets in queue: {queue_size}')
if queue_size <= 0:
telegram_updater.bot.send_message(chat_id=chat_id, text="Your queue is now empty. I will not tweet "
"tomorrow if you won't give me new stuff!")
redis.set(f'chat:{chat_id}:queue_size', queue_size)
def mq_clean():
collections = [
'stats_mq',
'stats_batch_run',
'stats_batch_stage',
'mq_spider_undo',
'mq_spider_ready',
'mq_spider_doing',
'mq_spider_done',
'mq_mirror_undo',
'mq_mirror_ready',
'mq_mirror_doing',
'mq_mirror_done',
'mq_piping_undo',
'mq_piping_ready',
'mq_piping_doing',
'mq_piping_done',
'mq_notify_undo',
'mq_notify_ready',
'mq_notify_doing',
'mq_notify_done',
'mq_snapshot_undo',
'mq_snapshot_ready',
'mq_snapshot_doing',
'mq_snapshot_done',
]
for key in collections:
mongoMq[key].drop()
for i in ['execute', 'spider', 'mirror', 'piping', 'notify', 'snapshot']:
key = "mq_%s_ready" % i
redis.delete(key)
mongoMq['stats_mq'].insert({
'mq_key': 'execute',
'undo': 0,
'ready': 0,
'doing': 0,
'done': 0
})
mongoMq['stats_mq'].insert({
'mq_key': 'spider',
'undo': 0,
'ready': 0,
'doing': 0,
'done': 0
})
mongoMq['stats_mq'].insert({
'mq_key': 'mirror',
'undo': 0,
'ready': 0,
'doing': 0,
'done': 0
})
mongoMq['stats_mq'].insert({
'mq_key': 'notify',
'undo': 0,
'ready': 0,
'doing': 0,
'done': 0
})
mongoMq['stats_mq'].insert({
'mq_key': 'piping',
'undo': 0,
'ready': 0,
'doing': 0,
'done': 0
})
def clean_all():
#危险，慎用
return True
#db.exec("TRUNCATE TABLE app CASCADE;")
for table in [
'task', 'task_execute', 'site', 'domain', 'setting', 'proxy',
'task_notify', 'task_piping', 'dk_black_list', 'dk_white_list',
'dk_filterword', 'task_piping_result', 'task_piping_snapshot',
'spider_url', 'spiderjs_url', 'piping_extend', 'scheduler'
]:
db.exec("delete from %s;" % table)
for seq in [
'app_id_seq', 'task_id_seq', 'task_execute_id_seq', 'site_id_seq',
'domain_id_seq', 'setting_id_seq', 'dk_black_list_id_seq',
'dk_filterword_id_seq', 'dk_white_list_id_seq', 'proxy_id_seq',
'task_notify_id_seq', 'task_piping_id_seq',
'task_piping_result_id_seq', 'task_piping_snapshot_id_seq',
'spider_url_id_seq', 'piping_extend_id_seq'
]:
db.exec("ALTER SEQUENCE %s RESTART WITH 1;" % seq)
itemSpider = {
'spiderurl': {
'unique': ['id'],
'index': ['task_id', 'execute_id']
},
'execute': {
'unique': ['id'],
'index': ['task_id']
},
'static': {
'unique': ['domain', 'md5_body'],
'index': ['domain', 'md5_url', 'md5_body']
},
'autoids': {
'unique': ['name'],
'index': []
},
'parse': {
'unique': [],
'index': ['task_id', 'execute_id', 'md5_url']
},
'snapshot': {
'unique': ['id'],
'index': ['app_key', 'batch']
},
'outlink': {
'unique': [],
'index': ['domain', 'md5_referer', 'md5_url', 'md5_body', 'date']
},
}
for key, item in itemSpider.items():
mongoSpider[key].drop()
if item['unique']:
uniqueField = [(field, 1) for field in item['unique']]
mongoSpider[key].ensure_index(uniqueField, unique=True)
if item['index']:
for field in item['index']:
mongoSpider[key].ensure_index(field)
itemMq = {
'mq_spider_undo': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_spider_ready': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_spider_doing': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_spider_done': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_mirror_undo': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_mirror_ready': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_mirror_doing': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_mirror_done': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_piping_undo': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_piping_ready': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_piping_doing': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_piping_done': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_notify_undo': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_notify_ready': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_notify_doing': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_notify_done': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_snapshot_undo': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_snapshot_ready': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_snapshot_doing': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'mq_snapshot_done': {
'unique': ['mq_id'],
'index': ['mq_batch']
},
'stats_mq': {
'unique': ['mqkey'],
'index': []
},
'stats_batch_run': {
'unique': ['mqkey', 'batch'],
'index': []
},
'stats_batch_stage': {
'unique': ['mqkey', 'batch'],
'index': []
},
'process_list': {
'unique': ['hostname', 'title'],
'index': ['mqkey', 'status']
},
# 'process_config': {'unique':['hostname'], 'index':[]},
}
for key, item in itemMq.items():
mongoMq[key].drop()
if item['unique']:
uniqueField = [(field, 1) for field in item['unique']]
mongoMq[key].ensure_index(uniqueField, unique=True)
if item['index']:
for field in item['index']:
mongoMq[key].ensure_index(field)
for key in ['spider', 'mirror', 'piping', 'notify', 'snapshot']:
redis.delete('mq_%s_ready' % key)