from html_telegraph_poster import TelegraphPoster
import yaml
from telegram_util import matchKey, log_on_fail, log, tryDelete, autoDestroy, getBasicLog
import plain_db
from bs4 import BeautifulSoup
import album_sender
with open('token') as f:
tele = Updater(f.read().strip(), use_context=True)
debug_group = tele.bot.get_chat(420074357)
info_log = tele.bot.get_chat(-1001436325054)
no_auth_link_users = [-1001399998441] # prevent token leak through @web_record
no_source_link = plain_db.loadKeyOnlyDB('no_source_link')
remove_origin = plain_db.loadKeyOnlyDB('remove_origin')
with open('telegraph_tokens') as f:
telegraph_tokens = {}
for k, v in yaml.load(f, Loader=yaml.FullLoader).items():
telegraph_tokens[int(k)] = v
def saveTelegraphTokens():
with open('telegraph_tokens', 'w') as f:
f.write(yaml.dump(telegraph_tokens, sort_keys=True, indent=2))
def getSource(msg):
if msg.from_user:
import plain_db
import webgram
from telegram_util import matchKey, log_on_fail
from common import isSimplified, log_call, debug_group, sendDebugMessage
import time
blocklist = plain_db.loadKeyOnlyDB('blocklist')
channels = plain_db.loadLargeDB('channels', isIntValue = True, default = 100)
index = plain_db.loadLargeDB('index')
maintext = plain_db.loadLargeDB('maintext')
timestamp = plain_db.loadLargeDB('timestamp', isIntValue = True)
channelrefer = plain_db.loadKeyOnlyDB('channelrefer')
suspect = plain_db.loadKeyOnlyDB('suspect')
status = {}
def setBadWord(text):
blocklist.add(text)
def setChannelScore(text):
score = int(text.split()[-1])
text = text.split()[0].strip('/').split('/')[-1]
channels.update(text, score)
def updateChannel(name, referer):
referer_score = channels.get(referer)
if (referer_score < 0 or
channels.get(name) < referer_score + 1):
return
channels.update(name, referer_score + 1)
channelrefer.add(name + ':' + referer)
for key in list(result):
if isinstance(key, str) and matchKey(key, ['filter']):
result.remove(key)
final_result = []
for key in result:
if isinstance(key, int):
final_result = [key] + final_result
else:
final_result.append(key)
return final_result
def hasMasterFilter(self, chat_id):
return 'hasMasterFilter' in self._db[chat_id]
def hasUserFilter(self, chat_id):
return 'hasUserFilter' in self._db[chat_id]
def save(self):
with open('db/subscription', 'w') as f:
f.write(
yaml.dump(self._db,
sort_keys=True,
indent=2,
allow_unicode=True))
commitRepo(delay_minute=0)
blocklist = plain_db.loadKeyOnlyDB('blocklist')
existing = plain_db.loadKeyOnlyDB('existing')
log_existing = plain_db.loadKeyOnlyDB('log_existing')
subscription = Subscription()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import plain_db
# counter = plain_db.load('counter')
# counter.update('abc', 2)
# counter.inc('abc', 1)
# print(counter.get('abc')) # 3
# print(counter.get('ab')) # None
existing = plain_db.loadKeyOnlyDB('existing')
# print(existing.items())
# print(existing.add(1))
# print(existing.add(1))
# print(existing.add(2))
# existing.remove(2)
print(existing.items())
# f = plain_db.loadLargeDB('index1', isIntValue=True, default=5)
# print(f.items())
# f.update('1', 5)
# f.update('4', 1)
# f.update('3', 9)
# print(f.items())
# print(f.get('6'))
# plain_db.cleanupLargeDB('index1')
# f.save_dont_call_in_prod()
# rating = plain_db.loadLargeDB('rating', isIntValue = True, default = 0)
# rating.update('1', 0)
# rating.update('2', 1)
from telegram_util import matchKey, getDisplayUser, cnWordCount, isCN
import plain_db
allowlist = plain_db.loadKeyOnlyDB('allowlist')
kicklist = plain_db.loadKeyOnlyDB('kicklist')
mutelist = plain_db.loadKeyOnlyDB('mutelist')
all_chat_ids = plain_db.loadKeyOnlyDB('all_chat_ids')
blocklist = plain_db.LargeDB('blocklist', isIntValue=True)
def addBlocklist(text):
text = text.strip()
if not text or len(text) < 2:
return 'no action'
text = text.lower()
blocklist.update(text, 6)
return 'success'
def badTextScore(text):
if not text:
return 0, []
result = {}
for key, score in blocklist.items() + [(item, 10)
for item in kicklist.items()]:
if key.lower() in text.lower():
result[key] = score
return sum(result.values()), result
def badText(text):
import export_to_telegraph
from html_telegraph_poster import TelegraphPoster
import yaml
from telegram_util import matchKey, log_on_fail, log, tryDelete
import plain_db
with open('CREDENTIALS') as f:
CREDENTIALS = yaml.load(f, Loader=yaml.FullLoader)
tele = Updater(CREDENTIALS['bot_token'], use_context=True)
debug_group = tele.bot.get_chat(420074357)
no_auth_link_users = [-1001399998441]
no_source_link = plain_db.loadKeyOnlyDB('no_source_link')
with open('TELEGRAPH_TOKENS') as f:
TELEGRAPH_TOKENS = {}
for k, v in yaml.load(f, Loader=yaml.FullLoader).items():
TELEGRAPH_TOKENS[int(k)] = v
def saveTelegraphTokens():
with open('TELEGRAPH_TOKENS', 'w') as f:
f.write(yaml.dump(TELEGRAPH_TOKENS, sort_keys=True, indent=2))
def getSource(msg):
if msg.from_user:
return msg.from_user.id, msg.from_user.first_name, msg.from_user.username
return msg.chat_id, msg.chat.title, msg.chat.username
import plain_db
from telegram_util import isInt
import yaml
import weiboo
existing = plain_db.loadKeyOnlyDB('existing')
blocklist = plain_db.loadKeyOnlyDB('blocklist')
popularlist = plain_db.loadKeyOnlyDB('popularlist')
weibo_name = plain_db.load('weibo_name', isIntValue=False)
def searchUser(text):
for key, value in weibo_name.items.items():
if text in [key, value]:
return key, value
result = weiboo.searchUser(text)
if result:
weibo_name.update(result[0], result[1])
return result
def clearText(text):
text = text.split('?')[0]
return text.strip('/').split('/')[-1]
def getMatches(text):
if not text:
return []
text = clearText(text)
user = searchUser(text)
if user:
return user
return [text]
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from telegram_util import log_on_fail, matchKey, getMatchedKey
import twitter_2_album
import album_sender
from db import existing, blocklist
from common import debug_group, twitterApi, bot
from util import getHash, getCount
import plain_db
channel = bot.getChat(-1001401112463)
keywords = plain_db.loadKeyOnlyDB('keywords')
queue = [('hometimeline', 200)]
existed_keys = set([None])
def getSearchResult(key, count):
if key == 'hometimeline':
return twitterApi.home_timeline(count=count)
return twitterApi.user_timeline(user_id=key, count=count)
def shouldSendAlbum(channel, album):
thash = str(''.join(album.cap[:20].split())) + str(channel.id)
if not existing.add(thash):
return False
if album.video or album.imgs:
return True
return len(album.cap) > 20
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from telegram.ext import Updater, MessageHandler, Filters
from telegram_util import log_on_fail, splitCommand, matchKey
import random
import threading
import plain_db
with open('credential') as f:
token = f.read().strip()
tele = Updater(token, use_context=True) # @channel_push_bot
debug_group = tele.bot.get_chat(420074357)
channel_push = tele.bot.get_chat('@channel_push')
channels = plain_db.loadKeyOnlyDB('existing')
recent = plain_db.loadKeyOnlyDB('recent')
@log_on_fail(debug_group)
def handlePrivate(update, context):
msg = update.effective_message
if msg.chat.id != debug_group.id:
return
command, text = splitCommand(msg.text)
if matchKey(command, ['remove']):
result = channels.remove(text)
msg.reply_text('Removed ' + str(result))
return
count = 0
for piece in msg.text.split():
import telepost
from telegram_util import getWid, matchKey
import time
import cached_url
import yaml
import random
import plain_db
import os
blocklist = plain_db.loadKeyOnlyDB('blocklist')
DAYS = 365
channels = [
'daily_feminist', 'freedom_watch', 'queer_liberation',
'life_with_disabilities', 'labor_one'
]
freq_count = {}
name = {}
description = {}
def process(status):
user = status.get('user')
if not user:
return
user_id = user.get('id')
if not user_id:
return
name[user_id] = user.get('screen_name')
description[user_id] = user.get('description').encode(
import web_2_album
import weibo_2_album
import twitter_2_album
import album_sender
from bs4 import BeautifulSoup
import plain_db
import threading
with open('CREDENTIALS') as f:
CREDENTIALS = yaml.load(f, Loader=yaml.FullLoader)
tele = Updater(CREDENTIALS['bot_token'], use_context=True)
debug_group = tele.bot.get_chat(420074357)
info_log = tele.bot.get_chat(-1001439828294)
waitlist_log = tele.bot.get_chat(-1001345995889)
remove_origin = plain_db.loadKeyOnlyDB('remove_origin')
def getUrl(msg):
if matchKey(msg.text_html_urled, ['source</a>']):
return
if (matchKey(msg.text_html_urled,
['mp.weixin.qq.com', 'telegra.ph'])
and msg.chat.username == 'web_record'):
return
soup = BeautifulSoup(msg.text_html_urled, 'html.parser')
for item in soup.find_all('a'):
if 'http' in item.get('href'):
return item.get('href')
def getResult(url, text, origin):
ranks = [web_2_album]