def __init__(self):
self.max_capacity = 200 * UNITS_MAPPING['MB']
self.max_clear_size = 100 * UNITS_MAPPING['MB']
assert self.max_capacity >= self.max_clear_size, "max_capacity cannot be less than max_clear_size"
self.poll_interval = 10
self.compressed_resource_types = {".zip", ".tar.gz", ".tar"}
self.db = Resource._meta.database
self.db.connect()
self.db.create_tables([Resource], safe=True)
self.db.close()
self.kv = KeyValue(database=self.db, value_field=BigIntegerField())
with self.db.atomic('EXCLUSIVE'):
# initializing current_size of DATA_CATALOG_DOWNLOAD_DIR in the database
if 'current_size' not in self.kv:
self.kv['current_size'] = sum(
f.stat().st_size
for f in Path(DATA_CATALOG_DOWNLOAD_DIR).rglob('*'))
else:
for resource in Resource.select():
if resource.is_downloading:
continue
path = Path(
os.path.join(DATA_CATALOG_DOWNLOAD_DIR,
str(resource.resource_id) + '.dat'))
if not path.exists():
path = Path(
os.path.join(DATA_CATALOG_DOWNLOAD_DIR,
str(resource.resource_id)))
if not path.exists():
self.kv['current_size'] -= resource.size
resource.delete_instance()
def __init__(self,
api_key: str,
lang: Mapping[str, str],
theme_image_ids: tuple[str],
keyboards: Mapping[str, InlineKeyboardMarkup],
guesslang_syntaxes: Mapping[str, str],
*args: Any,
admin_chat_id: Optional[str] = None,
db_path: str = str(
local.path(__file__).up() / 'user_themes.sqlite'),
**kwargs: Any):
self.lang = lang
self.theme_image_ids = theme_image_ids
self.kb = keyboards
self.guesslang_syntaxes = guesslang_syntaxes
self.admin_chat_id = admin_chat_id
self.db_path = db_path
self.user_themes = KeyValue(key_field=IntegerField(primary_key=True),
value_field=CharField(),
database=APSWDatabase(db_path))
self.log = mk_logger()
self.bot = TeleBot(api_key, *args, **kwargs)
self.register_handlers()
self.guesser = Guess()
EPG_PROTOCOL = env('EPG_PROTOCOL', default='http')
AWS_REGION_NAME = env('AWS_REGION_NAME')
AWS_ACCESS_KEY_ID = env('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = env('AWS_SECRET_ACCESS_KEY')
AWS_STORAGE_BUCKET_NAME = env('AWS_STORAGE_BUCKET_NAME')
AWS_S3_PREFIX = env('AWS_S3_PREFIX', default='')
DIRECTORY = env('DIRECTORY', default=env('PWD'))
DATABASE_PATH = env('DATABASE_PATH',
default=f'{DIRECTORY}/epg_downloader.db')
AWS_S3_ENDPOINT_URL = env(
'AWS_S3_ENDPOINT_URL',
'https://{}.digitaloceanspaces.com'.format(AWS_REGION_NAME),
)
CDN_ENDPOINT_URL = env(
'CDN_ENDPOINT_URL',
'https://{}.{}.cdn.digitaloceanspaces.com'.format(
AWS_STORAGE_BUCKET_NAME, AWS_REGION_NAME),
)
KEY_PREFIX = 'epgd'
database = SqliteExtDatabase(
settings.DATABASE_PATH,
pragmas=(
('cache_size', -1024 * 4), # 4MB page-cache.
('journal_mode', 'wal'), # Use WAL-mode (you should always use this!).
('foreign_keys', 1) # Enforce foreign-key constraints.
))
kv_store = KeyValue(database=database)
def create_kv(self, **kwargs):
kv = KeyValue(database=self.database, **kwargs)
self._kvs.append(kv)
return kv
class ColorCodeBot:
def __init__(self,
api_key: str,
lang: Mapping[str, str],
theme_image_ids: tuple[str],
keyboards: Mapping[str, InlineKeyboardMarkup],
guesslang_syntaxes: Mapping[str, str],
*args: Any,
admin_chat_id: Optional[str] = None,
db_path: str = str(
local.path(__file__).up() / 'user_themes.sqlite'),
**kwargs: Any):
self.lang = lang
self.theme_image_ids = theme_image_ids
self.kb = keyboards
self.guesslang_syntaxes = guesslang_syntaxes
self.admin_chat_id = admin_chat_id
self.db_path = db_path
self.user_themes = KeyValue(key_field=IntegerField(primary_key=True),
value_field=CharField(),
database=APSWDatabase(db_path))
self.log = mk_logger()
self.bot = TeleBot(api_key, *args, **kwargs)
self.register_handlers()
self.guesser = Guess()
def register_handlers(self):
self.welcome = self.bot.message_handler(commands=['start', 'help'])(
self.welcome)
self.browse_themes = self.bot.message_handler(
commands=['theme', 'themes'])(self.browse_themes)
self.mk_theme_previews = self.bot.message_handler(
commands=['previews'])(self.mk_theme_previews)
self.intake_snippet = self.bot.message_handler(
func=lambda m: m.content_type == 'text')(self.intake_snippet)
self.recv_photo = self.bot.message_handler(content_types=['photo'])(
self.recv_photo)
self.restore_kb = self.bot.callback_query_handler(
lambda q: yload(q.data)['action'] == 'restore')(self.restore_kb)
self.set_snippet_filetype = self.bot.callback_query_handler(
lambda q: yload(q.data)['action'] == 'set ext')(
self.set_snippet_filetype)
self.set_theme = self.bot.callback_query_handler(
lambda q: yload(q.data)['action'] == 'set theme')(self.set_theme)
self.send_photo_elsewhere = self.bot.inline_handler(
lambda q: q.query.startswith("img "))(self.send_photo_elsewhere)
self.switch_from_inline = self.bot.inline_handler(lambda q: True)(
self.switch_from_inline)
@retry
def switch_from_inline(self, inline_query: InlineQuery):
self.log.msg("receiving inline query",
user_id=inline_query.from_user.id,
user_first_name=inline_query.from_user.first_name,
query=inline_query.query)
self.bot.answer_inline_query(
inline_query.id, [],
switch_pm_text=self.lang['switch to direct'],
switch_pm_parameter='x')
@retry
def welcome(self, message: Message):
self.log.msg("introducing myself",
user_id=message.from_user.id,
user_first_name=message.from_user.first_name,
chat_id=message.chat.id)
self.bot.reply_to(
message,
self.lang['welcome'],
reply_markup=ForceReply(
input_field_placeholder=self.lang['input field placeholder']))
@retry
def mk_theme_previews(self, message: Message):
if not self.admin_chat_id or str(
message.chat.id) != self.admin_chat_id:
self.log.msg("naughty preview attempt",
user_id=message.from_user.id,
user_first_name=message.from_user.first_name,
chat_id=message.chat.id,
admin_chat_id=self.admin_chat_id)
return
sample_code = dedent("""
# palinDay :: Int -> [ISO Date]
def palinDay(y):
'''A possibly empty list containing the palindromic
date for the given year, if such a date exists.
'''
s = str(y)
r = s[::-1]
iso = '-'.join([s, r[0:2], r[2:]])
try:
datetime.strptime(iso, '%Y-%m-%d')
return [iso]
except ValueError:
return []
""")
for button in chain.from_iterable(self.kb['theme'].keyboard):
theme = button.text
html = mk_html(f"# {theme}{sample_code}", 'py', theme)
with local.tempdir() as folder:
png_path = mk_png(html, folder=folder)
send_image(bot=self.bot,
chat_id=message.chat.id,
png_path=png_path,
reply_msg_id=message.message_id)
@retry
def browse_themes(self, message: Message):
self.log.msg("browsing themes",
user_id=message.from_user.id,
user_first_name=message.from_user.first_name,
chat_id=message.chat.id)
albums = [
self.theme_image_ids[i:i + 10]
for i in range(0, len(self.theme_image_ids), 10)
]
for album in albums:
self.bot.send_media_group(message.chat.id,
map(InputMediaPhoto, album),
reply_to_message_id=message.message_id)
self.bot.reply_to(message,
self.lang['select theme'],
reply_markup=self.kb['theme'])
@retry
def set_theme(self, cb_query: CallbackQuery):
data = yload(cb_query.data)
user = cb_query.message.reply_to_message.from_user
self.log.msg("setting theme",
user_id=user.id,
user_first_name=user.first_name,
theme=data['theme'],
chat_id=cb_query.message.chat.id)
self.bot.edit_message_reply_markup(cb_query.message.chat.id,
cb_query.message.message_id,
reply_markup=minikb('theme'))
self.user_themes[user.id] = data['theme']
self.bot.answer_callback_query(
cb_query.id,
text=self.lang['acknowledge theme'].format(data['theme']))
if self.admin_chat_id:
with open(self.db_path, 'rb') as doc:
self.bot.send_document(self.admin_chat_id, doc)
def guess_ext(self,
code: str,
probability_min: float = .12) -> Optional[str]:
syntax, probability = self.guesser.probabilities(code)[0]
ext = self.guesslang_syntaxes.get(syntax)
self.log.msg("guessed syntax",
probability_min=probability_min,
probability=probability,
syntax=syntax,
ext=ext)
if probability >= probability_min:
return ext
for start, ext in {
'{': 'json',
'---\n': 'yaml',
'[[': 'toml',
'[': 'ini',
'<?php': 'php',
'<': 'xml',
'-- ': 'lua'
}.items():
if code.startswith(start):
return ext
@retry
def intake_snippet(self, message: Message):
self.log.msg("receiving code",
user_id=message.from_user.id,
user_first_name=message.from_user.first_name,
chat_id=message.chat.id)
ext = self.guess_ext(message.text)
if ext:
kb_msg = self.bot.reply_to(
message,
f"{self.lang['query ext']}\n\n{self.lang['guessed syntax'].format(ext)}",
reply_markup=minikb('syntax', self.lang['syntax picker']),
parse_mode='Markdown',
disable_web_page_preview=True)
self.set_snippet_filetype(cb_query=None,
query_message=kb_msg,
ext=ext)
else:
self.bot.reply_to(message,
self.lang['query ext'],
reply_markup=self.kb['syntax'],
parse_mode='Markdown',
disable_web_page_preview=True)
@retry
def send_photo_elsewhere(self, inline_query: InlineQuery):
file_id = inline_query.query.split('img ', 1)[-1]
self.log.msg("creating inline query result",
file_id=file_id,
file_info=self.bot.get_file(file_id))
self.bot.answer_inline_query(inline_query.id, [
InlineQueryResultCachedPhoto(
id=str(uuid4()), photo_file_id=file_id, title="Send Image")
],
is_personal=True)
@retry
def restore_kb(self, cb_query: CallbackQuery):
data = yload(cb_query.data)
self.bot.edit_message_reply_markup(
cb_query.message.chat.id,
cb_query.message.message_id,
reply_markup=self.kb[data['kb_name']])
self.bot.answer_callback_query(cb_query.id)
@retry
def set_snippet_filetype(self,
cb_query: Optional[CallbackQuery] = None,
query_message: Optional[Message] = None,
ext: Optional[str] = None):
if cb_query:
query_message = cb_query.message
ext = yload(cb_query.data)['ext']
elif not (query_message and ext):
raise Exception(
"Either cb_query or both query_message and ext are required")
self.log.msg("colorizing code",
user_id=query_message.reply_to_message.from_user.id,
user_first_name=query_message.reply_to_message.from_user.
first_name,
syntax=ext,
chat_id=query_message.chat.id)
if cb_query:
self.bot.edit_message_reply_markup(query_message.chat.id,
query_message.message_id,
reply_markup=minikb(
'syntax',
self.lang['syntax picker']))
snippet = query_message.reply_to_message
theme = self.user_themes.get(snippet.from_user.id,
'base16/gruvbox-dark-hard')
html = mk_html(snippet.text, ext, theme)
send_html(bot=self.bot,
chat_id=snippet.chat.id,
html=html,
reply_msg_id=snippet.message_id)
with local.tempdir() as folder:
png_path = mk_png(html, folder=folder)
did_send = False
if len(snippet.text.splitlines()) <= 30:
try:
photo_msg = send_image(bot=self.bot,
chat_id=snippet.chat.id,
png_path=png_path,
reply_msg_id=snippet.message_id)
except ApiException as e:
self.log.error("failed to send compressed image",
exc_info=e,
chat_id=snippet.chat.id)
else:
did_send = True
kb_to_chat = InlineKeyboardMarkup()
kb_to_chat.add(
InlineKeyboardButton(
self.lang['send to chat'],
switch_inline_query=
f"img {photo_msg.photo[-1].file_id}"))
self.bot.edit_message_reply_markup(photo_msg.chat.id,
photo_msg.message_id,
reply_markup=kb_to_chat)
if not did_send:
send_image(bot=self.bot,
chat_id=snippet.chat.id,
png_path=png_path,
reply_msg_id=snippet.message_id,
compress=False)
if cb_query:
self.bot.answer_callback_query(cb_query.id)
def recv_photo(self, message: Message):
self.log.msg('received photo',
file_id=message.photo[0].file_id,
user_id=message.from_user.id,
user_first_name=message.from_user.first_name,
chat_id=message.chat.id)