def test__get_document_id(self):
revision = Revision(
id=1,
content='hello',
timestamp=datetime(2020, 1, 1, 1, 1, 1),
document_id=5
)
assert revision.get_document_id() == 5
def is_new(self):
# today = datetime.date.today()
# delta = datetime.timedelta(days=settings.BOT_CONSIDERED_NEW)
# result = today - self.date_added <= delta
# return result
return self.revision >= Revision.get_instance(
).nr - settings.BOT_CONSIDERED_NEW + 1
def load_revs(title, revid):
url = 'https://ru.wikipedia.org/w/api.php?action=query&prop=revisions&rvstartid=' + revid \
+ '&rvprop=flags|timestamp|user|comment|ids|flagged|userid&titles=' + title \
+ '&rvlimit=50&format=json'
res = requests.get(url)
response = next(iter(res.json()["query"]["pages"].values())) # type: dict
raw_revs = response['revisions']
if len(raw_revs) == 1:
raise SkippedException()
revs = [
Revision(
id=x['revid'],
timestamp=parse_mw_date(
x['timestamp']
), #datetime.strptime(x['timestamp']+" UTC", '%Y-%m-%dT%H:%M:%SZ %z'),
comment=x['comment'] if 'comment' in x else '',
minor='minor' in x,
flagged='flagged' in x,
text='',
user=User(id=None if 'userid' not in x or x['userid'] == 0 else
x['userid'],
name=None if 'user' not in x else x['user'],
flags=(users.get_flags(x['userid']) or [])
if 'userid' in x else [])) for x in raw_revs
] # type: [Revision]
revs.reverse()
last_revision = revs[-1] # type: Revision
if 'bot' in last_revision.user.flags \
or PageProcessor.is_trusted(last_revision.user.flags):
raise TrustedUserException()
last_trusted = None
last_flagged = None
for rev in reversed(revs):
if last_trusted is None:
if PageProcessor.is_trusted(rev.user.flags) or rev.flagged:
last_trusted = rev
if last_flagged is None:
if rev.flagged:
last_flagged = rev
session_start = RevTools.session_start(revs)
return {
"page": {
"id": response['pageid'],
"title": title,
"ns": response['ns']
},
'revs': [x.to_object() for x in revs],
'last_trusted': None if last_trusted is None else last_trusted.id,
'last_flagged':
None if last_flagged is None else last_flagged.to_object(),
'session_start': session_start.id
}
def many_by_usernames(names: List):
results = Bot.select().where(
(fn.lower(Bot.username) << [n.lower() for n in names])
& (Bot.revision <= Revision.get_instance().nr)
& (Bot.approved == True) & (Bot.disabled == False))
if results:
return results
raise Bot.DoesNotExist
def build_document(self, data_dict: dict) -> None:
"""
Build a Document from the data passed in.
:param data_dict: dict
"""
self._validate_data(data_dict)
document = self.document_repository.get_by_title(data_dict['title'])
if document:
new_revision = Revision(content=data_dict['content'])
document.revisions.append(new_revision)
self.document_repository.save(document)
else:
revision = Revision(content=data_dict['content'])
document = Document(title=data_dict['title'], revisions=[revision])
self.document_repository.save(document)
def test__get_latest_revision(self):
timestamp = datetime(2020, 1, 1, 1, 1, 1)
timestamp_latest = datetime(2020, 2, 1, 1, 1, 1)
revision = Revision(id=1,
content='hello',
timestamp=timestamp,
document_id=5)
revision_latest = Revision(id=2,
content='hello again',
timestamp=timestamp_latest,
document_id=5)
document = Document(id=5,
title='blah',
revisions=[revision, revision_latest])
assert document.get_latest_revision() == revision_latest
def update_categories(self, categories: List[Category]):
self.notify_admin(
"Updating BotList categories to Revision {}...".format(
Revision.get_instance().nr))
for cat in categories:
text = _format_category_bots(cat)
log.info(f"Updating category {cat.name}...")
msg = self.send_or_edit(text, cat.current_message_id)
if msg:
cat.current_message_id = msg.message_id
self.sent['category'].append("{} {}".format(
'Resent' if self.resend else 'Updated', cat))
cat.save()
self._save_channel()
# Add "share", "up", and "down" buttons
for i in range(0, len(categories)):
buttons = list()
if i > 0:
# Not first category
# Add "Up" button
buttons.append(
InlineKeyboardButton(
"🔺",
url=BotList.create_hyperlink(
categories[i - 1].current_message_id)))
buttons.append(
InlineKeyboardButton("Share",
url="https://t.me/{}?start={}".format(
settings.SELF_BOT_NAME,
categories[i].id)))
if i < len(categories) - 1:
# Not last category
buttons.append(
InlineKeyboardButton(
"🔻",
url=BotList.create_hyperlink(
categories[i + 1].current_message_id)))
reply_markup = InlineKeyboardMarkup([buttons])
log.info(
f"Adding buttons to message with category {categories[i].name}..."
)
self.bot.edit_message_reply_markup(
self.channel.chat_id,
categories[i].current_message_id,
reply_markup=reply_markup,
timeout=60)
def test__gt_lt(self):
timestamp_lesser = datetime(2020, 1, 1, 1, 1, 1)
timestamp_greater = datetime(2020, 1, 1, 1, 1, 2)
revision_lesser = Revision(
id=1,
content='hello',
timestamp=timestamp_lesser,
document_id=5
)
revision_greater = Revision(
id=2,
content='hello',
timestamp=timestamp_greater,
document_id=5
)
assert revision_greater > revision_lesser
assert revision_lesser < revision_greater
def test__get_revision_by_timestamp(self):
timestamp = datetime(2020, 1, 1, 1, 1, 1)
revision = Revision(id=1,
content='hello',
timestamp=timestamp,
document_id=5)
document = Document(id=5, title='blah', revisions=[revision])
assert document.get_revision_by_timestamp(timestamp) == revision
def test__get_revision_by_timestamp_expect_most_recent(self):
timestamp = datetime(2020, 1, 1, 1, 1, 1)
timestamp_latest = datetime(2020, 2, 1, 1, 1, 1)
timestamp_in_between = datetime(2020, 1, 29, 1, 1, 1)
revision = Revision(id=1,
content='hello',
timestamp=timestamp,
document_id=5)
revision_latest = Revision(id=2,
content='hello again',
timestamp=timestamp_latest,
document_id=5)
document = Document(id=5,
title='blah',
revisions=[revision, revision_latest])
assert document.get_revision_by_timestamp(
timestamp_in_between) == revision
def test__ge_le(self):
timestamp = datetime(2020, 1, 1, 1, 1, 1)
timestamp_equal = datetime(2020, 1, 1, 1, 1, 1)
revision = Revision(
id=1,
content='hello',
timestamp=timestamp,
document_id=5
)
revision_equal = Revision(
id=2,
content='hello',
timestamp=timestamp_equal,
document_id=5
)
assert revision >= revision_equal
assert revision_equal >= revision
assert revision <= revision_equal
assert revision_equal <= revision
def search_bots(query):
query = query.lower().strip()
split = query.split(' ')
# easter egg
if query in ('awesome bot', 'great bot', 'superb bot', 'best bot', 'best bot ever'):
return [Bot.by_username('@botlistbot')]
# exact results
where_query = (
(fn.lower(Bot.username).contains(query) |
fn.lower(Bot.name) << split |
fn.lower(Bot.extra) ** query) &
(Bot.revision <= Revision.get_instance().nr &
Bot.approved == True & Bot.disabled == False)
)
results = set(Bot.select().distinct().where(where_query))
# keyword results
keyword_results = Bot.select(Bot).join(Keyword).where(
(fn.lower(Keyword.name) << split) &
(Bot.revision <= Revision.get_instance().nr) &
(Bot.approved == True & Bot.disabled == False)
)
results.update(keyword_results)
# many @usernames
usernames = re.findall(settings.REGEX_BOT_ONLY, query)
if usernames:
try:
bots = Bot.many_by_usernames(usernames)
print([b.username for b in bots])
results.update(bots)
except Bot.DoesNotExist:
pass
return list(results)
def send_botlist(bot, update, resend=False, silent=False):
log.info("Re-sending BotList..." if resend else "Updating BotList...")
channel = helpers.get_channel()
revision = Revision.get_instance()
revision.nr += 1
revision.save()
all_categories = Category.select_all()
botlist = BotList(bot, update, channel, resend, silent)
if resend:
botlist.delete_full_botlist()
botlist.update_intro()
botlist.update_categories(all_categories)
botlist.update_new_bots_list()
botlist.update_category_list()
botlist.send_footer()
botlist.finish()
channel.save()
Statistic.of(update, 'send', 'botlist (resend: {})'.format(str(resend)), Statistic.IMPORTANT)
def mutate(self, info, **kwargs):
sessionId = kwargs.get('session_id')
uId = kwargs.get('user').id
content = kwargs.get('content')
defaultOverlapDuration = 2
segment = json.loads(content)
# check file exist
myFile = Filedrive.query.filter_by(id=segment['fdId']).first()
if not myFile:
raise Exception("File not found.")
# get last revision
myLastRevision = Revision.query.filter(
Revision.u_id == uId, Revision.session_id == sessionId).order_by(
Revision.version.desc()).first()
if segment['action'] == 'crossfade':
if not myLastRevision:
# first revision
myRevision = Revision(u_id=uId,
session_id=sessionId,
version=1,
content=json.dumps([{
"fdId":
myFile.id,
"start":
0,
"end":
myFile.duration,
"color":
'rgba(%s, %s, %s, 0.1)' %
(r(), g(), b()),
"label":
myFile.name,
"durationOverlap":
segment['crossfadeDuration'],
"type":
"crossfade"
}]),
mixed_id=myFile.id,
files_used=myFile.id)
save(myRevision)
return CreateRevision(revision=myRevision)
else:
myLastRevisionFile = Filedrive.query.filter_by(
id=myLastRevision.mixed_id).first()
if not myLastRevisionFile:
raise Exception("Last revision file not found.")
# run ffmpeg command then save to revision table.
currentEpDir = os.path.join(upload_dir,
("audio/%s/%s" % (uId, sessionId)))
os.makedirs(currentEpDir, exist_ok=True)
mixedFilePath = os.path.join(
currentEpDir,
("%sx%s.mp3" % (myLastRevisionFile.id, myFile.id)))
out = subprocess.Popen([
'ffmpeg', '-y', '-i',
filedrive.getRelativePath('audio',
myLastRevisionFile.path), '-i',
filedrive.getRelativePath('audio',
myFile.path), '-filter_complex',
'[0][1]acrossfade=d=' + str(segment['crossfadeDuration']) +
':o=1:c1=tri:c2=tri', '-ar', '44100', '-ac', '2', '-b:a',
'128k', '-acodec', 'libmp3lame', '-f', 'mp3', mixedFilePath
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# waiting for command run complete
stdout, stderr = out.communicate()
print(stdout)
# success command
if os.path.isfile(mixedFilePath):
size = os.stat(mixedFilePath).st_size
duration = librosa.get_duration(filename=mixedFilePath)
# create mixed file
new_filedrive = Filedrive(
u_id=uId,
name=("%sx%s.mp3" %
(myLastRevisionFile.id, myFile.id)),
path=("%s/%s/%sx%s.mp3" %
(uId, sessionId, myLastRevisionFile.id,
myFile.id)),
size=size,
duration=duration,
type=Filedrive.TYPE_CROSSFADED,
is_tmp=Filedrive.IS_NOT_TMP,
is_common=Filedrive.IS_NOT_COMMON)
save(new_filedrive)
# create revision
revisionContent = json.loads(myLastRevision.content)
revisionContent.append({
"fdId":
myFile.id,
"start":
myLastRevisionFile.duration,
"end":
duration,
"color":
'rgba(%s, %s, %s, 0.1)' % (r(), g(), b()),
"label":
myFile.name,
"durationOverlap":
segment['crossfadeDuration'],
"type":
"crossfade"
})
myRevision = Revision(
session_id=sessionId,
u_id=uId,
version=myLastRevision.version + 1,
content=json.dumps(revisionContent),
mixed_id=new_filedrive.id,
files_used=("%s,%s" %
(myLastRevision.mixed_id, myFile.id)))
save(myRevision)
return CreateRevision(revision=myRevision)
elif segment['action'] == 'mix':
myLastRevisionFile = Filedrive.query.filter_by(
id=myLastRevision.mixed_id).first()
if not myLastRevisionFile:
raise Exception("Last revision file not found.")
# run ffmpeg command then save to revision table.
currentEpDir = os.path.join(upload_dir,
("audio/%s/%s" % (uId, sessionId)))
os.makedirs(currentEpDir, exist_ok=True)
mixedFilePath = os.path.join(currentEpDir,
("%sx%s.mp3" %
(myLastRevisionFile.id, myFile.id)))
out = subprocess.Popen([
'ffmpeg', '-y', '-i',
filedrive.getRelativePath('audio',
myLastRevisionFile.path), '-i',
filedrive.getRelativePath('audio',
myFile.path), '-filter_complex',
("[0:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo[a0];[1:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo[a1];[a1]atrim=%s:%s[a1trim];[a1trim]adelay=%s|%s[aud1];[aud1]amix=1,apad[a];[a0][a]amerge[a]"
%
(segment['duration'][0], segment['duration'][1],
int(segment['start']) * 1000, int(segment['start']) * 1000)),
'-map', '[a]', '-ar', '44100', '-ac', '2', '-b:a', '128k',
'-acodec', 'libmp3lame', '-f', 'mp3', mixedFilePath
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# waiting for command run complete
stdout, stderr = out.communicate()
# print(stdout)
# success command
if os.path.isfile(mixedFilePath):
size = os.stat(mixedFilePath).st_size
duration = librosa.get_duration(filename=mixedFilePath)
if segment['type'] == 'try':
is_tmp = Filedrive.IS_TMP
else:
is_tmp = Filedrive.IS_NOT_TMP
# create mixed file
new_filedrive = Filedrive(
u_id=uId,
name=("%sx%s.mp3" % (myLastRevisionFile.id, myFile.id)),
path=("%s/%s/%sx%s.mp3" %
(uId, sessionId, myLastRevisionFile.id, myFile.id)),
size=size,
duration=duration,
type=Filedrive.TYPE_MIXED,
is_tmp=is_tmp,
is_common=Filedrive.IS_NOT_COMMON)
save(new_filedrive)
# fake return old revision if type = try
if segment['type'] == 'try':
return CreateRevision(tmp_file=new_filedrive.path)
# create revision
revisionContent = json.loads(myLastRevision.content)
revisionContent.append({
"fdId":
myFile.id,
"start":
segment['start'],
"end":
int(segment['start']) + (int(segment['duration'][1]) -
int(segment['duration'][0])),
"color":
'rgba(%s, %s, %s, 0.1)' % (r(), g(), b()),
"label":
myFile.name,
"durationRange":
segment['duration'],
"type":
"mix"
})
myRevision = Revision(
session_id=sessionId,
u_id=uId,
version=myLastRevision.version + 1,
content=json.dumps(revisionContent),
mixed_id=new_filedrive.id,
files_used=("%s,%s" %
(myLastRevision.mixed_id, myFile.id)))
save(myRevision)
return CreateRevision(revision=myRevision)
def mutate(self, info, **kwargs):
uId = kwargs.get('user').id
sessionId = kwargs.get('session_id')
version = kwargs.get('version')
newTracksOrder = kwargs.get('new_tracks_order')
myRevision = Revision.query.filter(
Revision.session_id == sessionId,
Revision.version == version,
).order_by(Revision.version.desc()).first()
revisionContent = json.loads(myRevision.content)
newArrOrder = []
newRevisionContent = []
for index in newTracksOrder.split(','):
newArrOrder.append(revisionContent[int(index)])
cmd, fileName = buildCmd(newArrOrder, uId, sessionId)
# print(cmd.split(" "))
out = subprocess.Popen(cmd.split(" "),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# waiting for command run complete
stdout, stderr = out.communicate()
# print(stdout)
currentEpDir = os.path.join(upload_dir,
("audio/%s/%s" % (uId, sessionId)))
os.makedirs(currentEpDir, exist_ok=True)
mixedFilePath = os.path.join(currentEpDir, ("%s.mp3" % fileName))
# success command
if os.path.isfile(mixedFilePath):
size = os.stat(mixedFilePath).st_size
duration = librosa.get_duration(filename=mixedFilePath)
# create mixed file
new_filedrive = Filedrive(u_id=uId,
name=("%s.mp3" % fileName),
path=("%s/%s/%s.mp3" %
(uId, sessionId, fileName)),
size=size,
duration=duration,
type=Filedrive.TYPE_MIXED,
is_tmp=Filedrive.IS_NOT_TMP,
is_common=Filedrive.IS_NOT_COMMON)
save(new_filedrive)
for idx, track in enumerate(newArrOrder):
print(idx)
# print(track)
myFile = Filedrive.query.filter_by(id=track['fdId']).first()
# # calculate region start/end to display in visual
if idx == 0:
track['start'] = 0
track['end'] = myFile.duration
else:
myPreviousFile = Filedrive.query.filter_by(
id=newArrOrder[idx - 1]['fdId']).first()
track['start'] = myPreviousFile.duration
track['end'] = myFile.duration
if idx == len(newArrOrder) - 1:
track['end'] = duration
print(track)
print('----------------------')
# print(newArrOrder)
myRevision = Revision(session_id=sessionId,
u_id=uId,
version=myRevision.version + 1,
content=json.dumps(newArrOrder),
mixed_id=new_filedrive.id,
files_used=("%s" % myRevision.mixed_id))
save(myRevision)
return CreateRevision(revision=myRevision)
def new_bot_submission(bot, update, chat_data, args=None, bot_checker=None):
tg_user = update.message.from_user
user = User.from_telegram_object(tg_user)
if util.stop_banned(update, user):
return
reply_to = util.original_reply_id(update)
if args:
text = " ".join(args)
else:
text = update.message.text
command_no_args = (len(re.findall(r"^/new\s*$", text)) > 0
or text.lower().strip() == "/new@botlistbot")
if command_no_args:
update.message.reply_text(
util.action_hint(
"Please use this command with an argument. For example:\n/new @mybot 🔎"
),
reply_to_message_id=reply_to,
)
return
# `#new` is already checked by handler
try:
username = re.match(settings.REGEX_BOT_IN_TEXT, text).groups()[0]
if username.lower() == "@" + settings.SELF_BOT_NAME.lower():
log.info("Ignoring {}".format(text))
return
except AttributeError:
if args:
update.message.reply_text(
util.failure(
"Sorry, but you didn't send me a bot `@username`."),
quote=True,
parse_mode=ParseMode.MARKDOWN,
reply_to_message_id=reply_to,
)
log.info("Ignoring {}".format(text))
# no bot username, ignore update
return
try:
new_bot = Bot.by_username(username, include_disabled=True)
if new_bot.disabled:
update.message.reply_text(
util.failure("{} is banned from the @BotList.".format(
new_bot.username)),
reply_to_message_id=reply_to,
)
elif new_bot.approved:
update.message.reply_text(
util.action_hint(
"Sorry fool, but {} is already in the @BotList 😉".
format(new_bot.username)),
reply_to_message_id=reply_to,
)
else:
update.message.reply_text(
util.action_hint(
"{} has already been submitted. Please have patience...".
format(new_bot.username)),
reply_to_message_id=reply_to,
)
return
except Bot.DoesNotExist:
new_bot = Bot(
revision=Revision.get_instance().next,
approved=False,
username=username,
submitted_by=user,
)
new_bot.inlinequeries = "🔎" in text
new_bot.official = "🔹" in text
# find language
languages = Country.select().execute()
for lang in languages:
if lang.emoji in text:
new_bot.country = lang
new_bot.date_added = datetime.date.today()
description_reg = re.match(settings.REGEX_BOT_IN_TEXT + " -\s?(.*)", text)
description_notify = ""
if description_reg:
description = description_reg.group(2)
new_bot.description = description
description_notify = " Your description was included."
new_bot.save()
if (util.is_private_message(update)
and util.uid_from_update(update) in settings.MODERATORS):
from components.explore import send_bot_details
send_bot_details(bot, update, chat_data, new_bot)
else:
update.message.reply_text(
util.success("You submitted {} for approval.{}".format(
new_bot, description_notify)),
parse_mode=ParseMode.MARKDOWN,
reply_to_message_id=reply_to,
)
# Ask the user to fill in the bot details
util.send_md_message(
bot,
update.effective_user.id,
"Congratulations, you just submitted a bot to the @BotList. Please help us fill in the details below:",
)
edit_bot(bot, update, chat_data, to_edit=new_bot)
try:
check_submission(bot, bot_checker, new_bot)
except Exception as e:
log.exception(e)
return ConversationHandler.END
def select_pending_update():
return Bot.select().where(Bot.approved == True,
Bot.revision == Revision.get_instance().next,
Bot.disabled == False)
def select_approved():
return Bot.select().where(Bot.approved == True,
Bot.revision <= Revision.get_instance().nr,
Bot.disabled == False)
def select_new_bots():
return Bot.select().where(Bot.is_new == True,
Bot.revision < Revision.get_instance().next,
Bot.approved == True, Bot.disabled == False)
def explorable_bots():
results = Bot.select().where(
~(Bot.description.is_null()), (Bot.approved == True),
(Bot.revision <= Revision.get_instance().nr),
(Bot.offline == False), (Bot.disabled == False))
return list(results)
import os
from decouple import config
from playhouse.migrate import SqliteMigrator, IntegerField, migrate
from playhouse.sqlite_ext import SqliteExtDatabase
from models.revision import Revision
db_path = config('DATABASE_URL',
default=os.path.expanduser('~/botlistbot.sqlite3'))
db = SqliteExtDatabase(db_path)
migrator = SqliteMigrator(db)
revision = IntegerField(default=100)
with db.transaction():
migrate(migrator.add_column('bot', 'revision', revision), )
Revision.create_table(fail_silently=True)
Revision.insert({'nr': 101}).execute()
def of_category_without_new(category):
return Bot.select().where(
(Bot.category == category), (Bot.approved == True),
(Bot.revision <= Revision.get_instance().nr),
(Bot.disabled == False)).order_by(fn.Lower(Bot.username))