def cmd_ping(self, msg: tg.Message) -> str:
# Telegram's timestamps are only accurate to the second... so we have to do it manually
before = util.time_ms()
self.mresult(msg, 'Calculating response time...')
after = util.time_ms()
return 'Request response time: %.2f ms' % (after - before)
def index_file(self, inp, use_cutoff_freq=False):
n_indexed = 0
total_index_time = 0
cutoff = 10
max_clause = 1024
print "indexing:", inp
for event, elem in cElementTree.iterparse(open(inp)):
if elem.tag == "{http://www.openarchives.org/OAI/2.0/oai_dc/}dc":
doc_title = elem.findtext(
"{http://purl.org/dc/elements/1.1/}title")
doc_id = elem.findtext(
"{http://purl.org/dc/elements/1.1/}identifier")
if doc_title and doc_id: # document id and title are required
doc_title = doc_title.replace("\n", " ")
doc_id = re.sub("http.+//", "", doc_id)
doc = {
"id": doc_id,
"title": doc_title,
"index_time": time_ms()
}
started = time_ms()
abstract = elem.findtext(
"{http://purl.org/dc/elements/1.1/}description")
if abstract:
doc["abstract"] = abstract
try:
concepts = self.get_concepts(
doc_title + ". " + abstract,
max_clause,
cutoff,
doc_id,
cutoff_freq=use_cutoff_freq)
if concepts:
doc["concepts"] = concepts # TODO: trim to the top n concepts over a certain score?
except:
print "failed to retrieve concepts for document [" + doc_id + "] - skipping"
traceback.print_exc()
result = es.index(index="document",
doc_type="abstract",
id=doc_id,
body=doc)
# TODO: count failures?
elapsed = time_ms() - started
total_index_time += elapsed
print "indexed document [" + doc_id + "] in " + str(
elapsed) + "ms"
n_indexed += 1
# indexing complete for this file - show average indexing time
avg_index_time = total_index_time / n_indexed
print "indexing [" + inp + "] complete. average time per document is " + str(
avg_index_time) + "ms"
def index_file(self, inp, use_cutoff_freq=False):
context = cElementTree.iterparse(open(inp), events=("start", "end"))
context = iter(context)
event, root = context.next()
for event, elem in context:
if event == "end" and elem.tag == "{http://www.mediawiki.org/xml/export-0.10/}page":
page_id = elem.findtext(
"{http://www.mediawiki.org/xml/export-0.10/}id")
page_title = elem.findtext(
"{http://www.mediawiki.org/xml/export-0.10/}title")
revision = elem.find(
"{http://www.mediawiki.org/xml/export-0.10/}revision")
redirect = elem.find(
"{http://www.mediawiki.org/xml/export-0.10/}redirect")
namespace = elem.findtext(
"{http://www.mediawiki.org/xml/export-0.10/}ns")
if redirect is not None:
print "skipping redirect:", page_title
continue
if revision is not None and page_title is not None and namespace == "0":
markup = revision.findtext(
"{http://www.mediawiki.org/xml/export-0.10/}text")
try:
wikitext = mwparserfromhell.parse(markup).strip_code()
except:
print "FAILED to parse wiki markup - skipping:"
print traceback.print_exc()
if page_id and page_title and wikitext:
now = time_ms()
doc = {
"id": page_id,
"page_title": page_title,
"content": wikitext,
"index_time": now
}
print es.index(index="wikipedia",
doc_type="article",
id=page_id,
body=doc) # TODO check failure
else:
print "skipping article:", page_title
root.clear()
def cmd_ping(self, msg):
before = util.time_ms()
self.bot.mresult(msg, 'Calculating response time...')
after = util.time_ms()
return 'Request response time: %.2f ms' % (after - before)
def cmd_bsetup(self, msg: tg.Message, plain_params: str) -> str:
if not msg.chat: return '__This can only be used in groups.__'
cfg_err: str = '''**Invalid TOML config.** The following options are supported:
```
# Bot to setup
target = "MissRose_bot"
# Default rules
rules = ["No spam", "English only", "Respect others", "No NSFW"]
extra_rules = ["No extreme off-topic"]
# Add ":same" at the end of links to put buttons on the same line
[buttons]
"XDA Thread" = "https://forum.xda-developers.com/"
GitHub = "https://github.com/"```
{}'''
if plain_params.startswith('?') or plain_params.startswith('help'):
return cfg_err.format('')
extra_btn: str = ''
rules: List[str] = [
'No spam', 'English only', 'Respect others', 'No NSFW',
'No extreme off-topic'
]
target: str = 'MissRose_bot'
ex_btn_map: Dict[str, str] = {}
if plain_params:
try:
cfg = toml.loads(plain_params)
except Exception as e:
return cfg_err.format(str(e))
if 'target' in cfg:
target = cfg['target']
if 'rules' in cfg:
rules = cfg['rules']
if 'extra_rules' in cfg:
rules.extend(cfg['extra_rules'])
if 'buttons' in cfg:
for name, dest in cfg['buttons'].items():
ex_btn_map[name] = dest
rule_str = f' \u2022 {rules[0]}'
for rule in rules[1:]:
rule_str += f'\n \u2022 {rule}'
for name, dest in ex_btn_map.items():
extra_btn += f'\n[{name}](buttonurl://{dest})'
before = util.time_ms()
try:
self.client.promote_chat_member(msg.chat.id,
target,
can_change_info=False)
except Exception:
self.mresult(msg, f'**WARNING**: Unable to promote @{target}')
first = '{first}'
srules = '{rules}'
commands: List[str] = [
'welcome on', 'goodbye off', 'warnlimit 3', 'strongwarn off',
f'''setwelcome *Welcome*, {first}!
Please read the rules before chatting. {srules}{extra_btn}''',
'cleanwelcome on', f'setrules \u200b{rule_str}', 'setflood 16',
'gbanstat on', 'gmutestat on', 'reports on', 'cleanservice on'
]
for cmd in commands:
csplit = cmd.split(' ')
_cmd = '/' + csplit[0] + f'@{target} ' + ' '.join(csplit[1:])
self.client.send_message(msg.chat.id, _cmd, parse_mode='HTML')
time.sleep(0.180) # ratelimit
# Clean up the mess
if msg.reply_to_message:
msg.reply_to_message.reply(f'/purge@{target}')
else:
msg.reply(f'/purge@{target}')
after = util.time_ms()
return f'Finished in `{(after - before) / 1000.0}` seconds.'
async def cmd_ping(self, msg):
before = util.time_ms()
await msg.result("Calculating response time...")
after = util.time_ms()
return "Request response time: %d ms" % (after - before)