def format_tag_info_admin(tag_dict):
reply = ""
#prepara i dati lunghi da inserire
if tag_dict["reserved"] == True:
reply = texts.get_text("adm_info_msg_system")
else:
reply = texts.get_text("adm_info_msg_usr")
ow = tag_dict["owner"]["first_name"] + " " + tag_dict["owner"]["last_name"] + " @"+tag_dict["owner"]["username"]
chat = tag_dict["origin_chat"]["type"] + " chat: " + tag_dict["origin_chat"]["name"]
reports = ""
for report in tag_dict["reports"]:
reports +="\n - " + report["text"]
#formatta la stringa e retituiscila
return reply.format(
tag = tag_dict["hashtag"],
region = tag_dict["owner"]["region"],
owner = ow,
chat = chat,
type = tag_dict["data"]["type"],
used = str(tag_dict["use_count"]),
exp = dbManager.calculate_delta_now(tag_dict["last_use_date"]),
crt = tag_dict["creation_date"].strftime("%d/%m/%y"),
rep = reports,
inl = str(tag_dict["inline_count"] if "inline_count" in tag_dict else 0)
)
def remove_tag(bot, update, tag_id):
res = dbManager.get_tag_by_id(tag_id)
send_private_message( bot, res["origin_chat"]["id"] , texts.get_text("admc_bad_tag_warn").format(tag=res["hashtag"]) )
dbManager.delete_tag_by_id(tag_id)
user_id = res["owner"]["id"]
qr = update.callback_query
keyboard = [
[
InlineKeyboardButton(texts.get_text("admc_warn_usr"), callback_data = make_button_data("wu",user_id,tag_id) ),
InlineKeyboardButton(texts.get_text("admc_ban_usr"), callback_data = make_button_data("bu",user_id,tag_id) )
],
[
InlineKeyboardButton(texts.get_text("admc_close_case"), callback_data = make_button_data("nb",user_id,tag_id) ),
]
]
bot.edit_message_text(text = texts.get_text("admc_pending_adm_action"), chat_id=qr.message.chat_id, message_id=qr.message.message_id,
reply_markup=InlineKeyboardMarkup(keyboard) )
def admin_remove(bot, update):
if is_from_admin(update) == False:
return
tag = validate_cmd(update.message.text)
if tag is None:
update.message.reply_text(texts.get_text("adm_rm_use"))
return
dbManager.delete_hashtag(tag)
update.message.reply_text(texts.get_text("adm_rm_ok"))
def helpme(bot, update):
if SUSPEND_CREATION:
update.message.reply_text(
texts.get_text("suspend_input",
update.message.from_user.language_code) +
"\nTag backup: " + EOF_LINK)
return
update.message.reply_text(
texts.get_text("help_reply", update.message.from_user.language_code))
parallel.send_stats_event(update.message.from_user.id, update.message,
"help")
def admin_reserve(bot, update):
if is_from_admin(update) == False:
return
tag = validate_cmd(update.message.text)
if tag is None:
update.message.reply_text(texts.get_text("adm_reserve_use"))
return
dbManager.create_hashtag(tag, None, {
"type": "text",
"data": "Sorry this tag is reserved"
}, True)
update.message.reply_text(texts.get_text("adm_reserve_ok"))
def text_at_revision(ref, version, lang, revision):
"""
Returns the state of a text (identified by ref/version/lang) at revision number 'revision'
"""
changes = texts.db.history.find({
"ref": ref,
"version": version,
"language": lang
}).sort([['revision', -1]])
current = texts.get_text(ref,
context=0,
commentary=False,
version=version,
lang=lang)
if "error" in current:
return current
textField = "text" if lang == "en" else lang
text = current.get(textField, "")
for i in range(changes.count()):
r = changes[i]
if r["revision"] == revision: break
patch = dmp.patch_fromText(r["revert_patch"])
text = dmp.patch_apply(patch, text)[0]
return text
def start(bot, update):
if SUSPEND_CREATION:
update.message.reply_text(
texts.get_text("suspend_input",
update.message.from_user.language_code) +
"\nTag backup: " + EOF_LINK)
return
update.message.reply_text(
texts.get_text("welcome_message",
update.message.from_user.language_code))
dbManager.add_chat_to_bcast_list(update.message.chat.id)
parallel.send_stats_event(update.message.from_user.id, update.message,
"start")
def send_tag_data_to_private_chat(bot, update, tag_id):
result = dbManager.get_tag_by_id(tag_id)
if result is None:
bot.sendMessage(chat_id=update.callback_query.from_user.id, text = texts.get_text("admc_missing_tag_data"))
return
uid = update.callback_query.from_user.id
if result["data"]["type"] == "text":
bot.sendMessage(uid, result["data"]["data"])
if result["data"]["type"] == "image":
bot.sendPhoto(uid, result["data"]["data"])
if result["data"]["type"] == "gif":
bot.sendDocument(uid, result["data"]["data"])
if result["data"]["type"] == "sticker":
bot.sendSticker(uid, result["data"]["data"])
if result["data"]["type"] == "audio":
bot.sendAudio(uid, result["data"]["data"])
if result["data"]["type"] == "voice":
bot.sendVoice(uid, result["data"]["data"])
if result["data"]["type"] == "video":
bot.sendVideo(uid, result["data"]["data"])
def make_text_index_document(ref, version, lang):
"""
Create a document for indexing from the text specified by ref/version/lang
"""
text = texts.get_text(ref, context=0, commentary=False, version=version, lang=lang)
if "error" in text:
print text["error"]
return None
if text["type"] == "Talmud":
title = text["book"] + " Daf " + text["sections"][0]
elif text["type"] == "Commentary" and text["commentaryCategories"][0] == "Talmud":
title = text["book"] + " Daf " + text["sections"][0]
else:
title = text["book"] + " " + " ".join(["%s %d" % (p[0],p[1]) for p in zip(text["sectionNames"], text["sections"])])
title += " (%s)" % version
content = text["he"] if lang == 'he' else text["text"]
if isinstance(content, list):
content = " ".join(content)
return {
"title": title,
"ref": ref,
"version": version,
"lang": lang,
"titleVariants": text["titleVariants"],
"content": content,
"categories": text["categories"],
}
def mytags(bot, update):
uid = update.message.from_user.id
name = update.message.from_user.first_name
username = update.message.from_user.username
if username is None:
username = ""
print("Preparing mytags data for " + name + " @" + username)
firebase.send_user_data(uid, name, username,
dbManager.get_user_hashtags(uid))
print(name + " page is ready")
update.message.reply_text(
texts.get_text("mytags_message",
update.message.from_user.language_code) +
str(update.message.from_user.id))
parallel.send_stats_event(update.message.from_user.id, update.message,
"mytags")
if SUSPEND_CREATION:
update.message.reply_text("Tag backup: " + EOF_LINK)
return
def react(user_id, message):
if message == '/start':
bot.send_message(user_id, texts.get_text(texts.start_txt))
if message == '/help':
bot.send_message(user_id, texts.get_text(texts.help_txt))
elif message == '/add':
bot.send_message(user_id,
texts.get_text(texts.add_txt),
reply_markup=get_es_markup(user_id))
elif message == '/list':
emails = database.get_emails(user_id)
if emails:
result = ''
for email in emails:
result = result + '\n' + email.email
bot.send_message(user_id, texts.get_text(texts.list_txt) + result)
def admin_info(bot, update):
if is_from_admin(update) == False:
return
tag = validate_cmd(update.message.text)
if tag is None:
update.message.reply_text(texts.get_text("adm_info_use"))
return
res = dbManager.get_hashtag_info(tag)
if res is None:
update.message.reply_text(texts.get_text("not_found"))
return
update.message.reply_text(adminChannel.format_tag_info_admin(res))
def record_text_change(ref, version, lang, text, user, **kwargs):
"""
Record a change to a text (ref/version/lang) by user.
"""
# unpack text into smaller segments if necessary (e.g. chapter -> verse)
if isinstance(text, list):
for i in reversed(range(len(text))):
n = i + 1
record_text_change("%s.%d" % (ref, n), version, lang, text[i], user, **kwargs)
return
# get the current state of the text in question
current = texts.get_text(ref, context=0, commentary=False, version=version, lang=lang)
if "error" in current and current["error"].startswith("No text found"):
current = ""
elif "error" in current:
return current
elif lang == "en" and current["text"]:
current = current["text"]
elif lang == "he" and current["he"]:
current = current["he"]
else:
current = ""
# Don't record anything if there's no change.
if not text:
text = ""
if text == current:
return
# create a patch that turns the new version back into the old
backwards_diff = dmp.diff_main(text, current)
patch = dmp.patch_toText(dmp.patch_make(backwards_diff))
# get html displaying edits in this change.
forwards_diff = dmp.diff_main(current, text)
dmp.diff_cleanupSemantic(forwards_diff)
diff_html = dmp.diff_prettyHtml(forwards_diff)
# give this revision a new revision number
revision = next_revision_num()
log = {
"ref": texts.norm_ref(ref),
"version": version,
"language": lang,
"diff_html": diff_html,
"revert_patch": patch,
"user": user,
"date": datetime.now(),
"revision": revision,
"message": kwargs.get("message", ""),
"rev_type": kwargs.get("type", None) or "edit text" if len(current) else "add text",
"method": kwargs.get("method", "Site")
}
db.history.save(log)
def get_hashtag_top_list_message(update):
res = dbManager.get_top_list()
if res is None or not res:
return texts.get_text("top_list_error",
update.message.from_user.language_code)
text = texts.get_text("top_list_header",
update.message.from_user.language_code)
c = 0
for itm in res:
c += 1
text = text + str(c) + ". " + itm["hashtag"] + " - " + str(
itm["use_count"]) + "\n"
text += texts.get_text("complete_list",
update.message.from_user.language_code)
return text
def record_text_change(ref, version, lang, text, user, **kwargs):
"""
Record a change to a text (ref/version/lang) by user.
"""
# unpack text into smaller segments if necessary (e.g. chapter -> verse)
if isinstance(text, list):
for i in reversed(range(len(text))):
n = i + 1
record_text_change("%s.%d" % (ref, n), version, lang, text[i], user, **kwargs)
return
# get the current state of the text in question
current = texts.get_text(ref, context=0, commentary=False, version=version, lang=lang)
if "error" in current and current["error"].startswith("No text found"):
current = ""
elif "error" in current:
return current
elif lang == "en" and current["text"]:
current = current["text"]
elif lang == "he" and current["he"]:
current = current["he"]
else:
current = ""
# Don't record anything if there's no change.
if not text:
text = ""
if text == current:
return
# create a patch that turns the new version back into the old
backwards_diff = dmp.diff_main(text, current)
patch = dmp.patch_toText(dmp.patch_make(backwards_diff))
# get html displaying edits in this change.
forwards_diff = dmp.diff_main(current, text)
dmp.diff_cleanupSemantic(forwards_diff)
diff_html = dmp.diff_prettyHtml(forwards_diff)
# give this revision a new revision number
revision = next_revision_num()
log = {
"ref": texts.norm_ref(ref),
"version": version,
"language": lang,
"diff_html": diff_html,
"revert_patch": patch,
"user": user,
"date": datetime.now(),
"revision": revision,
"message": kwargs.get("message", ""),
"rev_type": kwargs.get("type", None) or "edit text" if len(current) else "add text",
"method": kwargs.get("method", "Site")
}
texts.db.history.save(log)
def report(bot, update):
if dbManager.is_user_banned(update.message.from_user.id) == True:
return
parallel.send_stats_event(update.message.from_user.id, update.message,
"report")
parts = update.message.text.split(" ", 2)
if len(parts) != 3:
update.message.reply_text(
texts.get_text("cmd_report_use",
update.message.from_user.language_code))
return
tag = check_if_hashtag(parts[1])
if tag is None:
update.message.reply_text(
texts.get_text("cmd_report_use",
update.message.from_user.language_code))
return
#controlla lunghezza tag
if len(parts[2]) < 6:
update.message.reply_text(
texts.get_text("report_short",
update.message.from_user.language_code))
return
res = dbManager.add_report(bot, tag, update.message.from_user.id, parts[2])
if res == 0:
update.message.reply_text(
texts.get_text("report_send_success",
update.message.from_user.language_code))
return
if res == 1:
update.message.reply_text(
texts.get_text("report_send_error",
update.message.from_user.language_code))
return
if res == 2:
update.message.reply_text(
texts.get_text("report_no_tag_error",
update.message.from_user.language_code))
return
if res == 3:
update.message.reply_text(
texts.get_text("report_not_allowed",
update.message.from_user.language_code))
return
if SUSPEND_CREATION:
update.message.reply_text("Tag backup: " + EOF_LINK)
return
def info(bot, update):
tag = validate_cmd(update.message.text)
parallel.send_stats_event(update.message.from_user.id, update.message,
"info")
if tag is None:
update.message.reply_text(
texts.get_text("cmd_rm_use",
update.message.from_user.language_code))
return
res = dbManager.get_hashtag_info(tag)
if res is None:
update.message.reply_text(
texts.get_text("not_found",
update.message.from_user.language_code))
return
#recupera la stringa giusta in base al tipo di tag e formattala di conseguenza
reply = ""
if res["reserved"] == True:
reply = texts.get_text("info_msg_system",
update.message.from_user.language_code)
else:
reply = texts.get_text("info_msg_user",
update.message.from_user.language_code)
reply = reply.format(
tag=tag,
type=res["data"]["type"],
used=str(res["use_count"]),
dtc=res["creation_date"].strftime("%d/%m/%y"),
dte=dbManager.calculate_delta_now(res["last_use_date"]),
inl=str(res["inline_count"] if "inline_count" in res else 0))
update.message.reply_text(reply)
if SUSPEND_CREATION:
update.message.reply_text("Tag backup: " + EOF_LINK)
return
def close_case(bot, update, db_data, message):
qr = update.callback_query
reply = texts.get_text("admc_case_closed")
head = ""
if db_data is not None:
head = "Tag "+ db_data["hashtag"]+":"
else:
head = texts.get_text("admc_case_delete_tag")
reply = reply.format(
head = head,
act = message,
aut = qr.from_user.first_name +" ("+qr.from_user.username+")",
dt = datetime.datetime.utcnow().strftime("%d/%m/%y")
)
bot.edit_message_text(text = reply, chat_id=qr.message.chat_id, message_id=qr.message.message_id)
def message_handler(message):
try:
user_id = message.chat.id
react(user_id, message.text)
botan.track(config.botan_api_key, user_id, {'input': message.text},
message.text)
except Exception as e:
bot.send_message(user_id, texts.get_text(texts.error))
logger.error(str(e))
def admin_bcast(bot, update):
if is_from_admin(update) == False:
return
parts = update.message.text.split(" ", 1)
if len(parts) != 2:
update.message.reply_text(texts.get_text("adm_bcast_use"))
return
parallel.broadcast_message(bot, update, parts[1])
def send_report_data(bot, tag_dict):
print("Sending report data to admin channel (" + str(CHANNEL_ID) + ")")
msg = format_tag_info_admin(tag_dict)
user_id = tag_dict["owner"]["id"]
tag_id = tag_dict["_id"]
if isinstance(tag_id, dict) and tag_dict["$oid"] is not None:
tag_id = tag_id["$oid"]
keyboard = [
[InlineKeyboardButton(texts.get_text("admc_clear_reports"), callback_data = make_button_data("cr",user_id,tag_id) ),
InlineKeyboardButton(texts.get_text("admc_remove_tag"), callback_data = make_button_data("rt",user_id,tag_id) )],
[InlineKeyboardButton(texts.get_text("admc_show_tag"), callback_data = make_button_data("st",user_id,tag_id) )]
]
reply_buttons = InlineKeyboardMarkup(keyboard)
bot.sendMessage(chat_id=CHANNEL_ID, text=msg, reply_markup=reply_buttons)
def query_handler(bot, update):
query = update.callback_query
if int(query.from_user.id) != int(ADMIN_ID): #check if admin
return
parts = query.data.split("_")
if parts[0] == "st":
send_tag_data_to_private_chat(bot,update,parts[2])
if parts[0] == "cr":
clear_reports(bot,update, parts[2])
if parts[0] == "rt":
remove_tag(bot,update, parts[2])
if parts[0] == "nb":
close_case(bot, update, None, texts.get_text("admc_case_delete_tag"))
if parts[0] == "wu":
r = dbManager.warn_user(int(parts[1]))
if r == 1:
send_private_message(bot, int(parts[1]), texts.get_text("warn_received", update.message.from_user.language_code))
close_case(bot, update, None, texts.get_text("admc_case_delete_warn_tag"))
else:
send_private_message(bot, int(parts[1]), texts.get_text("ban_received", update.message.from_user.language_code))
close_case(bot, update, None, texts.get_text("admc_case_delete_ban_tag"))
if parts[0] == "bu":
dbManager.ban_user(int(parts[1]))
close_case(bot, update, None, texts.get_text("admc_case_delete_ban_tag"))
def get_stats(self, id, name):
http = httplib2.Http()
resp, content = http.request('https://rocketleaguestats.com/profile/' + name + '/steam')
soup = BeautifulSoup(content, 'html.parser')
try:
imgSRC = soup.find('img')['src']
print('Found !')
src = 'https://rocketleaguestats.com/profile/' + name + '/steam'
return src
except TypeError:
print('Not found !')
return get_text(id)
def get_picture(self, id, name):
http = httplib2.Http()
resp, content = http.request('https://rocketleaguestats.com/profile/' + name + '/steam')
soup = BeautifulSoup(content, 'html.parser')
try:
imgSRC = soup.find('img')['src']
print('Found !')
urllib.request.urlretrieve(imgSRC, 'temp_stats_picture.png')
return
except TypeError:
print('Not found !')
return get_text(id)
def remove(bot, update):
if SUSPEND_CREATION:
update.message.reply_text(
texts.get_text("suspend_input",
update.message.from_user.language_code) +
"\nTag backup: " + EOF_LINK)
return
tag = validate_cmd(update.message.text)
parallel.send_stats_event(update.message.from_user.id, update.message,
"remove")
if tag is None:
update.message.reply_text(
texts.get_text("cmd_rm_use",
update.message.from_user.language_code))
return
#controlla se e' possibile rimuovere il tag
res = dbManager.can_write_hashtag(tag, update.message.from_user.id, True)
if res == 0:
#rimuovi il tag
dbManager.delete_hashtag(tag)
update.message.reply_text(
texts.get_text("tag_remove_ok",
update.message.from_user.language_code))
else:
if res == 3:
update.message.reply_text(
texts.get_text("rm_tag_free",
update.message.from_user.language_code))
else:
update.message.reply_text(
texts.get_text("tag_not_owned",
update.message.from_user.language_code))
def admin_set(bot, update):
if is_from_admin(update) == False:
return
tag = validate_cmd(update.message.text)
if tag is None:
update.message.reply_text(texts.get_text("adm_set_use"))
return
#controlla che sia stato quotato un messaggio
if update.message.reply_to_message is None:
update.message.reply_text(texts.get_text("quote_missing"))
return
#crea il tag
data = get_message_data(update.message.reply_to_message)
#controlla validita dati
if data is None:
update.message.reply_text(texts.get_text("too_much_chars"))
else:
dbManager.create_hashtag(tag, None, data, True)
update.message.reply_text(texts.get_text("adm_set_ok").format(tag=tag))
def index_text(tref, version=None, lang=None):
"""
Index the text designated by ref.
If no version and lang are given, this functon will be called for each availble version.
Currently assumes ref is at section level.
"""
#tref = texts.norm_ref(unicode(tref))
#todo: why the unicode()?
tref = model.Ref(tref).normal()
# Recall this function for each specific text version, if non provided
if not (version and lang):
for v in texts.get_version_list(tref):
index_text(tref, version=v["versionTitle"], lang=v["language"])
return
# Index each segment of this document individually
oref = model.Ref(tref).padded_ref()
if len(oref.sections) < len(oref.index.sectionNames):
text = texts.get_text(tref,
context=0,
commentary=False,
version=version,
lang=lang)
if "error" in text:
print text["error"]
else:
for i in range(max(len(text["text"]), len(text["he"]))):
index_text("%s:%d" % (tref, i + 1))
# Don't try to index docs with depth 3
if len(oref.sections) < len(oref.index.sectionNames) - 1:
return
# Index this document as a whole
doc = make_text_index_document(tref, version, lang)
if doc:
try:
global doc_count
if doc_count % 5000 == 0:
print "[%d] Indexing %s / %s / %s" % (doc_count, tref, version,
lang)
es.index('sefaria', 'text', doc,
make_text_doc_id(tref, version, lang))
doc_count += 1
except Exception, e:
print "ERROR indexing %s / %s / %s" % (tref, version, lang)
pprint(e)
def edit(bot, update):
if SUSPEND_CREATION:
update.message.reply_text(
texts.get_text("suspend_input",
update.message.from_user.language_code) +
"\nTag backup: " + EOF_LINK)
return
if dbManager.is_user_banned(update.message.from_user.id) == True:
return
parts = update.message.text.split()
parallel.send_stats_event(update.message.from_user.id, update.message,
"edit")
if len(parts) != 3:
update.message.reply_text(
texts.get_text("cmd_edit_use",
update.message.from_user.language_code))
return
#controlla i due tag
old_tag = check_if_hashtag(parts[1])
new_tag = check_if_hashtag(parts[2])
if old_tag is None or new_tag is None:
update.message.reply_text(
texts.get_text("cmd_edit_use",
update.message.from_user.language_code))
return
uid = update.message.from_user.id
#controlla che entrambi gli hashtag siano liberi o del propietario
if dbManager.can_write_hashtag(old_tag,
uid) == 0 and dbManager.can_write_hashtag(
new_tag, uid) == 0:
res = dbManager.change_hashtag(old_tag, new_tag)
if res == 0:
update.message.reply_text(
texts.get_text("edit_tag_error",
update.message.from_user.language_code))
else:
update.message.reply_text(
texts.get_text("edit_ok",
update.message.from_user.language_code).format(
old=old_tag, new=new_tag))
else:
update.message.reply_text(
texts.get_text("edit_perm_error",
update.message.from_user.language_code))
def make_text_index_document(tref, version, lang):
"""
Create a document for indexing from the text specified by ref/version/lang
"""
text = texts.get_text(tref,
context=0,
commentary=False,
version=version,
lang=lang)
if "error" in text:
print text["error"]
return None
if text["type"] == "Talmud":
title = text["book"] + " Daf " + text["sections"][0]
elif text["type"] == "Commentary" and text["commentaryCategories"][
0] == "Talmud":
title = text["book"] + " Daf " + text["sections"][0]
else:
title = text["book"] + " " + " ".join([
"%s %d" % (p[0], p[1])
for p in zip(text["sectionNames"], text["sections"])
])
title += " (%s)" % version
if lang == "he":
title = text.get("heTitle", "") + " " + title
content = text["he"] if lang == 'he' else text["text"]
if not content:
# Don't bother indexing if there's no content
return False
if isinstance(content, list):
content = " ".join(content)
return {
"title": title,
"ref": tref,
"version": version,
"lang": lang,
"titleVariants": text["titleVariants"],
"content": content,
"categories": text["categories"],
}
def text_at_revision(ref, version, lang, revision):
"""
Returns the state of a text (identified by ref/version/lang) at revision number 'revision'
"""
changes = db.history.find({"ref": ref, "version": version, "language": lang}).sort([['revision', -1]])
current = texts.get_text(ref, context=0, commentary=False, version=version, lang=lang)
if "error" in current and not current["error"].startswith("No text found"):
return current
textField = "text" if lang == "en" else lang
text = unicode(current.get(textField, ""))
for i in range(changes.count()):
r = changes[i]
if r["revision"] == revision: break
patch = dmp.patch_fromText(r["revert_patch"])
text = dmp.patch_apply(patch, text)[0]
return text
def index_text(ref, version=None, lang=None):
"""
Index the text designated by ref.
If no version and lang are given, this functon will be called for each availble version.
Currently assumes ref is at section level.
"""
ref = texts.norm_ref(unicode(ref))
# Recall this function for each specific text version, if non provided
if not (version and lang):
for v in texts.get_version_list(ref):
index_text(ref, version=v["versionTitle"], lang=v["language"])
return
# Index each segment of this document individually
pRef = texts.parse_ref(ref)
if len(pRef["sections"]) < len(pRef["sectionNames"]):
text = texts.get_text(ref,
context=0,
commentary=False,
version=version,
lang=lang)
if "error" in text:
print text["error"]
else:
for i in range(max(len(text["text"]), len(text["he"]))):
index_text("%s:%d" % (ref, i + 1))
# Don't try to index docs with depth 3
if len(pRef["sections"]) < len(pRef["sectionNames"]) - 1:
return
# Index this document as a whole
doc = make_text_index_document(ref, version, lang)
if doc:
try:
es.index(doc, 'sefaria', 'text',
make_text_doc_id(ref, version, lang))
global doc_count
doc_count += 1
except Exception, e:
print "Error indexing %s / %s / %s" % (ref, version, lang)
print e
def index_text(tref, version=None, lang=None):
"""
Index the text designated by ref.
If no version and lang are given, this functon will be called for each availble version.
Currently assumes ref is at section level.
"""
#tref = texts.norm_ref(unicode(tref))
#todo: why the unicode()?
tref = model.Ref(tref).normal()
# Recall this function for each specific text version, if non provided
if not (version and lang):
for v in texts.get_version_list(tref):
index_text(tref, version=v["versionTitle"], lang=v["language"])
return
# Index each segment of this document individually
oref = model.Ref(tref).padded_ref()
if len(oref.sections) < len(oref.index.sectionNames):
text = texts.get_text(tref, context=0, commentary=False, version=version, lang=lang)
if "error" in text:
print text["error"]
else:
for i in range(max(len(text["text"]), len(text["he"]))):
index_text("%s:%d" % (tref, i+1))
# Don't try to index docs with depth 3
if len(oref.sections) < len(oref.index.sectionNames) - 1:
return
# Index this document as a whole
doc = make_text_index_document(tref, version, lang)
if doc:
try:
global doc_count
if doc_count % 5000 == 0:
print "[%d] Indexing %s / %s / %s" % (doc_count, tref, version, lang)
es.index('sefaria', 'text', doc, make_text_doc_id(tref, version, lang))
doc_count += 1
except Exception, e:
print "ERROR indexing %s / %s / %s" % (tref, version, lang)
pprint(e)
def claim(bot, update):
if SUSPEND_CREATION:
update.message.reply_text(
texts.get_text("suspend_input",
update.message.from_user.language_code) +
"\nTag backup: " + EOF_LINK)
return
if dbManager.is_user_banned(update.message.from_user.id) == True:
return
tag = validate_cmd(update.message.text)
parallel.send_stats_event(update.message.from_user.id, update.message,
"claim")
if tag is None:
update.message.reply_text(
texts.get_text("claim_reply",
update.message.from_user.language_code))
return
#controlla che sia stato quotato un messaggio
if update.message.reply_to_message is None:
update.message.reply_text(
texts.get_text("quote_missing",
update.message.from_user.language_code))
return
#cerca di creare il tag se possibile
if dbManager.can_write_hashtag(tag, update.message.from_user.id) == 0:
data = get_message_data(update.message.reply_to_message)
#controlla validita dati
if data is None:
update.message.reply_text(
texts.get_text("too_much_chars",
update.message.from_user.language_code))
else:
dbManager.create_hashtag(tag, update, data, False)
update.message.reply_text(
texts.get_text("claim_ok",
update.message.from_user.language_code) + tag)
else:
update.message.reply_text(
texts.get_text("claim_error",
update.message.from_user.language_code))
def count_terms(query={}, lang=None):
#todo: move to object model. Maybe. What's this doing?
"""
Counts all terms in texts matching query, lang
Saves reults to terms collection in db.
"""
terms = {}
bavli_names = db.index.find(query).distinct("title")
query = {"title": {"$in": bavli_names}}
refs = counts.generate_refs_list(query)
lookup_lang = "he" if lang == "ar" else lang
for ref in refs:
print ref
text = texts.get_text(ref, commentary=False)
for i, line in enumerate(text.get(lookup_lang, [])):
# strip punctuation
for c in string.punctuation:
line = line.replace(c, "")
these_terms = line.split(" ")
for term in these_terms:
line_ref = "%s:%d" % (ref, i + 1)
term = hebrew.strip_nikkud(term)
if term in terms:
terms[term]["occurrences"] += 1
terms[term]["refs"].add(line_ref)
else:
terms[term] = {
"term": term,
"occurrences": 1,
"language": lang,
"refs": set([line_ref])
}
for term in terms:
print term
# only include up to 20 random ref samples
sample_size = len(
terms[term]["refs"]) if len(terms[term]["refs"]) < 20 else 20
terms[term]["refs"] = list(sample(terms[term]["refs"], sample_size))
db.terms.save(terms[term])
def count_terms(query={}, lang=None):
#todo: move to object model. Maybe. What's this doing?
"""
Counts all terms in texts matching query, lang
Saves reults to terms collection in db.
"""
terms = {}
bavli_names = db.index.find(query).distinct("title")
query = {"title": {"$in": bavli_names}}
refs = counts.generate_refs_list(query)
lookup_lang = "he" if lang == "ar" else lang
for ref in refs:
print ref
text = texts.get_text(ref, commentary=False)
for i, line in enumerate(text.get(lookup_lang, [])):
# strip punctuation
for c in string.punctuation:
line = line.replace(c,"")
these_terms = line.split(" ")
for term in these_terms:
line_ref = "%s:%d" % (ref, i+1)
term = hebrew.strip_nikkud(term)
if term in terms:
terms[term]["occurrences"] += 1
terms[term]["refs"].add(line_ref)
else:
terms[term] = {
"term": term,
"occurrences": 1,
"language": lang,
"refs": set([line_ref])
}
for term in terms:
print term
# only include up to 20 random ref samples
sample_size = len(terms[term]["refs"]) if len(terms[term]["refs"]) < 20 else 20
terms[term]["refs"] = list(sample(terms[term]["refs"], sample_size))
db.terms.save(terms[term])
def index_text(ref, version=None, lang=None):
"""
Index the text designated by ref.
If no version and lang are given, this functon will be called for each availble version.
Currently assumes ref is at section level.
"""
ref = texts.norm_ref(unicode(ref))
# Recall this function for each specific text version, if non provided
if not (version and lang):
for v in texts.get_version_list(ref):
index_text(ref, version=v["versionTitle"], lang=v["language"])
return
# Index each segment of this document individually
pRef = texts.parse_ref(ref)
if len(pRef["sections"]) < len(pRef["sectionNames"]):
text = texts.get_text(ref, context=0, commentary=False, version=version, lang=lang)
if "error" in text:
print text["error"]
else:
for i in range(max(len(text["text"]), len(text["he"]))):
index_text("%s:%d" % (ref, i+1))
# Don't try to index docs with depth 3
if len(pRef["sections"]) < len(pRef["sectionNames"]) - 1:
return
# Index this document as a whole
doc = make_text_index_document(ref, version, lang)
if doc:
try:
es.index(doc, 'sefaria', 'text', make_text_doc_id(ref, version, lang))
global doc_count
doc_count += 1
except Exception, e:
print "Error indexing %s / %s / %s" % (ref, version, lang)
print e
def clear_reports(bot, update, tag_id):
res = dbManager.remove_reports(tag_id)
close_case(bot,update,res,texts.get_text("admc_report_clear"))