예제 #1
0
def format_tag_info_admin(tag_dict):

    reply = ""

    #prepara i dati lunghi da inserire
    if tag_dict["reserved"] == True:
        reply = texts.get_text("adm_info_msg_system") 
    else:
        reply = texts.get_text("adm_info_msg_usr")

    ow = tag_dict["owner"]["first_name"] + " " + tag_dict["owner"]["last_name"] + " @"+tag_dict["owner"]["username"]
    chat = tag_dict["origin_chat"]["type"] + " chat: " + tag_dict["origin_chat"]["name"]

    reports = ""
    for report in tag_dict["reports"]:
        reports +="\n - " + report["text"]

    #formatta la stringa e retituiscila
    return reply.format(
        tag = tag_dict["hashtag"],
        region = tag_dict["owner"]["region"],
        owner = ow,
        chat = chat,
        type = tag_dict["data"]["type"],
        used = str(tag_dict["use_count"]),
        exp = dbManager.calculate_delta_now(tag_dict["last_use_date"]),
        crt = tag_dict["creation_date"].strftime("%d/%m/%y"),
        rep = reports,
        inl = str(tag_dict["inline_count"] if "inline_count" in tag_dict else 0)
        )
예제 #2
0
def remove_tag(bot, update, tag_id):

    res = dbManager.get_tag_by_id(tag_id)

    send_private_message( bot, res["origin_chat"]["id"] , texts.get_text("admc_bad_tag_warn").format(tag=res["hashtag"]) )

    dbManager.delete_tag_by_id(tag_id)

    user_id = res["owner"]["id"] 
    
    qr = update.callback_query

    keyboard = [
                    [
                        InlineKeyboardButton(texts.get_text("admc_warn_usr"), callback_data = make_button_data("wu",user_id,tag_id) ),
                        InlineKeyboardButton(texts.get_text("admc_ban_usr"), callback_data = make_button_data("bu",user_id,tag_id) )
                    ],
                    [
                        InlineKeyboardButton(texts.get_text("admc_close_case"), callback_data = make_button_data("nb",user_id,tag_id) ),
                    ]


                ]
    
    bot.edit_message_text(text = texts.get_text("admc_pending_adm_action"), chat_id=qr.message.chat_id, message_id=qr.message.message_id, 
        reply_markup=InlineKeyboardMarkup(keyboard) )
예제 #3
0
def admin_remove(bot, update):
    if is_from_admin(update) == False:
        return

    tag = validate_cmd(update.message.text)
    if tag is None:
        update.message.reply_text(texts.get_text("adm_rm_use"))
        return

    dbManager.delete_hashtag(tag)
    update.message.reply_text(texts.get_text("adm_rm_ok"))
예제 #4
0
def helpme(bot, update):
    if SUSPEND_CREATION:
        update.message.reply_text(
            texts.get_text("suspend_input",
                           update.message.from_user.language_code) +
            "\nTag backup: " + EOF_LINK)
        return

    update.message.reply_text(
        texts.get_text("help_reply", update.message.from_user.language_code))
    parallel.send_stats_event(update.message.from_user.id, update.message,
                              "help")
예제 #5
0
def admin_reserve(bot, update):
    if is_from_admin(update) == False:
        return

    tag = validate_cmd(update.message.text)
    if tag is None:
        update.message.reply_text(texts.get_text("adm_reserve_use"))
        return

    dbManager.create_hashtag(tag, None, {
        "type": "text",
        "data": "Sorry this tag is reserved"
    }, True)
    update.message.reply_text(texts.get_text("adm_reserve_ok"))
예제 #6
0
def text_at_revision(ref, version, lang, revision):
    """
	Returns the state of a text (identified by ref/version/lang) at revision number 'revision'
	"""

    changes = texts.db.history.find({
        "ref": ref,
        "version": version,
        "language": lang
    }).sort([['revision', -1]])
    current = texts.get_text(ref,
                             context=0,
                             commentary=False,
                             version=version,
                             lang=lang)
    if "error" in current:
        return current

    textField = "text" if lang == "en" else lang
    text = current.get(textField, "")

    for i in range(changes.count()):
        r = changes[i]
        if r["revision"] == revision: break
        patch = dmp.patch_fromText(r["revert_patch"])
        text = dmp.patch_apply(patch, text)[0]

    return text
예제 #7
0
def start(bot, update):

    if SUSPEND_CREATION:
        update.message.reply_text(
            texts.get_text("suspend_input",
                           update.message.from_user.language_code) +
            "\nTag backup: " + EOF_LINK)
        return

    update.message.reply_text(
        texts.get_text("welcome_message",
                       update.message.from_user.language_code))
    dbManager.add_chat_to_bcast_list(update.message.chat.id)

    parallel.send_stats_event(update.message.from_user.id, update.message,
                              "start")
예제 #8
0
def send_tag_data_to_private_chat(bot, update, tag_id):

    result = dbManager.get_tag_by_id(tag_id)
    if result is None:
       bot.sendMessage(chat_id=update.callback_query.from_user.id, text = texts.get_text("admc_missing_tag_data"))
       return

    uid = update.callback_query.from_user.id

    if result["data"]["type"] == "text":
        bot.sendMessage(uid, result["data"]["data"])

    if result["data"]["type"] == "image":
        bot.sendPhoto(uid, result["data"]["data"])

    if result["data"]["type"] == "gif":
        bot.sendDocument(uid, result["data"]["data"])

    if result["data"]["type"] == "sticker":
        bot.sendSticker(uid, result["data"]["data"])

    if result["data"]["type"] == "audio":
        bot.sendAudio(uid, result["data"]["data"])

    if result["data"]["type"] == "voice":
        bot.sendVoice(uid, result["data"]["data"])

    if result["data"]["type"] == "video":
        bot.sendVideo(uid, result["data"]["data"])
예제 #9
0
def make_text_index_document(ref, version, lang):
    """
    Create a document for indexing from the text specified by ref/version/lang
    """
    text = texts.get_text(ref, context=0, commentary=False, version=version, lang=lang)

    if "error" in text:
        print text["error"]
        return None

    if text["type"] == "Talmud":
        title = text["book"] + " Daf " + text["sections"][0]
    elif text["type"] == "Commentary" and text["commentaryCategories"][0] == "Talmud":
        title = text["book"] + " Daf " + text["sections"][0]
    else:
        title = text["book"] + " " + " ".join(["%s %d" % (p[0],p[1]) for p in zip(text["sectionNames"], text["sections"])])
    title += " (%s)" % version

    content = text["he"] if lang == 'he' else text["text"] 
    if isinstance(content, list):
        content = " ".join(content)

    return {
        "title": title, 
        "ref": ref, 
        "version": version, 
        "lang": lang,
        "titleVariants": text["titleVariants"],
        "content": content,
        "categories": text["categories"],
        }
예제 #10
0
def mytags(bot, update):

    uid = update.message.from_user.id
    name = update.message.from_user.first_name
    username = update.message.from_user.username
    if username is None:
        username = ""

    print("Preparing mytags data for " + name + " @" + username)

    firebase.send_user_data(uid, name, username,
                            dbManager.get_user_hashtags(uid))

    print(name + " page is ready")
    update.message.reply_text(
        texts.get_text("mytags_message",
                       update.message.from_user.language_code) +
        str(update.message.from_user.id))

    parallel.send_stats_event(update.message.from_user.id, update.message,
                              "mytags")

    if SUSPEND_CREATION:
        update.message.reply_text("Tag backup: " + EOF_LINK)
        return
예제 #11
0
def react(user_id, message):
    if message == '/start':
        bot.send_message(user_id, texts.get_text(texts.start_txt))
    if message == '/help':
        bot.send_message(user_id, texts.get_text(texts.help_txt))
    elif message == '/add':
        bot.send_message(user_id,
                         texts.get_text(texts.add_txt),
                         reply_markup=get_es_markup(user_id))
    elif message == '/list':
        emails = database.get_emails(user_id)
        if emails:
            result = ''
            for email in emails:
                result = result + '\n' + email.email

            bot.send_message(user_id, texts.get_text(texts.list_txt) + result)
예제 #12
0
def admin_info(bot, update):

    if is_from_admin(update) == False:
        return

    tag = validate_cmd(update.message.text)
    if tag is None:
        update.message.reply_text(texts.get_text("adm_info_use"))
        return

    res = dbManager.get_hashtag_info(tag)

    if res is None:
        update.message.reply_text(texts.get_text("not_found"))
        return

    update.message.reply_text(adminChannel.format_tag_info_admin(res))
예제 #13
0
def record_text_change(ref, version, lang, text, user, **kwargs):
	"""
	Record a change to a text (ref/version/lang) by user. 
	"""

	# unpack text into smaller segments if necessary (e.g. chapter -> verse)
	if isinstance(text, list):
		for i in reversed(range(len(text))):
			n = i + 1
			record_text_change("%s.%d" % (ref, n), version, lang, text[i], user, **kwargs)
		return

	# get the current state of the text in question
	current = texts.get_text(ref, context=0, commentary=False, version=version, lang=lang)
	if "error" in current and current["error"].startswith("No text found"):
		current = ""
	elif "error" in current:
		return current
	elif lang == "en" and current["text"]:
		current = current["text"]
	elif lang == "he" and current["he"]:
		current = current["he"]
	else: 
		current = ""

	# Don't record anything if there's no change. 
	if not text:
		text = ""
	if text == current: 
		return

	# create a patch that turns the new version back into the old	
	backwards_diff = dmp.diff_main(text, current)
	patch = dmp.patch_toText(dmp.patch_make(backwards_diff))
	# get html displaying edits in this change.
	forwards_diff = dmp.diff_main(current, text)
	dmp.diff_cleanupSemantic(forwards_diff)
	diff_html = dmp.diff_prettyHtml(forwards_diff) 

	# give this revision a new revision number
	revision = next_revision_num()

	log = {
		"ref": texts.norm_ref(ref),
		"version": version,
		"language": lang,
		"diff_html": diff_html,
		"revert_patch": patch,
		"user": user,
		"date": datetime.now(),
		"revision": revision,
		"message": kwargs.get("message", ""),
		"rev_type": kwargs.get("type", None) or "edit text" if len(current) else "add text",
		"method": kwargs.get("method", "Site")
	}

	db.history.save(log)
예제 #14
0
def get_hashtag_top_list_message(update):
    res = dbManager.get_top_list()
    if res is None or not res:
        return texts.get_text("top_list_error",
                              update.message.from_user.language_code)

    text = texts.get_text("top_list_header",
                          update.message.from_user.language_code)
    c = 0
    for itm in res:
        c += 1
        text = text + str(c) + ". " + itm["hashtag"] + " - " + str(
            itm["use_count"]) + "\n"

    text += texts.get_text("complete_list",
                           update.message.from_user.language_code)

    return text
예제 #15
0
def record_text_change(ref, version, lang, text, user, **kwargs):
	"""
	Record a change to a text (ref/version/lang) by user. 
	"""

	# unpack text into smaller segments if necessary (e.g. chapter -> verse)
	if isinstance(text, list):
		for i in reversed(range(len(text))):
			n = i + 1
			record_text_change("%s.%d" % (ref, n), version, lang, text[i], user, **kwargs)
		return

	# get the current state of the text in question
	current = texts.get_text(ref, context=0, commentary=False, version=version, lang=lang)
	if "error" in current and current["error"].startswith("No text found"):
		current = ""
	elif "error" in current:
		return current
	elif lang == "en" and current["text"]:
		current = current["text"]
	elif lang == "he" and current["he"]:
		current = current["he"]
	else: 
		current = ""

	# Don't record anything if there's no change. 
	if not text:
		text = ""
	if text == current: 
		return

	# create a patch that turns the new version back into the old	
	backwards_diff = dmp.diff_main(text, current)
	patch = dmp.patch_toText(dmp.patch_make(backwards_diff))
	# get html displaying edits in this change.
	forwards_diff = dmp.diff_main(current, text)
	dmp.diff_cleanupSemantic(forwards_diff)
	diff_html = dmp.diff_prettyHtml(forwards_diff) 

	# give this revision a new revision number
	revision = next_revision_num()

	log = {
		"ref": texts.norm_ref(ref),
		"version": version,
		"language": lang,
		"diff_html": diff_html,
		"revert_patch": patch,
		"user": user,
		"date": datetime.now(),
		"revision": revision,
		"message": kwargs.get("message", ""),
		"rev_type": kwargs.get("type", None) or "edit text" if len(current) else "add text",
		"method": kwargs.get("method", "Site")
	}

	texts.db.history.save(log)
예제 #16
0
def report(bot, update):

    if dbManager.is_user_banned(update.message.from_user.id) == True:
        return

    parallel.send_stats_event(update.message.from_user.id, update.message,
                              "report")

    parts = update.message.text.split(" ", 2)

    if len(parts) != 3:
        update.message.reply_text(
            texts.get_text("cmd_report_use",
                           update.message.from_user.language_code))
        return

    tag = check_if_hashtag(parts[1])
    if tag is None:
        update.message.reply_text(
            texts.get_text("cmd_report_use",
                           update.message.from_user.language_code))
        return

    #controlla lunghezza tag
    if len(parts[2]) < 6:
        update.message.reply_text(
            texts.get_text("report_short",
                           update.message.from_user.language_code))
        return

    res = dbManager.add_report(bot, tag, update.message.from_user.id, parts[2])

    if res == 0:
        update.message.reply_text(
            texts.get_text("report_send_success",
                           update.message.from_user.language_code))
        return

    if res == 1:
        update.message.reply_text(
            texts.get_text("report_send_error",
                           update.message.from_user.language_code))
        return

    if res == 2:
        update.message.reply_text(
            texts.get_text("report_no_tag_error",
                           update.message.from_user.language_code))
        return

    if res == 3:
        update.message.reply_text(
            texts.get_text("report_not_allowed",
                           update.message.from_user.language_code))
        return

    if SUSPEND_CREATION:
        update.message.reply_text("Tag backup: " + EOF_LINK)
        return
예제 #17
0
def info(bot, update):

    tag = validate_cmd(update.message.text)

    parallel.send_stats_event(update.message.from_user.id, update.message,
                              "info")

    if tag is None:
        update.message.reply_text(
            texts.get_text("cmd_rm_use",
                           update.message.from_user.language_code))
        return

    res = dbManager.get_hashtag_info(tag)

    if res is None:
        update.message.reply_text(
            texts.get_text("not_found",
                           update.message.from_user.language_code))
        return

    #recupera la stringa giusta in base al tipo di tag e formattala di conseguenza
    reply = ""
    if res["reserved"] == True:
        reply = texts.get_text("info_msg_system",
                               update.message.from_user.language_code)
    else:
        reply = texts.get_text("info_msg_user",
                               update.message.from_user.language_code)

    reply = reply.format(
        tag=tag,
        type=res["data"]["type"],
        used=str(res["use_count"]),
        dtc=res["creation_date"].strftime("%d/%m/%y"),
        dte=dbManager.calculate_delta_now(res["last_use_date"]),
        inl=str(res["inline_count"] if "inline_count" in res else 0))

    update.message.reply_text(reply)

    if SUSPEND_CREATION:
        update.message.reply_text("Tag backup: " + EOF_LINK)
        return
예제 #18
0
def close_case(bot, update, db_data, message):
    qr = update.callback_query

    reply = texts.get_text("admc_case_closed")
    head = ""

    if db_data is not None:
        head = "Tag "+ db_data["hashtag"]+":"
    else:
        head = texts.get_text("admc_case_delete_tag")

    reply = reply.format(
        head = head,
        act = message,
        aut = qr.from_user.first_name +" ("+qr.from_user.username+")",
        dt = datetime.datetime.utcnow().strftime("%d/%m/%y")

    )

    bot.edit_message_text(text = reply, chat_id=qr.message.chat_id, message_id=qr.message.message_id)
예제 #19
0
def message_handler(message):
    try:
        user_id = message.chat.id

        react(user_id, message.text)

        botan.track(config.botan_api_key, user_id, {'input': message.text},
                    message.text)
    except Exception as e:
        bot.send_message(user_id, texts.get_text(texts.error))
        logger.error(str(e))
예제 #20
0
def admin_bcast(bot, update):
    if is_from_admin(update) == False:
        return

    parts = update.message.text.split(" ", 1)

    if len(parts) != 2:
        update.message.reply_text(texts.get_text("adm_bcast_use"))
        return

    parallel.broadcast_message(bot, update, parts[1])
예제 #21
0
def send_report_data(bot, tag_dict):
    print("Sending report data to admin channel (" + str(CHANNEL_ID) + ")")
    msg = format_tag_info_admin(tag_dict)

    user_id = tag_dict["owner"]["id"]
    tag_id = tag_dict["_id"]

    if isinstance(tag_id, dict) and tag_dict["$oid"] is not None:
        tag_id = tag_id["$oid"]

    keyboard = [
                    [InlineKeyboardButton(texts.get_text("admc_clear_reports"), callback_data = make_button_data("cr",user_id,tag_id) ),
                     InlineKeyboardButton(texts.get_text("admc_remove_tag"), callback_data = make_button_data("rt",user_id,tag_id) )],

                    [InlineKeyboardButton(texts.get_text("admc_show_tag"), callback_data = make_button_data("st",user_id,tag_id) )] 
                ]
    

    reply_buttons = InlineKeyboardMarkup(keyboard)

    bot.sendMessage(chat_id=CHANNEL_ID, text=msg, reply_markup=reply_buttons)
예제 #22
0
def query_handler(bot, update):
    query = update.callback_query

    if int(query.from_user.id) != int(ADMIN_ID): #check if admin
        return

    parts = query.data.split("_")

    if parts[0] == "st":
        send_tag_data_to_private_chat(bot,update,parts[2])

    if parts[0] == "cr":
        clear_reports(bot,update, parts[2])

    if parts[0] == "rt":
        remove_tag(bot,update, parts[2])

    if parts[0] == "nb":
        close_case(bot, update, None, texts.get_text("admc_case_delete_tag"))
    
    if parts[0] == "wu":
        r = dbManager.warn_user(int(parts[1]))

        if r == 1:
            send_private_message(bot, int(parts[1]), texts.get_text("warn_received", update.message.from_user.language_code))
            close_case(bot, update, None, texts.get_text("admc_case_delete_warn_tag"))
        else:
            send_private_message(bot, int(parts[1]), texts.get_text("ban_received", update.message.from_user.language_code))
            close_case(bot, update, None, texts.get_text("admc_case_delete_ban_tag"))


    if parts[0] == "bu":
        dbManager.ban_user(int(parts[1]))
        close_case(bot, update, None, texts.get_text("admc_case_delete_ban_tag"))
예제 #23
0
 def get_stats(self, id, name):
     http = httplib2.Http()
     resp, content = http.request('https://rocketleaguestats.com/profile/' + name + '/steam')
     soup = BeautifulSoup(content, 'html.parser')
     
     try:
         imgSRC = soup.find('img')['src']
         print('Found !')
         src = 'https://rocketleaguestats.com/profile/' + name + '/steam'
         return src
     except TypeError:
         print('Not found !')
         return get_text(id)
예제 #24
0
 def get_picture(self, id, name):
     http = httplib2.Http()
     resp, content = http.request('https://rocketleaguestats.com/profile/' + name + '/steam')
     soup = BeautifulSoup(content, 'html.parser')
     
     try:
         imgSRC = soup.find('img')['src']
         print('Found !')
         urllib.request.urlretrieve(imgSRC, 'temp_stats_picture.png')
         return
     except TypeError:
         print('Not found !')
         return get_text(id)
예제 #25
0
def remove(bot, update):

    if SUSPEND_CREATION:
        update.message.reply_text(
            texts.get_text("suspend_input",
                           update.message.from_user.language_code) +
            "\nTag backup: " + EOF_LINK)
        return

    tag = validate_cmd(update.message.text)

    parallel.send_stats_event(update.message.from_user.id, update.message,
                              "remove")

    if tag is None:
        update.message.reply_text(
            texts.get_text("cmd_rm_use",
                           update.message.from_user.language_code))
        return

    #controlla se e' possibile rimuovere il tag
    res = dbManager.can_write_hashtag(tag, update.message.from_user.id, True)
    if res == 0:
        #rimuovi il tag
        dbManager.delete_hashtag(tag)
        update.message.reply_text(
            texts.get_text("tag_remove_ok",
                           update.message.from_user.language_code))
    else:
        if res == 3:
            update.message.reply_text(
                texts.get_text("rm_tag_free",
                               update.message.from_user.language_code))
        else:
            update.message.reply_text(
                texts.get_text("tag_not_owned",
                               update.message.from_user.language_code))
예제 #26
0
def admin_set(bot, update):
    if is_from_admin(update) == False:
        return

    tag = validate_cmd(update.message.text)

    if tag is None:
        update.message.reply_text(texts.get_text("adm_set_use"))
        return

    #controlla che sia stato quotato un messaggio
    if update.message.reply_to_message is None:
        update.message.reply_text(texts.get_text("quote_missing"))
        return

    #crea il tag
    data = get_message_data(update.message.reply_to_message)

    #controlla validita dati
    if data is None:
        update.message.reply_text(texts.get_text("too_much_chars"))
    else:
        dbManager.create_hashtag(tag, None, data, True)
        update.message.reply_text(texts.get_text("adm_set_ok").format(tag=tag))
예제 #27
0
def index_text(tref, version=None, lang=None):
    """
    Index the text designated by ref.
    If no version and lang are given, this functon will be called for each availble version.
    Currently assumes ref is at section level. 
    """
    #tref = texts.norm_ref(unicode(tref))
    #todo: why the unicode()?
    tref = model.Ref(tref).normal()

    # Recall this function for each specific text version, if non provided
    if not (version and lang):
        for v in texts.get_version_list(tref):
            index_text(tref, version=v["versionTitle"], lang=v["language"])
        return

    # Index each segment of this document individually
    oref = model.Ref(tref).padded_ref()
    if len(oref.sections) < len(oref.index.sectionNames):
        text = texts.get_text(tref,
                              context=0,
                              commentary=False,
                              version=version,
                              lang=lang)
        if "error" in text:
            print text["error"]
        else:
            for i in range(max(len(text["text"]), len(text["he"]))):
                index_text("%s:%d" % (tref, i + 1))

    # Don't try to index docs with depth 3
    if len(oref.sections) < len(oref.index.sectionNames) - 1:
        return

    # Index this document as a whole
    doc = make_text_index_document(tref, version, lang)
    if doc:
        try:
            global doc_count
            if doc_count % 5000 == 0:
                print "[%d] Indexing %s / %s / %s" % (doc_count, tref, version,
                                                      lang)
            es.index('sefaria', 'text', doc,
                     make_text_doc_id(tref, version, lang))
            doc_count += 1
        except Exception, e:
            print "ERROR indexing %s / %s / %s" % (tref, version, lang)
            pprint(e)
예제 #28
0
def edit(bot, update):

    if SUSPEND_CREATION:
        update.message.reply_text(
            texts.get_text("suspend_input",
                           update.message.from_user.language_code) +
            "\nTag backup: " + EOF_LINK)
        return

    if dbManager.is_user_banned(update.message.from_user.id) == True:
        return

    parts = update.message.text.split()

    parallel.send_stats_event(update.message.from_user.id, update.message,
                              "edit")

    if len(parts) != 3:
        update.message.reply_text(
            texts.get_text("cmd_edit_use",
                           update.message.from_user.language_code))
        return

    #controlla i due tag
    old_tag = check_if_hashtag(parts[1])
    new_tag = check_if_hashtag(parts[2])

    if old_tag is None or new_tag is None:
        update.message.reply_text(
            texts.get_text("cmd_edit_use",
                           update.message.from_user.language_code))
        return

    uid = update.message.from_user.id
    #controlla che entrambi gli hashtag siano liberi o del propietario
    if dbManager.can_write_hashtag(old_tag,
                                   uid) == 0 and dbManager.can_write_hashtag(
                                       new_tag, uid) == 0:
        res = dbManager.change_hashtag(old_tag, new_tag)

        if res == 0:
            update.message.reply_text(
                texts.get_text("edit_tag_error",
                               update.message.from_user.language_code))
        else:
            update.message.reply_text(
                texts.get_text("edit_ok",
                               update.message.from_user.language_code).format(
                                   old=old_tag, new=new_tag))

    else:
        update.message.reply_text(
            texts.get_text("edit_perm_error",
                           update.message.from_user.language_code))
예제 #29
0
def make_text_index_document(tref, version, lang):
    """
    Create a document for indexing from the text specified by ref/version/lang
    """
    text = texts.get_text(tref,
                          context=0,
                          commentary=False,
                          version=version,
                          lang=lang)

    if "error" in text:
        print text["error"]
        return None

    if text["type"] == "Talmud":
        title = text["book"] + " Daf " + text["sections"][0]
    elif text["type"] == "Commentary" and text["commentaryCategories"][
            0] == "Talmud":
        title = text["book"] + " Daf " + text["sections"][0]
    else:
        title = text["book"] + " " + " ".join([
            "%s %d" % (p[0], p[1])
            for p in zip(text["sectionNames"], text["sections"])
        ])
    title += " (%s)" % version

    if lang == "he":
        title = text.get("heTitle", "") + " " + title

    content = text["he"] if lang == 'he' else text["text"]
    if not content:
        # Don't bother indexing if there's no content
        return False
    if isinstance(content, list):
        content = " ".join(content)

    return {
        "title": title,
        "ref": tref,
        "version": version,
        "lang": lang,
        "titleVariants": text["titleVariants"],
        "content": content,
        "categories": text["categories"],
    }
예제 #30
0
def text_at_revision(ref, version, lang, revision):
	"""
	Returns the state of a text (identified by ref/version/lang) at revision number 'revision'
	"""
	changes = db.history.find({"ref": ref, "version": version, "language": lang}).sort([['revision', -1]])
	current = texts.get_text(ref, context=0, commentary=False, version=version, lang=lang)
	if "error" in current and not current["error"].startswith("No text found"):
		return current

	textField = "text" if lang == "en" else lang
	text = unicode(current.get(textField, ""))

	for i in range(changes.count()):
		r = changes[i]
		if r["revision"] == revision: break
		patch = dmp.patch_fromText(r["revert_patch"])
		text = dmp.patch_apply(patch, text)[0]

	return text
예제 #31
0
def index_text(ref, version=None, lang=None):
    """
    Index the text designated by ref.
    If no version and lang are given, this functon will be called for each availble version.
    Currently assumes ref is at section level. 
    """
    ref = texts.norm_ref(unicode(ref))

    # Recall this function for each specific text version, if non provided
    if not (version and lang):
        for v in texts.get_version_list(ref):
            index_text(ref, version=v["versionTitle"], lang=v["language"])
        return

    # Index each segment of this document individually
    pRef = texts.parse_ref(ref)
    if len(pRef["sections"]) < len(pRef["sectionNames"]):
        text = texts.get_text(ref,
                              context=0,
                              commentary=False,
                              version=version,
                              lang=lang)
        if "error" in text:
            print text["error"]
        else:
            for i in range(max(len(text["text"]), len(text["he"]))):
                index_text("%s:%d" % (ref, i + 1))

    # Don't try to index docs with depth 3
    if len(pRef["sections"]) < len(pRef["sectionNames"]) - 1:
        return

    # Index this document as a whole
    doc = make_text_index_document(ref, version, lang)
    if doc:
        try:
            es.index(doc, 'sefaria', 'text',
                     make_text_doc_id(ref, version, lang))
            global doc_count
            doc_count += 1
        except Exception, e:
            print "Error indexing %s / %s / %s" % (ref, version, lang)
            print e
예제 #32
0
def index_text(tref, version=None, lang=None):
    """
    Index the text designated by ref.
    If no version and lang are given, this functon will be called for each availble version.
    Currently assumes ref is at section level. 
    """
    #tref = texts.norm_ref(unicode(tref))
    #todo: why the unicode()?
    tref = model.Ref(tref).normal()

    # Recall this function for each specific text version, if non provided
    if not (version and lang):
        for v in texts.get_version_list(tref):
            index_text(tref, version=v["versionTitle"], lang=v["language"])
        return

    # Index each segment of this document individually
    oref = model.Ref(tref).padded_ref()
    if len(oref.sections) < len(oref.index.sectionNames):
        text = texts.get_text(tref, context=0, commentary=False, version=version, lang=lang)
        if "error" in text:
            print text["error"]
        else:
            for i in range(max(len(text["text"]), len(text["he"]))):
                index_text("%s:%d" % (tref, i+1))

    # Don't try to index docs with depth 3
    if len(oref.sections) < len(oref.index.sectionNames) - 1:
        return

    # Index this document as a whole
    doc = make_text_index_document(tref, version, lang)
    if doc:
        try:
            global doc_count
            if doc_count % 5000 == 0:
                print "[%d] Indexing %s / %s / %s" % (doc_count, tref, version, lang)
            es.index('sefaria', 'text', doc, make_text_doc_id(tref, version, lang))
            doc_count += 1
        except Exception, e:
            print "ERROR indexing %s / %s / %s" % (tref, version, lang)
            pprint(e)
예제 #33
0
def claim(bot, update):

    if SUSPEND_CREATION:
        update.message.reply_text(
            texts.get_text("suspend_input",
                           update.message.from_user.language_code) +
            "\nTag backup: " + EOF_LINK)
        return

    if dbManager.is_user_banned(update.message.from_user.id) == True:
        return

    tag = validate_cmd(update.message.text)

    parallel.send_stats_event(update.message.from_user.id, update.message,
                              "claim")

    if tag is None:
        update.message.reply_text(
            texts.get_text("claim_reply",
                           update.message.from_user.language_code))
        return

    #controlla che sia stato quotato un messaggio
    if update.message.reply_to_message is None:
        update.message.reply_text(
            texts.get_text("quote_missing",
                           update.message.from_user.language_code))
        return

    #cerca di creare il tag se possibile
    if dbManager.can_write_hashtag(tag, update.message.from_user.id) == 0:
        data = get_message_data(update.message.reply_to_message)

        #controlla validita dati
        if data is None:
            update.message.reply_text(
                texts.get_text("too_much_chars",
                               update.message.from_user.language_code))
        else:
            dbManager.create_hashtag(tag, update, data, False)
            update.message.reply_text(
                texts.get_text("claim_ok",
                               update.message.from_user.language_code) + tag)

    else:
        update.message.reply_text(
            texts.get_text("claim_error",
                           update.message.from_user.language_code))
예제 #34
0
def count_terms(query={}, lang=None):
    #todo: move to object model.  Maybe.  What's this doing?
    """
	Counts all terms in texts matching query, lang
	Saves reults to terms collection in db. 
	"""
    terms = {}
    bavli_names = db.index.find(query).distinct("title")
    query = {"title": {"$in": bavli_names}}
    refs = counts.generate_refs_list(query)
    lookup_lang = "he" if lang == "ar" else lang

    for ref in refs:
        print ref
        text = texts.get_text(ref, commentary=False)
        for i, line in enumerate(text.get(lookup_lang, [])):
            # strip punctuation
            for c in string.punctuation:
                line = line.replace(c, "")
            these_terms = line.split(" ")
            for term in these_terms:
                line_ref = "%s:%d" % (ref, i + 1)
                term = hebrew.strip_nikkud(term)
                if term in terms:
                    terms[term]["occurrences"] += 1
                    terms[term]["refs"].add(line_ref)
                else:
                    terms[term] = {
                        "term": term,
                        "occurrences": 1,
                        "language": lang,
                        "refs": set([line_ref])
                    }

    for term in terms:
        print term
        # only include up to 20 random ref samples
        sample_size = len(
            terms[term]["refs"]) if len(terms[term]["refs"]) < 20 else 20
        terms[term]["refs"] = list(sample(terms[term]["refs"], sample_size))
        db.terms.save(terms[term])
예제 #35
0
def count_terms(query={}, lang=None):
    #todo: move to object model.  Maybe.  What's this doing?
	"""
	Counts all terms in texts matching query, lang
	Saves reults to terms collection in db. 
	"""
	terms = {}
	bavli_names = db.index.find(query).distinct("title")
	query = {"title": {"$in": bavli_names}}
	refs = counts.generate_refs_list(query)
	lookup_lang = "he" if lang == "ar" else lang

	for ref in refs:
		print ref
		text = texts.get_text(ref, commentary=False)
		for i, line in enumerate(text.get(lookup_lang, [])):
			# strip punctuation
			for c in string.punctuation:
				line = line.replace(c,"")
			these_terms = line.split(" ")
			for term in these_terms:
				line_ref = "%s:%d" % (ref, i+1)
				term = hebrew.strip_nikkud(term)
				if term in terms:
					terms[term]["occurrences"] += 1
					terms[term]["refs"].add(line_ref)
				else:
					terms[term] = {
						"term": term,
						"occurrences": 1,
						"language": lang,
						"refs": set([line_ref])
					}

	for term in terms:
		print term
		# only include up to 20 random ref samples
		sample_size = len(terms[term]["refs"]) if len(terms[term]["refs"]) < 20 else 20
		terms[term]["refs"] = list(sample(terms[term]["refs"], sample_size)) 
		db.terms.save(terms[term])
예제 #36
0
def index_text(ref, version=None, lang=None):
    """
    Index the text designated by ref.
    If no version and lang are given, this functon will be called for each availble version.
    Currently assumes ref is at section level. 
    """
    ref = texts.norm_ref(unicode(ref))

    # Recall this function for each specific text version, if non provided
    if not (version and lang):
        for v in texts.get_version_list(ref):
            index_text(ref, version=v["versionTitle"], lang=v["language"])
        return

    # Index each segment of this document individually
    pRef = texts.parse_ref(ref)
    if len(pRef["sections"]) < len(pRef["sectionNames"]):
        text = texts.get_text(ref, context=0, commentary=False, version=version, lang=lang)
        if "error" in text:
            print text["error"]
        else:
            for i in range(max(len(text["text"]), len(text["he"]))):
                index_text("%s:%d" % (ref, i+1))

    # Don't try to index docs with depth 3
    if len(pRef["sections"]) < len(pRef["sectionNames"]) - 1:
        return

    # Index this document as a whole
    doc = make_text_index_document(ref, version, lang)
    if doc:
        try:
            es.index(doc, 'sefaria', 'text', make_text_doc_id(ref, version, lang))
            global doc_count
            doc_count += 1
        except Exception, e:
            print "Error indexing %s / %s / %s" % (ref, version, lang)
            print e
예제 #37
0
def clear_reports(bot, update, tag_id):
    res = dbManager.remove_reports(tag_id)
    close_case(bot,update,res,texts.get_text("admc_report_clear"))