Example #1
0
def db_sha1(domain, family, bookname):
    conn = db.create_conn(domain=domain, family=family)
    cursor = db.use_db(conn, domain, family)

    q = 'SELECT img_sha1 FROM image WHERE img_name = %s'
    cursor.execute(q, [bookname])
    data = cursor.fetchall()

    cursor.close()
    conn.close()

    return data[0][0] if len(data) else None
Example #2
0
def db_sha1(domain, family, bookname):
    conn = db.create_conn(domain = domain, family = family)
    cursor = db.use_db(conn, domain, family)

    q = 'SELECT img_sha1 FROM image WHERE img_name = %s'
    cursor.execute(q, [bookname])
    data = cursor.fetchall()

    cursor.close()
    conn.close()

    return data[0][0] if len(data) else None
Example #3
0
def get_images_credit(cursor, images):
    if not len(images):
        return []

    images = [ x.replace(' ', '_') for x in images ]
    results = []

    conn = db.create_conn(domain = 'commons', family = 'wiki')
    common_cursor = db.use_db(conn, 'commons', 'wiki')
    for r in get_images_credit_db(common_cursor, images):
        results.append(r[0])
    common_cursor.close()
    conn.close()

    for r in get_images_credit_db(cursor, images):
        results.append(r[0])

    return results
Example #4
0
def get_images_credit(cursor, images):
    if not len(images):
        return []

    images = [x.replace(' ', '_') for x in images]

    conn = db.create_conn(domain = 'commons', family = 'wiki')
    common_cursor = db.use_db(conn, 'commons', 'wiki')
    temp = []
    for r in get_images_credit_db(common_cursor, images):
        temp.append(r[0])
    results = [x[1] for x in get_user_names(common_cursor, temp)]
    common_cursor.close()
    conn.close()

    temp = []
    for r in get_images_credit_db(cursor, images):
        temp.append(r[0])
    results += [x[1] for x in get_user_names(cursor, temp)]

    return results
Example #5
0
def handle_scan_query(params, start_response):
    text = common_html.get_head('pages without scan', css = 'shared.css').encode('utf-8') + '\n  <body>\n'

    if params['lang']:
        try:
            offset = int(params.get('offset', 0))
            limit = min(500, int(params.get('limit', 500)))
            lang = params['lang']
            conn = db.create_conn(domain = lang, family = 'wikisource')
            cursor = db.use_db(conn, domain = lang, family = 'wikisource')
            ns = ws_category.domain_urls[lang][0]
            result = pages_without_scan(ns, cursor)
            result_len = len(result)
            result = result[offset:offset+limit]
            result = [( unicode(x[0], 'utf-8'), x[1]) for x in result]
            text += 'Total: ' + str(result_len) + '<br />'
            next_link = prev_next_link(False, result_len, lang, limit, offset)
            prev_link = prev_next_link(True, result_len, lang, limit, offset)
            text += prev_link + '&#160;' + next_link + '<br /><br />'

            for x in result:
                text += u'<a href="//%s.wikisource.org/wiki/%s">' % (lang, x[0]) +  x[0].replace('_', ' ') + u'</a>, ' + str(x[1]) + u'<br />'

            text += u'<br />' + prev_link + '&#160;' + next_link
            cursor.close()
            conn.close()
            ret_code = '200 OK'
        except:
            utils.print_traceback()
            ret_code = '500 Internal Server Error'
            text = '<h1>' + ret_code + '</h1>'
    else:
        ret_code = '400 Bad Request'
        text = '<h1>' + ret_code + '</h1>'

    text += '  </body>\n</html>'

    return return_response(start_response, text.encode('utf-8'), False, ret_code, 'text/html')
Example #6
0
def get_credit(domain, family, books, pages, images):
    conn = db.create_conn(domain = domain, family = family)
    ns = get_source_ns(domain, family)
    cursor = db.use_db(conn, domain, family)

    books_name = []
    results = {}
    for book in books:
        contribs = get_book_credit(domain, family, cursor, book, ns)
        merge_contrib(results, contribs)
        books_name.append(get_index_ns(domain, family) + ':' + book)

    contribs = get_pages_credit(cursor, pages + books_name, ns)
    merge_contrib(results, contribs)

    for user_name in get_images_credit(cursor, images):
        results.setdefault(user_name, default_userdict())
        results[user_name]['count'] += 1

    conn.close()
    cursor.close()

    return results
Example #7
0
def get_credit(domain, family, books, pages, images):
    conn = db.create_conn(domain = domain, family = family)
    ns = get_source_ns(domain, family)
    cursor = db.use_db(conn, domain, family)

    books_name = []
    results = {}
    for book in books:
        contribs = get_book_credit(domain, family, cursor, book, ns)
        merge_contrib(results, contribs)
        books_name.append(get_index_ns(domain, family) + ':' + book)

    contribs = get_pages_credit(cursor, pages + books_name, ns)
    merge_contrib(results, contribs)

    for user_name in get_images_credit(cursor, images):
        results.setdefault(user_name, default_userdict())
        results[user_name]['count'] += 1

    conn.close()
    cursor.close()

    return results
Example #8
0
        print >> out_fd, '<ol>'

        for d in result:
            print >> out_fd, format_html_line(domain, d[1], d[0])

        print >> out_fd, '</ol>'
    else:
        "Empty result, no Index meet the criteria to be listed in this file."

    print >> out_fd, '\n</body>\n</html>'
    out_fd.close()

    return len(result)

if __name__ == "__main__":
    tot_count = 0
    for domain in all_domain:
        print domain

        #if domain != 'fr':
        #    continue

        conn = db.create_conn(domain = domain, family = 'wikisource')
        cursor = db.use_db(conn, domain = domain, family = 'wikisource')

        tot_count += not_transcluded(domain, cursor)

        cursor.close()
        conn.close()
    print "total:", tot_count
Example #9
0
def get_stats(domains):

    import urllib
    res = {}

    for dom in domains:
        print dom
        conn = db.create_conn(domain = dom, family = 'wikisource')
        cursor = db.use_db(conn, domain = dom, family = 'wikisource')
	ns = urls[dom][0]

	q="select /* SLOW_OK */ count(page_id) as num from page where page_namespace=%d and page_is_redirect=0"%ns
	cursor.execute(q)
	row = cursor.fetchone ()
        num_pages = int(row[0])

	if len(urls[dom])>1:
	    cat3 = urllib.unquote(urls[dom][1])
  	    cat4 = urllib.unquote(urls[dom][2])
	    cursor.execute(catreq(cat3,ns))
	    row = cursor.fetchall()[0]
            num_q3 = int(row[0])
	    cursor.execute(catreq(cat4,ns))
	    row = cursor.fetchall()[0]
            num_q4 = int(row[0])
	else:
	    num_q3 = 0
	    num_q4 = 0

	if len(urls[dom])>3:
	    cat0 = urls[dom][3]
  	    cat2 = urls[dom][4]
	    cursor.execute(catreq(cat0,ns))
	    row = cursor.fetchall()[0]
            num_q0 = int(row[0])
	    cursor.execute(catreq(cat2,ns))
	    row = cursor.fetchall()[0]
            num_q2 = int(row[0])
	else:
	    num_q0 = 0
	    num_q2 = 0

	q = "select /* SLOW_OK */ count(distinct tl_from) as num from templatelinks left join page on page_id=tl_from where tl_namespace=%d and page_namespace=0;"%ns
	cursor.execute(q)
	row = cursor.fetchone ()
        num_trans = int(row[0])

	cursor.execute ("select /* SLOW_OK */ count(distinct page_id) from page where page_namespace=0 and page_is_redirect=0;")
	row = cursor.fetchone ()
        num_texts = int(row[0])

        #disambiguation pages
        # first try the __DISAMBIG__ keyword
        q_disamb = "select count(page_title) from page where page_namespace = 0 and page_is_redirect = 0 and page_id in (select pp_page from page_props where pp_propname = 'disambiguation')"
        cursor.execute(q_disamb)
        row = cursor.fetchone ()
        num_disambig = int(row[0])

        if num_disambig == 0:
            #then test if the message is a template...
            q_disamb = "select /* SLOW_OK */ count(page_title) from page left join templatelinks on page_id=tl_from where page_namespace=0 and tl_namespace=10 and tl_title in ( select pl_title from page left join pagelinks on page_id=pl_from where pl_namespace=10 and page_namespace=8 and page_title='Disambiguationspage' )"
            cursor.execute (q_disamb)
            row = cursor.fetchone ()
            num_disambig = int(row[0])

            if num_disambig==0 and disambiguations.get(dom) :
                q_disamb = "select /* SLOW_OK */ count(page_title) from page left join templatelinks on page_id=tl_from where page_namespace=0 and tl_namespace=10 and tl_title='%s'"%disambiguations.get(dom)
                cursor.execute (q_disamb)
                row = cursor.fetchone ()
                num_disambig = int(row[0])

        if dom in []: #['no']:
            import pywikibot
            qq = "select /* SLOW_OK */ page_title from page where page_namespace=0 and page_is_redirect=0 and page_id not in ( select distinct tl_from from templatelinks left join page on page_id=tl_from where tl_namespace=104 and page_namespace=0 ) and page_id not in ( %s );" % q_disamb.replace("count(page_title)","page_id")
            cursor.execute(qq)
            rows = cursor.fetchall()
	    site = pywikibot.getSite(dom,fam='wikisource')
            f = codecs.open(os.path.expanduser('~/public_html/data/nakedtexts_')+dom+'.html','w',"utf-8")
            f.write("<html><head></head><body>")
            f.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />")
            f.write("<b>Naked texts at "+dom+".wikisource.org</b> (%d)</br/>"%len(rows) )
            f.write("<ul>")
            for row in rows:
                pagename = row[0]
                page = pywikibot.Page(site, pagename.decode('utf8'))
                page_path = site.nice_get_address(page.title(asUrl = True))
                page_url = "http://"+dom+".wikisource.org"+page_path
                s="<li><a href=\"%s\">%s</a></li>"%(page_url,page.title())
                f.write(s)
            f.write("</ul>")
            f.write("</body>")
        
        res[dom] = (num_pages, num_q0, num_q2, num_q3, num_q4, num_trans, num_texts, num_disambig)
        
        cursor.close ()
        conn.close ()
    return res
Example #10
0
def open_db(domain, family, cursor_class = None):
    conn = db.create_conn(domain = domain, family = family)
    cursor = db.use_db(conn, domain, family, cursor_class)

    return conn, cursor
Example #11
0
        for d in result:
            print >> out_fd, format_html_line(domain, d[1], d[0])

        print >> out_fd, '</ol>'
    else:
        "Empty result, no Index meet the criteria to be listed in this file."

    print >> out_fd, '\n</body>\n</html>'
    out_fd.close()

    return len(result)


if __name__ == "__main__":
    tot_count = 0
    for domain in all_domain:
        print domain

        #if domain != 'fr':
        #    continue

        conn = db.create_conn(domain=domain, family='wikisource')
        cursor = db.use_db(conn, domain=domain, family='wikisource')

        tot_count += not_transcluded(domain, cursor)

        cursor.close()
        conn.close()
    print "total:", tot_count
Example #12
0
def get_stats(domains):

    import urllib
    res = {}

    for dom in domains:
        print dom
        conn = db.create_conn(domain = dom, family = 'wikisource')
        cursor = db.use_db(conn, domain = dom, family = 'wikisource')
	ns = urls[dom][0]

	q="select /* SLOW_OK */ count(page_id) as num from page where page_namespace=%d and page_is_redirect=0"%ns
	cursor.execute(q)
	row = cursor.fetchone ()
        num_pages = int(row[0])

	if len(urls[dom])>1:
	    cat3 = urllib.unquote(urls[dom][1])
  	    cat4 = urllib.unquote(urls[dom][2])
	    cursor.execute(catreq(cat3,ns))
	    row = cursor.fetchall()[0]
            num_q3 = int(row[0])
	    cursor.execute(catreq(cat4,ns))
	    row = cursor.fetchall()[0]
            num_q4 = int(row[0])
	else:
	    num_q3 = 0
	    num_q4 = 0

	if len(urls[dom])>3:
	    cat0 = urls[dom][3]
  	    cat2 = urls[dom][4]
	    cursor.execute(catreq(cat0,ns))
	    row = cursor.fetchall()[0]
            num_q0 = int(row[0])
	    cursor.execute(catreq(cat2,ns))
	    row = cursor.fetchall()[0]
            num_q2 = int(row[0])
	else:
	    num_q0 = 0
	    num_q2 = 0

	q = "select /* SLOW_OK */ count(distinct tl_from) as num from templatelinks left join page on page_id=tl_from where tl_namespace=%d and page_namespace=0;"%ns
	cursor.execute(q)
	row = cursor.fetchone ()
        num_trans = int(row[0])

	cursor.execute ("select /* SLOW_OK */ count(distinct page_id) from page where page_namespace=0 and page_is_redirect=0;")
	row = cursor.fetchone ()
        num_texts = int(row[0])

        #disambiguation pages
        # first try the __DISAMBIG__ keyword
        q_disamb = "select count(page_title) from page where page_namespace = 0 and page_is_redirect = 0 and page_id in (select pp_page from page_props where pp_propname = 'disambiguation')"
        cursor.execute(q_disamb)
        row = cursor.fetchone ()
        num_disambig = int(row[0])

        if num_disambig == 0:
            #then test if the message is a template...
            q_disamb = "select /* SLOW_OK */ count(page_title) from page left join templatelinks on page_id=tl_from where page_namespace=0 and tl_namespace=10 and tl_title in ( select pl_title from page left join pagelinks on page_id=pl_from where pl_namespace=10 and page_namespace=8 and page_title='Disambiguationspage' )"
            cursor.execute (q_disamb)
            row = cursor.fetchone ()
            num_disambig = int(row[0])

            if num_disambig==0 and disambiguations.get(dom) :
                q_disamb = "select /* SLOW_OK */ count(page_title) from page left join templatelinks on page_id=tl_from where page_namespace=0 and tl_namespace=10 and tl_title='%s'"%disambiguations.get(dom)
                cursor.execute (q_disamb)
                row = cursor.fetchone ()
                num_disambig = int(row[0])

        if dom=='no':
            import pywikibot
            qq = "select /* SLOW_OK */ page_title from page where page_namespace=0 and page_is_redirect=0 and page_id not in ( select distinct tl_from from templatelinks left join page on page_id=tl_from where tl_namespace=104 and page_namespace=0 ) and page_id not in ( %s );" % q_disamb.replace("count(page_title)","page_id")
            cursor.execute(qq)
            rows = cursor.fetchall()
	    site = pywikibot.getSite(dom,fam='wikisource')
            f = codecs.open(os.path.expanduser('~/public_html/data/nakedtexts_')+dom+'.html','w',"utf-8")
            f.write("<html><head></head><body>")
            f.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />")
            f.write("<b>Naked texts at "+dom+".wikisource.org</b> (%d)</br/>"%len(rows) )
            f.write("<ul>")
            for row in rows:
                pagename = row[0]
                page = pywikibot.Page(site, pagename.decode('utf8'))
                page_path = site.nice_get_address(page.title(asUrl = True))
                page_url = "http://"+dom+".wikisource.org"+page_path
                s="<li><a href=\"%s\">%s</a></li>"%(page_url,page.title())
                f.write(s)
            f.write("</ul>")
            f.write("</body>")
        
        res[dom] = (num_pages, num_q0, num_q2, num_q3, num_q4, num_trans, num_texts, num_disambig)
        
        cursor.close ()
        conn.close ()
    return res