def db_sha1(domain, family, bookname):
conn = db.create_conn(domain=domain, family=family)
cursor = db.use_db(conn, domain, family)
q = 'SELECT img_sha1 FROM image WHERE img_name = %s'
cursor.execute(q, [bookname])
data = cursor.fetchall()
cursor.close()
conn.close()
return data[0][0] if len(data) else None
def db_sha1(domain, family, bookname):
conn = db.create_conn(domain = domain, family = family)
cursor = db.use_db(conn, domain, family)
q = 'SELECT img_sha1 FROM image WHERE img_name = %s'
cursor.execute(q, [bookname])
data = cursor.fetchall()
cursor.close()
conn.close()
return data[0][0] if len(data) else None
def get_images_credit(cursor, images):
if not len(images):
return []
images = [ x.replace(' ', '_') for x in images ]
results = []
conn = db.create_conn(domain = 'commons', family = 'wiki')
common_cursor = db.use_db(conn, 'commons', 'wiki')
for r in get_images_credit_db(common_cursor, images):
results.append(r[0])
common_cursor.close()
conn.close()
for r in get_images_credit_db(cursor, images):
results.append(r[0])
return results
def get_images_credit(cursor, images):
if not len(images):
return []
images = [x.replace(' ', '_') for x in images]
conn = db.create_conn(domain = 'commons', family = 'wiki')
common_cursor = db.use_db(conn, 'commons', 'wiki')
temp = []
for r in get_images_credit_db(common_cursor, images):
temp.append(r[0])
results = [x[1] for x in get_user_names(common_cursor, temp)]
common_cursor.close()
conn.close()
temp = []
for r in get_images_credit_db(cursor, images):
temp.append(r[0])
results += [x[1] for x in get_user_names(cursor, temp)]
return results
def handle_scan_query(params, start_response):
text = common_html.get_head('pages without scan', css = 'shared.css').encode('utf-8') + '\n <body>\n'
if params['lang']:
try:
offset = int(params.get('offset', 0))
limit = min(500, int(params.get('limit', 500)))
lang = params['lang']
conn = db.create_conn(domain = lang, family = 'wikisource')
cursor = db.use_db(conn, domain = lang, family = 'wikisource')
ns = ws_category.domain_urls[lang][0]
result = pages_without_scan(ns, cursor)
result_len = len(result)
result = result[offset:offset+limit]
result = [( unicode(x[0], 'utf-8'), x[1]) for x in result]
text += 'Total: ' + str(result_len) + '<br />'
next_link = prev_next_link(False, result_len, lang, limit, offset)
prev_link = prev_next_link(True, result_len, lang, limit, offset)
text += prev_link + ' ' + next_link + '<br /><br />'
for x in result:
text += u'<a href="//%s.wikisource.org/wiki/%s">' % (lang, x[0]) + x[0].replace('_', ' ') + u'</a>, ' + str(x[1]) + u'<br />'
text += u'<br />' + prev_link + ' ' + next_link
cursor.close()
conn.close()
ret_code = '200 OK'
except:
utils.print_traceback()
ret_code = '500 Internal Server Error'
text = '<h1>' + ret_code + '</h1>'
else:
ret_code = '400 Bad Request'
text = '<h1>' + ret_code + '</h1>'
text += ' </body>\n</html>'
return return_response(start_response, text.encode('utf-8'), False, ret_code, 'text/html')
def get_credit(domain, family, books, pages, images):
conn = db.create_conn(domain = domain, family = family)
ns = get_source_ns(domain, family)
cursor = db.use_db(conn, domain, family)
books_name = []
results = {}
for book in books:
contribs = get_book_credit(domain, family, cursor, book, ns)
merge_contrib(results, contribs)
books_name.append(get_index_ns(domain, family) + ':' + book)
contribs = get_pages_credit(cursor, pages + books_name, ns)
merge_contrib(results, contribs)
for user_name in get_images_credit(cursor, images):
results.setdefault(user_name, default_userdict())
results[user_name]['count'] += 1
conn.close()
cursor.close()
return results
def get_credit(domain, family, books, pages, images):
conn = db.create_conn(domain = domain, family = family)
ns = get_source_ns(domain, family)
cursor = db.use_db(conn, domain, family)
books_name = []
results = {}
for book in books:
contribs = get_book_credit(domain, family, cursor, book, ns)
merge_contrib(results, contribs)
books_name.append(get_index_ns(domain, family) + ':' + book)
contribs = get_pages_credit(cursor, pages + books_name, ns)
merge_contrib(results, contribs)
for user_name in get_images_credit(cursor, images):
results.setdefault(user_name, default_userdict())
results[user_name]['count'] += 1
conn.close()
cursor.close()
return results
print >> out_fd, '<ol>'
for d in result:
print >> out_fd, format_html_line(domain, d[1], d[0])
print >> out_fd, '</ol>'
else:
"Empty result, no Index meet the criteria to be listed in this file."
print >> out_fd, '\n</body>\n</html>'
out_fd.close()
return len(result)
if __name__ == "__main__":
tot_count = 0
for domain in all_domain:
print domain
#if domain != 'fr':
# continue
conn = db.create_conn(domain = domain, family = 'wikisource')
cursor = db.use_db(conn, domain = domain, family = 'wikisource')
tot_count += not_transcluded(domain, cursor)
cursor.close()
conn.close()
print "total:", tot_count
def get_stats(domains):
import urllib
res = {}
for dom in domains:
print dom
conn = db.create_conn(domain = dom, family = 'wikisource')
cursor = db.use_db(conn, domain = dom, family = 'wikisource')
ns = urls[dom][0]
q="select /* SLOW_OK */ count(page_id) as num from page where page_namespace=%d and page_is_redirect=0"%ns
cursor.execute(q)
row = cursor.fetchone ()
num_pages = int(row[0])
if len(urls[dom])>1:
cat3 = urllib.unquote(urls[dom][1])
cat4 = urllib.unquote(urls[dom][2])
cursor.execute(catreq(cat3,ns))
row = cursor.fetchall()[0]
num_q3 = int(row[0])
cursor.execute(catreq(cat4,ns))
row = cursor.fetchall()[0]
num_q4 = int(row[0])
else:
num_q3 = 0
num_q4 = 0
if len(urls[dom])>3:
cat0 = urls[dom][3]
cat2 = urls[dom][4]
cursor.execute(catreq(cat0,ns))
row = cursor.fetchall()[0]
num_q0 = int(row[0])
cursor.execute(catreq(cat2,ns))
row = cursor.fetchall()[0]
num_q2 = int(row[0])
else:
num_q0 = 0
num_q2 = 0
q = "select /* SLOW_OK */ count(distinct tl_from) as num from templatelinks left join page on page_id=tl_from where tl_namespace=%d and page_namespace=0;"%ns
cursor.execute(q)
row = cursor.fetchone ()
num_trans = int(row[0])
cursor.execute ("select /* SLOW_OK */ count(distinct page_id) from page where page_namespace=0 and page_is_redirect=0;")
row = cursor.fetchone ()
num_texts = int(row[0])
#disambiguation pages
# first try the __DISAMBIG__ keyword
q_disamb = "select count(page_title) from page where page_namespace = 0 and page_is_redirect = 0 and page_id in (select pp_page from page_props where pp_propname = 'disambiguation')"
cursor.execute(q_disamb)
row = cursor.fetchone ()
num_disambig = int(row[0])
if num_disambig == 0:
#then test if the message is a template...
q_disamb = "select /* SLOW_OK */ count(page_title) from page left join templatelinks on page_id=tl_from where page_namespace=0 and tl_namespace=10 and tl_title in ( select pl_title from page left join pagelinks on page_id=pl_from where pl_namespace=10 and page_namespace=8 and page_title='Disambiguationspage' )"
cursor.execute (q_disamb)
row = cursor.fetchone ()
num_disambig = int(row[0])
if num_disambig==0 and disambiguations.get(dom) :
q_disamb = "select /* SLOW_OK */ count(page_title) from page left join templatelinks on page_id=tl_from where page_namespace=0 and tl_namespace=10 and tl_title='%s'"%disambiguations.get(dom)
cursor.execute (q_disamb)
row = cursor.fetchone ()
num_disambig = int(row[0])
if dom in []: #['no']:
import pywikibot
qq = "select /* SLOW_OK */ page_title from page where page_namespace=0 and page_is_redirect=0 and page_id not in ( select distinct tl_from from templatelinks left join page on page_id=tl_from where tl_namespace=104 and page_namespace=0 ) and page_id not in ( %s );" % q_disamb.replace("count(page_title)","page_id")
cursor.execute(qq)
rows = cursor.fetchall()
site = pywikibot.getSite(dom,fam='wikisource')
f = codecs.open(os.path.expanduser('~/public_html/data/nakedtexts_')+dom+'.html','w',"utf-8")
f.write("<html><head></head><body>")
f.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />")
f.write("<b>Naked texts at "+dom+".wikisource.org</b> (%d)</br/>"%len(rows) )
f.write("<ul>")
for row in rows:
pagename = row[0]
page = pywikibot.Page(site, pagename.decode('utf8'))
page_path = site.nice_get_address(page.title(asUrl = True))
page_url = "http://"+dom+".wikisource.org"+page_path
s="<li><a href=\"%s\">%s</a></li>"%(page_url,page.title())
f.write(s)
f.write("</ul>")
f.write("</body>")
res[dom] = (num_pages, num_q0, num_q2, num_q3, num_q4, num_trans, num_texts, num_disambig)
cursor.close ()
conn.close ()
return res
def open_db(domain, family, cursor_class = None):
conn = db.create_conn(domain = domain, family = family)
cursor = db.use_db(conn, domain, family, cursor_class)
return conn, cursor
for d in result:
print >> out_fd, format_html_line(domain, d[1], d[0])
print >> out_fd, '</ol>'
else:
"Empty result, no Index meet the criteria to be listed in this file."
print >> out_fd, '\n</body>\n</html>'
out_fd.close()
return len(result)
if __name__ == "__main__":
tot_count = 0
for domain in all_domain:
print domain
#if domain != 'fr':
# continue
conn = db.create_conn(domain=domain, family='wikisource')
cursor = db.use_db(conn, domain=domain, family='wikisource')
tot_count += not_transcluded(domain, cursor)
cursor.close()
conn.close()
print "total:", tot_count
def get_stats(domains):
import urllib
res = {}
for dom in domains:
print dom
conn = db.create_conn(domain = dom, family = 'wikisource')
cursor = db.use_db(conn, domain = dom, family = 'wikisource')
ns = urls[dom][0]
q="select /* SLOW_OK */ count(page_id) as num from page where page_namespace=%d and page_is_redirect=0"%ns
cursor.execute(q)
row = cursor.fetchone ()
num_pages = int(row[0])
if len(urls[dom])>1:
cat3 = urllib.unquote(urls[dom][1])
cat4 = urllib.unquote(urls[dom][2])
cursor.execute(catreq(cat3,ns))
row = cursor.fetchall()[0]
num_q3 = int(row[0])
cursor.execute(catreq(cat4,ns))
row = cursor.fetchall()[0]
num_q4 = int(row[0])
else:
num_q3 = 0
num_q4 = 0
if len(urls[dom])>3:
cat0 = urls[dom][3]
cat2 = urls[dom][4]
cursor.execute(catreq(cat0,ns))
row = cursor.fetchall()[0]
num_q0 = int(row[0])
cursor.execute(catreq(cat2,ns))
row = cursor.fetchall()[0]
num_q2 = int(row[0])
else:
num_q0 = 0
num_q2 = 0
q = "select /* SLOW_OK */ count(distinct tl_from) as num from templatelinks left join page on page_id=tl_from where tl_namespace=%d and page_namespace=0;"%ns
cursor.execute(q)
row = cursor.fetchone ()
num_trans = int(row[0])
cursor.execute ("select /* SLOW_OK */ count(distinct page_id) from page where page_namespace=0 and page_is_redirect=0;")
row = cursor.fetchone ()
num_texts = int(row[0])
#disambiguation pages
# first try the __DISAMBIG__ keyword
q_disamb = "select count(page_title) from page where page_namespace = 0 and page_is_redirect = 0 and page_id in (select pp_page from page_props where pp_propname = 'disambiguation')"
cursor.execute(q_disamb)
row = cursor.fetchone ()
num_disambig = int(row[0])
if num_disambig == 0:
#then test if the message is a template...
q_disamb = "select /* SLOW_OK */ count(page_title) from page left join templatelinks on page_id=tl_from where page_namespace=0 and tl_namespace=10 and tl_title in ( select pl_title from page left join pagelinks on page_id=pl_from where pl_namespace=10 and page_namespace=8 and page_title='Disambiguationspage' )"
cursor.execute (q_disamb)
row = cursor.fetchone ()
num_disambig = int(row[0])
if num_disambig==0 and disambiguations.get(dom) :
q_disamb = "select /* SLOW_OK */ count(page_title) from page left join templatelinks on page_id=tl_from where page_namespace=0 and tl_namespace=10 and tl_title='%s'"%disambiguations.get(dom)
cursor.execute (q_disamb)
row = cursor.fetchone ()
num_disambig = int(row[0])
if dom=='no':
import pywikibot
qq = "select /* SLOW_OK */ page_title from page where page_namespace=0 and page_is_redirect=0 and page_id not in ( select distinct tl_from from templatelinks left join page on page_id=tl_from where tl_namespace=104 and page_namespace=0 ) and page_id not in ( %s );" % q_disamb.replace("count(page_title)","page_id")
cursor.execute(qq)
rows = cursor.fetchall()
site = pywikibot.getSite(dom,fam='wikisource')
f = codecs.open(os.path.expanduser('~/public_html/data/nakedtexts_')+dom+'.html','w',"utf-8")
f.write("<html><head></head><body>")
f.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />")
f.write("<b>Naked texts at "+dom+".wikisource.org</b> (%d)</br/>"%len(rows) )
f.write("<ul>")
for row in rows:
pagename = row[0]
page = pywikibot.Page(site, pagename.decode('utf8'))
page_path = site.nice_get_address(page.title(asUrl = True))
page_url = "http://"+dom+".wikisource.org"+page_path
s="<li><a href=\"%s\">%s</a></li>"%(page_url,page.title())
f.write(s)
f.write("</ul>")
f.write("</body>")
res[dom] = (num_pages, num_q0, num_q2, num_q3, num_q4, num_trans, num_texts, num_disambig)
cursor.close ()
conn.close ()
return res