def rows2txt ():
db = DdwDb()
count = db.get_book_count()['count']
pageSize = 100
page = count / pageSize + 1
for i in range(0,page+1):
step = i * pageSize+94
rows = db.list(step)
if rows is not None:
for item in rows:
print "book -------------------------------" + str(item["id"])
path = mkdir(item["id"])
chs = db.get_chapters(item['id'])
if chs is not None:
for ch in chs:
#db.update_book_last_one(item['id'], one['id'], one['title'], one['create_at'])
chtext = db.get_chapter_text(ch['id'])
if chtext is not None:
try:
file_path = os.path.join(path, str(ch['id']))
text = htmlstrip(chtext["text"])
#write to file
write2file(file_path, text)
#update chapter size and text
db.update_chapter_size(ch['id'], len(text))
db.update_chapter_text(ch['id'], text)
print ch['id']
except Exception,ex:
log("Error: ch "+ str(ch['id']) + " " + str(Exception) + ":" + str(ex) )
def rows2txt():
db = DdwDb()
count = db.get_book_count()["count"]
pageSize = 100
page = count / pageSize + 1
for i in range(0, page + 1):
step = i * pageSize + 94
rows = db.list(step)
if rows is not None:
for item in rows:
print "book -------------------------------" + str(item["id"])
path = mkdir(item["id"])
chs = db.get_chapters(item["id"])
if chs is not None:
for ch in chs:
# db.update_book_last_one(item['id'], one['id'], one['title'], one['create_at'])
chtext = db.get_chapter_text(ch["id"])
if chtext is not None:
try:
file_path = os.path.join(path, str(ch["id"]))
text = htmlstrip(chtext["text"])
# write to file
write2file(file_path, text)
# update chapter size and text
db.update_chapter_size(ch["id"], len(text))
db.update_chapter_text(ch["id"], text)
print ch["id"]
except Exception, ex:
log("Error: ch " + str(ch["id"]) + " " + str(Exception) + ":" + str(ex))
def save_book(id):
dw = DdwDb()
book = dw.get_book(id)
if book is None:
print '-------------begin ' + str(id) + '-------------'
book = get_book_detail(id)
if book is not None:
print '-------------saving-------------'
#author
author_id = 0
author = dw.get_author(book['author'])
if author is None:
author_id = dw.insert_author(book['author'])
else:
author_id = int(author['id'])
#category
cate_id = 0
cate = dw.get_category(book['cate'])
if cate is None:
cate_id = dw.insert_category(book['cate'])
else:
cate_id = int(cate['id'])
#book
desc = book['desc']
try:
desc = htmlstrip(desc).strip()
except Exception, ex:
log("Error: desc.error")
dw.insert_book(id, book['name'], desc, author_id, cate_id,
book['is_finish'], book['recommend_count'])
print '-------------end of ' + str(id) + '-----------'
def save_book (id):
dw = DdwDb()
book = dw.get_book(id)
if book is None:
print '-------------begin '+ str(id)+'-------------'
book = get_book_detail(id)
if book is not None:
print '-------------saving-------------'
#author
author_id = 0
author = dw.get_author(book['author'])
if author is None:
author_id = dw.insert_author(book['author'])
else:
author_id = int(author['id'])
#category
cate_id = 0
cate = dw.get_category(book['cate'])
if cate is None:
cate_id = dw.insert_category(book['cate'])
else:
cate_id = int(cate['id'])
#book
desc = book['desc']
try:
desc = htmlstrip(desc).strip()
except Exception,ex:
log("Error: desc.error")
dw.insert_book(id, book['name'], desc, author_id, cate_id, book['is_finish'],book['recommend_count'])
print '-------------end of '+ str(id)+'-----------'
def write_urls():
db = DdwDb()
print os.path.dirname(__file__)
_tmp_path = "D:\\liubaikui\\jiushulou\\script\\urls"
_count = 24221
for i in range(24154, _count + 1):
book = db.get_book(i)
if book is not None:
chs = db.get_chapters(i)
if chs is not None and len(chs) > 0:
print "write to " + str(i)
path = os.path.join(_tmp_path, str(i))
obj_file = open(path, 'w+')
urls = []
for var in chs:
t = db.get_chapter_text(var['id'])
if t is None:
urls.append(str(var['id']) + "\n")
#print urls
obj_file.writelines(urls)
obj_file.close()
def write_urls ():
db = DdwDb()
print os.path.dirname(__file__)
_tmp_path = "D:\\liubaikui\\jiushulou\\script\\urls"
_count = 24221
for i in range(24154, _count+1):
book = db.get_book(i)
if book is not None:
chs = db.get_chapters(i)
if chs is not None and len(chs) > 0:
print "write to " + str(i)
path = os.path.join(_tmp_path, str(i))
obj_file = open(path, 'w+')
urls = []
for var in chs:
t = db.get_chapter_text(var['id'])
if t is None:
urls.append(str(var['id'])+"\n")
#print urls
obj_file.writelines(urls)
obj_file.close()
def save_chapters(id):
dw = DdwDb()
book = dw.get_book(id)
if book is not None:
print '-------------begin ' + str(id) + '-------------'
count = dw.get_chapters_count(id)
if count is None or count['count'] > 0:
print count['count']
print 'Info: get_chapters_count %s > 0' % int(id)
return
chapters = get_chapters(id)
#print chapters
for item in chapters:
var = chapters[item]
v_title = var['title']
print v_title
#volume
v = dw.get_volume(v_title, id)
v_id = 0
if v is None:
v_id = dw.insert_volume(v_title, id)
else:
v_id = int(v['id'])
#chapters
for ch in var['chs']:
chapter = dw.get_chapter(ch[0])
if chapter is None:
print ch[1]
dw.insert_chapter(ch[0], ch[1], ch[2] * 100, id, v_id)
#
one = db.get_book_last_one(id)
if one is not None:
#print one
db.update_book_last_one(id, one['id'], one['title'],
one['create_at'])
print '-------------end of ' + str(id) + '-----------'
else:
print "Info: save_chapters " + str(id) + " not exists!"
def mk_last ():
db = DdwDb()
count = db.get_book_count()['count']
pageSize = 100
page = count / pageSize + 1
for i in range(0,page+1):
step = i * pageSize
rows = db.list(step)
if rows is not None:
for item in rows:
one = db.get_book_last_one(item['id'])
if one is not None:
#print one
db.update_book_last_one(item['id'], one['id'], one['title'], one['create_at'])
print item['id']
def downcover ():
db = DdwDb()
count = db.get_book_count()['count']
pageSize = 100
page = count / pageSize + 1
for i in range(0,page+1):
step = i * pageSize
rows = db.list(step)
if rows is not None:
for item in rows:
print "book -------------------------------" + str(item["id"])
b = db.get_book(item["id"])
if b is not None and b["cover"] == "":
cover = downimg(item["id"])
print cover
if cover != "":
db.update_book_cover(item['id'], cover)
def mkdesc ():
db = DdwDb()
count = db.get_book_count()['count']
pageSize = 100
page = count / pageSize + 1
for i in range(0,page+1):
step = i * pageSize
rows = db.list(step)
if rows is not None:
for item in rows:
print "book -------------------------------" + str(item["id"])
b = db.get_book(item['id'])
try:
desc = htmlstrip(b["description"]).strip()
db.update_book_desc(item['id'], desc)
except Exception,ex:
log("Error: mkdesc "+ str(item['id']) + " " + str(Exception) + ":" + str(ex),"desc.error")
def save_chapters (id):
dw = DdwDb()
book = dw.get_book(id)
if book is not None:
print '-------------begin '+ str(id)+'-------------'
count = dw.get_chapters_count(id)
if count is None or count['count'] > 0:
print count['count']
print 'Info: get_chapters_count %s > 0' % int(id)
return
chapters = get_chapters(id)
#print chapters
for item in chapters :
var = chapters[item]
v_title = var['title']
print v_title
#volume
v = dw.get_volume(v_title,id)
v_id = 0
if v is None:
v_id = dw.insert_volume(v_title,id)
else:
v_id = int(v['id'])
#chapters
for ch in var['chs']:
chapter = dw.get_chapter(ch[0])
if chapter is None:
print ch[1]
dw.insert_chapter(ch[0], ch[1], ch[2]*100, id, v_id)
#
one = db.get_book_last_one(id)
if one is not None:
#print one
db.update_book_last_one(id, one['id'], one['title'], one['create_at'])
print '-------------end of '+ str(id)+'-----------'
else:
print "Info: save_chapters "+ str(id) + " not exists!"
def mk_last():
db = DdwDb()
count = db.get_book_count()["count"]
pageSize = 100
page = count / pageSize + 1
for i in range(0, page + 1):
step = i * pageSize
rows = db.list(step)
if rows is not None:
for item in rows:
one = db.get_book_last_one(item["id"])
if one is not None:
# print one
db.update_book_last_one(item["id"], one["id"], one["title"], one["create_at"])
print item["id"]
def downcover():
db = DdwDb()
count = db.get_book_count()["count"]
pageSize = 100
page = count / pageSize + 1
for i in range(0, page + 1):
step = i * pageSize
rows = db.list(step)
if rows is not None:
for item in rows:
print "book -------------------------------" + str(item["id"])
b = db.get_book(item["id"])
if b is not None and b["cover"] == "":
cover = downimg(item["id"])
print cover
if cover != "":
db.update_book_cover(item["id"], cover)
def mkdesc():
db = DdwDb()
count = db.get_book_count()["count"]
pageSize = 100
page = count / pageSize + 1
for i in range(0, page + 1):
step = i * pageSize
rows = db.list(step)
if rows is not None:
for item in rows:
print "book -------------------------------" + str(item["id"])
b = db.get_book(item["id"])
try:
desc = htmlstrip(b["description"]).strip()
db.update_book_desc(item["id"], desc)
except Exception, ex:
log("Error: mkdesc " + str(item["id"]) + " " + str(Exception) + ":" + str(ex), "desc.error")
def save_text(id):
dw = DdwDb()
book = dw.get_book(id)
if book is not None:
print '-------------begin ' + str(id) + '-------------'
chs = dw.get_chapters(id)
if chs is not None or len(chs) > 0:
for var in chs:
print var['id']
text = ''
t = dw.get_chapter_text(var['id'])
if t is None:
try:
text = get_chapter(id, var['id'])
if text is not None and text != '':
dw.insert_chapter_text(var['id'], text)
print '-------------end of ' + str(
id) + '-----------'
except Exception, ex:
log("Error: get_chapter " + str(id) + " " +
str(Exception) + ":" + str(ex))
print "Info: chapter " + str(var['id']) + " exists"
else:
print 'Info: have no chapters'
def save_text (id):
dw = DdwDb()
book = dw.get_book(id)
if book is not None:
print '-------------begin '+ str(id)+'-------------'
chs = dw.get_chapters(id)
if chs is not None or len(chs) > 0:
for var in chs:
print var['id']
text = ''
t = dw.get_chapter_text(var['id'])
if t is None:
try:
text = get_chapter(id, var['id'])
if text is not None and text != '':
dw.insert_chapter_text(var['id'], text)
print '-------------end of '+ str(id)+'-----------'
except Exception,ex:
log("Error: get_chapter "+ str(id) + " " + str(Exception)+":"+str(ex) )
print "Info: chapter "+str(var['id'])+" exists"
else:
print 'Info: have no chapters'