Beispiel #1
0
def rows2txt ():
    db = DdwDb()
    count = db.get_book_count()['count']

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0,page+1):
        step = i * pageSize+94
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                print "book -------------------------------" + str(item["id"])
                path = mkdir(item["id"])
                chs = db.get_chapters(item['id'])
                if chs is not None:
                    for ch in chs:
                        #db.update_book_last_one(item['id'], one['id'], one['title'], one['create_at'])
                        chtext = db.get_chapter_text(ch['id'])
                        if chtext is not None:
                            try:
                                file_path = os.path.join(path, str(ch['id']))
                                text = htmlstrip(chtext["text"])
                                #write to file
                                write2file(file_path, text)
                                #update chapter size and text
                                db.update_chapter_size(ch['id'], len(text))
                                db.update_chapter_text(ch['id'], text)
                                print ch['id']
                            except Exception,ex:
                                log("Error: ch "+ str(ch['id']) + " " + str(Exception) + ":" + str(ex) )
Beispiel #2
0
def rows2txt():
    db = DdwDb()
    count = db.get_book_count()["count"]

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0, page + 1):
        step = i * pageSize + 94
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                print "book -------------------------------" + str(item["id"])
                path = mkdir(item["id"])
                chs = db.get_chapters(item["id"])
                if chs is not None:
                    for ch in chs:
                        # db.update_book_last_one(item['id'], one['id'], one['title'], one['create_at'])
                        chtext = db.get_chapter_text(ch["id"])
                        if chtext is not None:
                            try:
                                file_path = os.path.join(path, str(ch["id"]))
                                text = htmlstrip(chtext["text"])
                                # write to file
                                write2file(file_path, text)
                                # update chapter size and text
                                db.update_chapter_size(ch["id"], len(text))
                                db.update_chapter_text(ch["id"], text)
                                print ch["id"]
                            except Exception, ex:
                                log("Error: ch " + str(ch["id"]) + " " + str(Exception) + ":" + str(ex))
Beispiel #3
0
def save_book(id):
    dw = DdwDb()
    book = dw.get_book(id)
    if book is None:
        print '-------------begin ' + str(id) + '-------------'
        book = get_book_detail(id)

        if book is not None:
            print '-------------saving-------------'
            #author
            author_id = 0
            author = dw.get_author(book['author'])
            if author is None:
                author_id = dw.insert_author(book['author'])
            else:
                author_id = int(author['id'])
            #category
            cate_id = 0
            cate = dw.get_category(book['cate'])
            if cate is None:
                cate_id = dw.insert_category(book['cate'])
            else:
                cate_id = int(cate['id'])
            #book
            desc = book['desc']
            try:
                desc = htmlstrip(desc).strip()
            except Exception, ex:
                log("Error: desc.error")
            dw.insert_book(id, book['name'], desc, author_id, cate_id,
                           book['is_finish'], book['recommend_count'])

            print '-------------end of ' + str(id) + '-----------'
Beispiel #4
0
def save_book (id):
    dw = DdwDb()
    book = dw.get_book(id)
    if book is None:
        print '-------------begin '+ str(id)+'-------------'
        book = get_book_detail(id)

        if book is not None:
            print '-------------saving-------------'
            #author
            author_id = 0
            author = dw.get_author(book['author'])
            if author is None:
                author_id = dw.insert_author(book['author'])
            else:
                author_id = int(author['id'])
            #category
            cate_id = 0
            cate = dw.get_category(book['cate'])
            if cate is None:
                cate_id = dw.insert_category(book['cate'])
            else:
                cate_id = int(cate['id'])
            #book
            desc = book['desc']
            try:
                desc = htmlstrip(desc).strip()
            except Exception,ex:
                log("Error: desc.error")
            dw.insert_book(id, book['name'], desc, author_id, cate_id, book['is_finish'],book['recommend_count'])

            print '-------------end of '+ str(id)+'-----------'
Beispiel #5
0
def write_urls():
    db = DdwDb()
    print os.path.dirname(__file__)
    _tmp_path = "D:\\liubaikui\\jiushulou\\script\\urls"
    _count = 24221
    for i in range(24154, _count + 1):
        book = db.get_book(i)
        if book is not None:
            chs = db.get_chapters(i)
            if chs is not None and len(chs) > 0:
                print "write to " + str(i)
                path = os.path.join(_tmp_path, str(i))
                obj_file = open(path, 'w+')
                urls = []
                for var in chs:
                    t = db.get_chapter_text(var['id'])
                    if t is None:
                        urls.append(str(var['id']) + "\n")
                #print urls
                obj_file.writelines(urls)
                obj_file.close()
Beispiel #6
0
def write_urls ():
    db = DdwDb()
    print os.path.dirname(__file__)
    _tmp_path = "D:\\liubaikui\\jiushulou\\script\\urls"
    _count = 24221
    for i in range(24154, _count+1):
        book = db.get_book(i)
        if book is not None:
            chs = db.get_chapters(i)
            if chs is not None and len(chs) > 0:
                print "write to " + str(i)
                path = os.path.join(_tmp_path, str(i))
                obj_file = open(path, 'w+')
                urls = []
                for var in chs:
                    t = db.get_chapter_text(var['id'])
                    if t is None:
                        urls.append(str(var['id'])+"\n")
                #print urls
                obj_file.writelines(urls)
                obj_file.close()
Beispiel #7
0
def save_chapters(id):
    dw = DdwDb()
    book = dw.get_book(id)
    if book is not None:
        print '-------------begin ' + str(id) + '-------------'
        count = dw.get_chapters_count(id)
        if count is None or count['count'] > 0:
            print count['count']
            print 'Info: get_chapters_count %s > 0' % int(id)
            return
        chapters = get_chapters(id)
        #print chapters
        for item in chapters:
            var = chapters[item]
            v_title = var['title']
            print v_title
            #volume
            v = dw.get_volume(v_title, id)
            v_id = 0
            if v is None:
                v_id = dw.insert_volume(v_title, id)
            else:
                v_id = int(v['id'])
            #chapters
            for ch in var['chs']:
                chapter = dw.get_chapter(ch[0])
                if chapter is None:
                    print ch[1]
                    dw.insert_chapter(ch[0], ch[1], ch[2] * 100, id, v_id)
        #
        one = db.get_book_last_one(id)
        if one is not None:
            #print one
            db.update_book_last_one(id, one['id'], one['title'],
                                    one['create_at'])
        print '-------------end of ' + str(id) + '-----------'
    else:
        print "Info: save_chapters " + str(id) + " not exists!"
Beispiel #8
0
def mk_last ():
    db = DdwDb()
    count = db.get_book_count()['count']

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0,page+1):
        step = i * pageSize
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                one = db.get_book_last_one(item['id'])
                if one is not None:
                    #print one
                    db.update_book_last_one(item['id'], one['id'], one['title'], one['create_at'])
                    print item['id']
Beispiel #9
0
def downcover ():
    db = DdwDb()
    count = db.get_book_count()['count']

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0,page+1):
        step = i * pageSize
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                print "book -------------------------------" + str(item["id"])
                b = db.get_book(item["id"])
                if b is not None and b["cover"] == "":
                    cover = downimg(item["id"])
                    print cover
                    if cover != "":
                        db.update_book_cover(item['id'], cover)
Beispiel #10
0
def mkdesc ():
    db = DdwDb()
    count = db.get_book_count()['count']

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0,page+1):
        step = i * pageSize
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                print "book -------------------------------" + str(item["id"])
                b = db.get_book(item['id'])
                try:
                    desc = htmlstrip(b["description"]).strip()
                    db.update_book_desc(item['id'], desc)
                except Exception,ex:
                    log("Error: mkdesc "+ str(item['id']) + " " + str(Exception) + ":" + str(ex),"desc.error")
Beispiel #11
0
def save_chapters (id):
    dw = DdwDb()
    book = dw.get_book(id)
    if book is not None:
        print '-------------begin '+ str(id)+'-------------'
        count = dw.get_chapters_count(id)
        if count is None or count['count'] > 0:
            print count['count']
            print 'Info: get_chapters_count %s > 0' % int(id)
            return
        chapters = get_chapters(id)
        #print chapters
        for item in chapters :
            var = chapters[item]
            v_title = var['title']
            print v_title
            #volume
            v = dw.get_volume(v_title,id)
            v_id = 0
            if v is None:
                v_id = dw.insert_volume(v_title,id)
            else:
                v_id = int(v['id'])
            #chapters
            for ch in var['chs']:
                chapter = dw.get_chapter(ch[0])
                if chapter is None:
                    print ch[1]
                    dw.insert_chapter(ch[0], ch[1], ch[2]*100, id, v_id)
        #
        one = db.get_book_last_one(id)
        if one is not None:
            #print one
            db.update_book_last_one(id, one['id'], one['title'], one['create_at'])
        print '-------------end of '+ str(id)+'-----------'
    else:
        print "Info: save_chapters "+ str(id) + " not exists!"
Beispiel #12
0
def mk_last():
    db = DdwDb()
    count = db.get_book_count()["count"]

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0, page + 1):
        step = i * pageSize
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                one = db.get_book_last_one(item["id"])
                if one is not None:
                    # print one
                    db.update_book_last_one(item["id"], one["id"], one["title"], one["create_at"])
                    print item["id"]
Beispiel #13
0
def downcover():
    db = DdwDb()
    count = db.get_book_count()["count"]

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0, page + 1):
        step = i * pageSize
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                print "book -------------------------------" + str(item["id"])
                b = db.get_book(item["id"])
                if b is not None and b["cover"] == "":
                    cover = downimg(item["id"])
                    print cover
                    if cover != "":
                        db.update_book_cover(item["id"], cover)
Beispiel #14
0
def mkdesc():
    db = DdwDb()
    count = db.get_book_count()["count"]

    pageSize = 100
    page = count / pageSize + 1

    for i in range(0, page + 1):
        step = i * pageSize
        rows = db.list(step)
        if rows is not None:
            for item in rows:
                print "book -------------------------------" + str(item["id"])
                b = db.get_book(item["id"])
                try:
                    desc = htmlstrip(b["description"]).strip()
                    db.update_book_desc(item["id"], desc)
                except Exception, ex:
                    log("Error: mkdesc " + str(item["id"]) + " " + str(Exception) + ":" + str(ex), "desc.error")
Beispiel #15
0
def save_text(id):
    dw = DdwDb()
    book = dw.get_book(id)
    if book is not None:
        print '-------------begin ' + str(id) + '-------------'
        chs = dw.get_chapters(id)
        if chs is not None or len(chs) > 0:
            for var in chs:
                print var['id']
                text = ''
                t = dw.get_chapter_text(var['id'])
                if t is None:
                    try:
                        text = get_chapter(id, var['id'])
                        if text is not None and text != '':
                            dw.insert_chapter_text(var['id'], text)
                            print '-------------end of ' + str(
                                id) + '-----------'
                    except Exception, ex:
                        log("Error: get_chapter " + str(id) + " " +
                            str(Exception) + ":" + str(ex))
                print "Info: chapter " + str(var['id']) + " exists"
        else:
            print 'Info: have no chapters'
Beispiel #16
0
def save_text (id):
    dw = DdwDb()
    book = dw.get_book(id)
    if book is not None:
        print '-------------begin '+ str(id)+'-------------'
        chs = dw.get_chapters(id)
        if chs is not None or len(chs) > 0:
            for var in chs:
                print var['id']
                text = ''
                t = dw.get_chapter_text(var['id'])
                if t is None:
                    try:
                        text = get_chapter(id, var['id'])
                        if text is not None and text != '':
                            dw.insert_chapter_text(var['id'], text)
                            print '-------------end of '+ str(id)+'-----------'
                    except Exception,ex:
                        log("Error: get_chapter "+ str(id) + " " + str(Exception)+":"+str(ex) )
                print "Info: chapter "+str(var['id'])+" exists"
        else:
            print 'Info: have no chapters'