def series_view(self):
_ = self.R
slug = _.matchdict.get('seriesSlug', "No Title")
print(slug)
qry = Manga.query
manga = qry.filter(Manga.slug == slug.strip()).first()
if manga is not None:
filename = '/'.join([manga.id, manga.thumb])
thumb = _.storage.url(filename)
aka = manga.aka
artists = manga.get_artist()
authors = manga.get_authors()
description = manga.description
name = manga.title
last = Manga.last_chapter(manga.id)
last_chapter = ' '.join([str(last.chapter), last.title])
return dict(
aka=aka,
url='/manga/{slug}'.format(slug=slug),
thumb_url=thumb,
artists=artists,
authors=authors,
description=description,
name=name,
last_chapter=last_chapter
)
return {'project': 'moori'}
def chapter_view(self):
_ = self.R
slug = _.matchdict.get('seriesSlug', "No Title")
chap_slug = _.matchdict.get('chapterSlug', "No Title")
# cari manga by slug
manga = Manga.query.filter(Manga.slug == slug).first()
if manga is not None:
filename = '/'.join([manga.id, manga.thumb])
thumb = _.storage.url(filename)
aka = manga.aka
artists = manga.get_artist()
authors = manga.get_authors()
description = manga.description
name = manga.title
last = Manga.last_chapter(manga.id)
last_chapter = ' '.join([str(last.chapter), last.title])
# cari chapter manga
chapter = manga.get_chapter(manga, chap_slug)
return dict(
aka=aka,
url='/chapter/{slug}/{chap}'.format(slug=slug, chap=chap_slug),
thumb_url=thumb,
artists=artists,
authors=authors,
description=description,
name=' '.join([name, '|', 'Ch.', str(chapter.chapter).replace('.0', ''), chapter.title]),
last_chapter=last_chapter
)
return {'project': 'moori'}
def save_new(self):
_ = self.R
with ResponseHTTP(_.response) as resp:
_in = u'Failed'
code, status = ResponseHTTP.BAD_REQUEST
# for key, value in _.params.iteritems():
# print(":".join([key, value]))
if _.params.get('title', None) is not None:
manga = Manga(
_.params.get('type', 'kk'),
_.params.get('title', None),
_.params.get('released', None),
_.params.get('genres', None),
_.params.get('authors', None),
_.params.get('artist', None),
_.params.get('aka', None),
_.params.get('description', None),
_.params.get('status', None)
)
manga.category = _.params.get('category', 'ja')
_.db.add(manga)
mng_tmp = Manga.query.filter(Manga.slug == slugist("-".join([_.params.get('type', 'kk'), _.params.get('title', None)]))).first()
temps_path = _.storage.path('/'.join(['temps', _.params.get('uuid', None)]))
for root, dirs, files in walk(temps_path):
LOG.info(files)
for f in files:
fpath = '/'.join([temps_path, f])
ext = str(f).split('.')[-1]
LOG.info(fpath)
fdest = _.storage.path('/'.join([mng_tmp.id]))
folder_zip = '/'.join([fdest, 'cover.{ext}'.format(ext=ext)])
if '.jpg' in folder_zip or '.png' in folder_zip:
# LOG.info(folder_zip)
if not path.exists(fdest):
makedirs(fdest)
shutil.copy(fpath, folder_zip)
mng_tmp.thumb = '.'.join(['cover', ext])
shutil.rmtree(temps_path)
_in = u'Success'
code, status = ResponseHTTP.OK
return resp.to_json(_in,
code=code,
status=status)
def series_page(self):
_ = self.R
slug = _.matchdict.get('series_slug', "No Title")
present = arrow.utcnow()
qry = Manga.query
manga = qry.filter(Manga.slug == slug.strip()).first()
if manga is not None:
filename = '/'.join([manga.id, manga.thumb])
thumb = _.storage.url(filename)
aka = manga.aka
artists = manga.get_artist()
authors = manga.get_authors()
description = manga.description
name = manga.title
status = manga.status
stags = manga.get_genre_tostr()
tags = [dict(label=tag, value=slugist(tag)) for tag in stags.split(',')]
time = arrow.get(manga.chapter_updated.replace(tzinfo=tz.tzlocal())).humanize(present)
origin = manga.origin
last = Manga.last_chapter(manga.id)
results = []
last_chapter = ''
last_url = ''
if last is not None:
last_chapter = ' '.join([str(last.chapter), last.title])
last_url = '/'.join([manga.slug, last.slug])
manga.updated_viewed()
chapters = Chapter.query.filter_by(tb_manga_id=manga.id).order_by(desc(Chapter.sortorder)).all()
for chapter in chapters:
results.append(dict(
name=' '.join(['Ch.', str(chapter.chapter).replace('.0', ''), chapter.title]),
url='/'.join([manga.slug, chapter.slug]),
time=arrow.get(chapter.updated.replace(tzinfo=tz.tzlocal())).humanize(present)
))
return dict(
origin=origin,
aka=aka,
thumb_url='/{thumb}'.format(thumb=thumb),
artists=artists,
authors=authors,
description=description,
name=name,
tags=tags,
status=status,
time=time,
last_chapter=last_chapter,
last_url=last_url,
chapters=results
)
return None
def series_view(self):
_ = self.R
slug = _.matchdict.get('seriesSlug', "No Title")
present = arrow.utcnow()
qry = Manga.query
manga = qry.filter(Manga.slug == slug.strip()).first()
if manga is not None:
filename = '/'.join([manga.id, manga.thumb])
thumb = _.storage.url(filename)
aka = utils.HTMLUnscaped(manga.aka)
artists = utils.HTMLUnscaped(manga.get_artist())
authors = utils.HTMLUnscaped(manga.get_authors())
description = utils.HTMLUnscaped(manga.description)
name = utils.HTMLUnscaped(manga.title)
last = Manga.last_chapter(manga.id)
time = manga.chapter_updated.strftime('%b %d, %Y')
stags = manga.get_genre_tostr()
tags = [dict(label=tag, value=slugist(tag)) for tag in stags.split(',')]
results = []
last_chapter = ''
last_url = ''
if last is not None:
last_chapter = ' '.join([str(last.chapter), last.title])
last_url = '/'.join([manga.slug, last.slug])
manga.updated_viewed()
chapters = Chapter.query.filter_by(tb_manga_id=manga.id).order_by(desc(Chapter.sortorder)).all()
for chapter in chapters:
results.append(dict(
name=' '.join(['Ch.', str(chapter.chapter).replace('.0', ''), chapter.title]),
url='/'.join([manga.slug, chapter.slug, "0"]),
time=chapter.updated.strftime('%b %d, %Y')
))
return dict(
aka=aka,
url='/manga/{slug}'.format(slug=slug),
thumb_url=thumb,
artists=artists,
authors=authors,
description=description,
name=name,
tags=tags,
time=time,
last_chapter=last_chapter,
last_url=last_url,
chapters=results
)
return None
def insert_scrap_data(objs):
"""
{
"origin": "http://bato.to/comic/_/comics/youkai-shoujo-monsuga-r11767",
"last_chapter": " Ch.39: The wind has died down...",
"name": "Youkai Shoujo - Monsuga",
"time": 1434953127.0,
"last_url": "http://bato.to/read/_/328045/youkai-shoujo-monsuga_ch39_by_japanzai",
"site": "batoto",
"thumb": "http://img.bato.to/forums/uploads/6a258c7c65ea8478d0fb28e23964462b.jpg"
}
:param objs:
:return:
"""
manga = Manga()
def chapter_view(self):
_ = self.R
slug = _.matchdict.get('seriesSlug', "No Title")
chap_slug = _.matchdict.get('chapterSlug', "No Title")
# cari manga by slug
manga = Manga.query.filter(Manga.slug == slug).first()
if manga is not None:
filename = '/'.join([manga.id, manga.thumb])
thumb = _.storage.url(filename)
aka = utils.HTMLUnscaped(manga.aka)
artists = utils.HTMLUnscaped(manga.get_artist())
authors = utils.HTMLUnscaped(manga.get_authors())
description = utils.HTMLUnscaped(manga.description)
name = utils.HTMLUnscaped(manga.title)
last = Manga.last_chapter(manga.id)
last_chapter = ' '.join([str(last.chapter), last.title])
# cari chapter manga
chapter = manga.get_chapter(manga, chap_slug)
ch = chapter.title if chapter.title == str(chapter.chapter) \
.replace('.0', '') else '{ch} {chapter}' \
.format(ch=str(chapter.chapter).replace('.0', ''), chapter=chapter.title)
return dict(
aka=aka,
url='/chapter/{slug}/{chap}'.format(slug=slug, chap=chap_slug),
thumb_url=thumb,
artists=artists,
authors=authors,
description='Read newest {title} {ch} online'.format(
ch=ch,
title=manga.title
),
name='{title} {ch}'.format(
ch=ch,
title=manga.title
),
genres=manga.get_genre_tostr(),
last_chapter=last_chapter,
series_url=manga.slug
)
return {'project': 'moori'}
def build_to_sys(site, source):
try:
lt = LocalDateTime.now()
"""
dict(
thumb=self.netlocs[3] + "/".join([image_thumb.split('/')[-2], image_thumb.split('/')[-1]]),
origin=origin_url,
name=title,
# time=self.parseDate.human_to_date_stamp(time),
time=time,
last_chapter=last_title,
last_url=last_url,
site=self.netlocs[1]
)
"""
# list latest
# scrap series info
# url = "/".join([site.netlocs[2], source.get('origin')])
url = source.get('origin')
# print(url)
respcontent = site.get_html(url)
series_info = site.series_info(respcontent)
# series == manga
qry = Manga.query
manga = qry.filter(Manga.slug == utils.slugist(
"-".join([site.netlocs[4], source.get('name', None)])
)).first()
if manga is None:
with transaction.manager:
manga = Manga(
site.netlocs[4],
series_info.get('name', []),
0,
", ".join(series_info.get('tags', [])),
", ".join(series_info.get('authors', [])),
", ".join(series_info.get('artists', [])),
', '.join(series_info.get('aka', [])),
",".join(series_info.get('description', None)),
1 if 'ongoing' in series_info.get('status', '').lower()
else 2 if 'completed' in series_info.get('status', '').lower() else 0
)
# manga.id = utils.guid()
manga.origin = source.get('origin', '')
manga.chapter_updated = lt.from_time_stamp(source.get('time', 'now'))
ext = series_info.get('thumb_url', '').lower().rsplit('.', 1)[-1]
manga.thumb = '.'.join(['cover', ext])
manga.category = 'ja'
DBSession.add(manga)
DBSession.flush()
manga = qry.filter(Manga.slug == utils.slugist(
"-".join([site.netlocs[4], source.get('name', None)])
)).first()
manga_id, manga_thumb, manga_slug = manga.id, manga.thumb, manga.slug
ini_path = path.join(
path.dirname(
path.dirname(__file__)
),
'/'.join(['rak', 'manga', manga_id])
)
r = requests.get(source.get('thumb'))
path_img = '/'.join([ini_path, manga_thumb])
print(path_img)
if not path.exists(ini_path):
makedirs(ini_path)
with open(path_img, "wb") as code:
code.write(r.content)
chapters_info = series_info.get('chapters', [])
for i, ch in enumerate(chapters_info):
print(ch.get('name', ''))
# batoto slug
slug_bt = ch.get('name', '')
if ':' in slug_bt:
slug_bt = slug_bt.split(':')
slug_bt.pop(0)
slug_bt = '-'.join(slug_bt)
slug_chapter = ' '.join([manga_slug, slug_bt])
# cek chapter sudah didownload
chapter = Chapter.query.filter(Chapter.slug == utils.slugist(slug_chapter)).first()
if chapter is None:
v = utils.parse_number(ch.get('name', ''), "Vol")
v = 0 if v is None else v
c = utils.parse_number(ch.get('name', ''), "Ch")
c = 0 if c is None else c
with transaction.manager:
chapter = Chapter(
slug_bt,
c,
v
)
time = lt.human_to_date(ch.get('time', 'now'))
# chapter.id = utils.guid()
ch_manga = Manga.query.get(manga_id)
ch_manga.chapter_count += 1
chapter.lang = ISOLang.query.filter(ISOLang.iso == 'en').first()
chapter.updated = time
chapter.manga = ch_manga
# s = 1000v + c
# chapter.sortorder = (1000*float(v)) + float(c)
chapter.sortorder = float(c)
chapter.slug = slug_chapter
DBSession.add(chapter)
DBSession.flush()
chapter = Chapter.query.filter(Chapter.slug == utils.slugist(slug_chapter)).first()
# batoto
html = site.get_html(ch.get('url'))
# # ambil image dan download locally di folder chapter.id
chapter_info = site.chapter_info(html)
try:
# series info
# chapter info and images
session = FuturesSession(executor=ThreadPoolExecutor(max_workers=10))
for n, page in enumerate(chapter_info.get('pages', [])):
ini_chapter = '/'.join([ini_path, chapter.id])
print(page)
r = session.get(page).result()
if r.status_code != 200:
raise HtmlError('cannot fetch')
ext = page.split('/')[-1].rsplit('.', 1)[-1]
path_img = '/'.join([ini_chapter, "{num:03d}.{ext}".format(num=n, ext=ext)])
# path_img = '/'.join([ini_chapter, page.split('/')[-1]])
print(path_img)
if not path.exists(ini_chapter):
makedirs(ini_chapter)
with open(path_img, "wb") as code:
code.write(r.content)
except ConnectionError as Conn:
print(Conn)
chapter = Chapter.query.get(chapter.id)
DBSession.delete(chapter)
shutil.rmtree(ini_chapter)
except AttributeError as e:
print(e.message)
except KeyError as e:
print(e.message)
except ValueError as e:
print(e.message)
def script_to_sys(self, source_url, source_origin, time_str):
"""
untuk url API mangaeden + id manga[:24]
python scripts/mangascrapper.py -s https://www.mangaeden.com/api/manga/4e70ea1dc092255ef7004d5c/ -o http://www.mangaeden.com/en/en-manga/fairy-tail/ -t "Aug 31, 2015"
:param self: Manga API
:param source_url: url data manga untuk di scrap
:param source_origin: url sumber manga
:param time: chapter release terakhir Agust 30, 2015(string time)
: fairy tail https://www.mangaeden.com/api/manga/4e70ea1dc092255ef7004d5c/
: naruto http://www.mangaeden.com/api/manga/4e70ea03c092255ef70046f0/
: one piece http://www.mangaeden.com/api/manga/4e70ea10c092255ef7004aa2/
: bleach http://www.mangaeden.com/api/manga/4e70e9efc092255ef7004274/
: nanatsu http://www.mangaeden.com/api/manga/5099a865c092254a2000daf4/
:return:
"""
try:
# print(url)
# "{url}/api/manga/{id}/".format(url=self.netlocs[2], id=origin_url[:24])
# https://www.mangaeden.com/api/manga/:id[:24]/
resp_content = self.get_html(source_url)
series_info = self.series_info(resp_content)
time_long = self._parse_update_date(time_str) if isinstance(
time_str, basestring) else long(time_str)
# series == manga
qry = Manga.query
manga = qry.filter(Manga.slug == utils.slugist("-".join(
[self.netlocs[4],
series_info.get('name', None)]))).first()
if manga is None:
with transaction.manager:
manga = Manga(
self.netlocs[4],
utils.HTMLUnscaped(series_info.get('name', u'')), 0,
", ".join(series_info.get('tags', [])),
series_info.get('authors', u''),
series_info.get('artists', u''),
utils.HTMLUnscaped(series_info.get('aka', u'')),
utils.HTMLUnscaped(series_info.get('description',
u'')),
1 if 'ongoing' in series_info.get(
'status', '').lower() else 2 if 'completed'
in series_info.get('status', '').lower() else 0)
# manga.id = utils.guid()
manga.origin = source_origin
manga.chapter_updated = datetime.fromtimestamp(time_long)
ext = series_info.get('thumb_url',
'').lower().split('.')[-1]
manga.thumb = '.'.join(['cover', ext])
manga.category = 'ja'
DBSession.add(manga)
DBSession.flush()
manga = qry.filter(Manga.slug == utils.slugist("-".join(
[self.netlocs[4],
series_info.get('name', None)]))).first()
manga_id, manga_thumb, manga_slug = manga.id, manga.thumb, manga.slug
ini_path = path.join(path.dirname(path.dirname(__file__)),
'/'.join(['rak', 'manga', manga_id]))
r = requests.get(series_info.get('thumb_url', ''))
path_img = '/'.join([ini_path, manga_thumb])
print(path_img)
if not path.exists(ini_path):
makedirs(ini_path)
with open(path_img, "wb") as code:
code.write(r.content)
chapters_info = series_info.get('chapters', [])
for i, ch in enumerate(chapters_info):
print(ch.get('name', ''))
ch_name = str(ch.get(
'order',
0)) if ch.get('name', '') is None else utils.HTMLUnscaped(
ch.get('name', u''))
# edenapi slug
slug_bt = ch_name
# if ':' in slug_bt:
# slug_bt = slug_bt.split(':')
# slug_bt.pop(0)
# slug_bt = '-'.join(slug_bt)
slug_chapter = ' '.join([manga_slug, slug_bt])
# cek chapter sudah didownload
chapter = Chapter.query.filter(
Chapter.slug == utils.slugist(slug_chapter)).first()
if chapter is None:
v = utils.parse_number(ch_name, "Vol")
v = 0 if v is None else v
c = ch.get('order', 0)
with transaction.manager:
chapter = Chapter(slug_bt, c, v)
time = datetime.fromtimestamp(
ch.get('time', datetime.now()))
# chapter.id = utils.guid()
ch_manga = Manga.query.get(manga_id)
ch_manga.chapter_count += 1
chapter.lang = ISOLang.query.filter(
ISOLang.iso == 'en').first()
chapter.updated = time
chapter.manga = ch_manga
# s = 1000v + c
# chapter.sortorder = (1000*float(v)) + float(c)
chapter.sortorder = float(c)
chapter.slug = slug_chapter
DBSession.add(chapter)
DBSession.flush()
chapter = Chapter.query.filter(
Chapter.slug == utils.slugist(slug_chapter)).first()
# eden
headers = {
'content-type': 'application/json; charset=utf8'
}
html = self.get_html(ch.get('url'), headers=headers)
# # ambil image dan download locally di folder chapter.id
chapter_info = self.chapter_info(html)
try:
# series info
# chapter info and images
session = FuturesSession(executor=ThreadPoolExecutor(
max_workers=10))
for n, page in enumerate(chapter_info.get('pages',
[])):
ini_chapter = '/'.join([ini_path, chapter.id])
print(page)
r = session.get(page).result()
if r.status_code != 200:
print('continue chapter')
continue
# raise HtmlError('cannot fetch')
# path_img = '/'.join([ini_chapter, page.split('/')[-1]])
ext = page.split('/')[-1].rsplit('.', 1)[-1]
path_img = '/'.join([
ini_chapter, "{num:03d}.{ext}".format(num=n,
ext=ext)
])
print(path_img)
if not path.exists(ini_chapter):
makedirs(ini_chapter)
with open(path_img, "wb") as code:
code.write(r.content)
except ConnectionError as Conn:
print(Conn)
chapter = Chapter.query.get(chapter.id)
DBSession.delete(chapter)
shutil.rmtree(ini_chapter)
except AttributeError as e:
print(e.message)
except KeyError as e:
print(e.message)
except ValueError as e:
print(e.message)
def build_to_sys(self, site, source):
try:
url = source.get('last_url')
# print(url)
resp_content = site.get_html(url)
series_info = site.series_info(resp_content)
# series == manga
qry = Manga.query
manga = qry.filter(Manga.slug == utils.slugist("-".join(
[site.netlocs[4],
series_info.get('name', None)]))).first()
if manga is None:
with transaction.manager:
manga = Manga(
site.netlocs[4],
utils.HTMLUnscaped(series_info.get('name', u'')), 0,
", ".join(series_info.get('tags', [])),
series_info.get('authors', u''),
series_info.get('artists', u''),
utils.HTMLUnscaped(series_info.get('aka', u'')),
utils.HTMLUnscaped(series_info.get('description',
u'')),
1 if 'ongoing' in series_info.get(
'status', '').lower() else 2 if 'completed'
in series_info.get('status', '').lower() else 0)
# manga.id = utils.guid()
manga.origin = source.get('origin', '')
manga.chapter_updated = datetime.fromtimestamp(
source.get('time', 'now'))
ext = series_info.get('thumb_url',
'').lower().rsplit('.', 1)[-1]
manga.thumb = '.'.join(['cover', ext])
manga.category = 'ja'
DBSession.add(manga)
DBSession.flush()
manga = qry.filter(Manga.slug == utils.slugist("-".join(
[site.netlocs[4],
series_info.get('name', None)]))).first()
manga_id, manga_thumb, manga_slug = manga.id, manga.thumb, manga.slug
ini_path = path.join(path.dirname(path.dirname(__file__)),
'/'.join(['rak', 'manga', manga_id]))
r = requests.get(series_info.get('thumb_url', ''))
path_img = '/'.join([ini_path, manga_thumb])
print(path_img)
if not path.exists(ini_path):
makedirs(ini_path)
with open(path_img, "wb") as code:
code.write(r.content)
chapters_info = series_info.get('chapters', [])
for i, ch in enumerate(chapters_info):
print(ch.get('name', ''))
ch_name = str(ch.get(
'order',
0)) if ch.get('name', '') is None else utils.HTMLUnscaped(
ch.get('name', u''))
# edenapi slug
slug_bt = ch_name
# if ':' in slug_bt:
# slug_bt = slug_bt.split(':')
# slug_bt.pop(0)
# slug_bt = '-'.join(slug_bt)
slug_chapter = ' '.join([manga_slug, slug_bt])
# cek chapter sudah didownload
chapter = Chapter.query.filter(
Chapter.slug == utils.slugist(slug_chapter)).first()
if chapter is None:
v = utils.parse_number(ch_name, "Vol")
v = 0 if v is None else v
c = ch.get('order', 0)
with transaction.manager:
chapter = Chapter(slug_bt, c, v)
time = datetime.fromtimestamp(
ch.get('time', datetime.now()))
# chapter.id = utils.guid()
ch_manga = Manga.query.get(manga_id)
ch_manga.chapter_count += 1
chapter.lang = ISOLang.query.filter(
ISOLang.iso == 'en').first()
chapter.updated = time
chapter.manga = ch_manga
# s = 1000v + c
# chapter.sortorder = (1000*float(v)) + float(c)
chapter.sortorder = float(c)
chapter.slug = slug_chapter
DBSession.add(chapter)
DBSession.flush()
chapter = Chapter.query.filter(
Chapter.slug == utils.slugist(slug_chapter)).first()
# eden
headers = {
'content-type': 'application/json; charset=utf8'
}
html = site.get_html(ch.get('url'), headers=headers)
# # ambil image dan download locally di folder chapter.id
chapter_info = site.chapter_info(html)
try:
# series info
# chapter info and images
session = FuturesSession(executor=ThreadPoolExecutor(
max_workers=10))
for n, page in enumerate(chapter_info.get('pages',
[])):
ini_chapter = '/'.join([ini_path, chapter.id])
print(page)
r = session.get(page).result()
if r.status_code != 200:
# raise HtmlError('cannot fetch')
continue
# path_img = '/'.join([ini_chapter, page.split('/')[-1]])
ext = page.split('/')[-1].rsplit('.', 1)[-1]
path_img = '/'.join([
ini_chapter, "{num:03d}.{ext}".format(num=n,
ext=ext)
])
print(path_img)
if not path.exists(ini_chapter):
makedirs(ini_chapter)
with open(path_img, "wb") as code:
code.write(r.content)
except ConnectionError as Conn:
print(Conn)
chapter = Chapter.query.get(chapter.id)
DBSession.delete(chapter)
shutil.rmtree(ini_chapter)
except AttributeError as e:
print(e.message)
except KeyError as e:
print(e.message)
except ValueError as e:
print(e.message)
def series_page(self):
_ = self.R
slug = _.matchdict.get('series_slug', "No Title")
present = arrow.utcnow()
qry = Manga.query
manga = qry.filter(Manga.slug == slug.strip()).first()
if manga is not None:
filename = '/'.join([manga.id, manga.thumb])
thumb = _.storage.url(filename)
aka = manga.aka
artists = manga.get_artist()
authors = manga.get_authors()
description = manga.description
name = manga.title
status = manga.status
stags = manga.get_genre_tostr()
tags = [
dict(label=tag, value=slugist(tag)) for tag in stags.split(',')
]
time = arrow.get(manga.chapter_updated.replace(
tzinfo=tz.tzlocal())).humanize(present)
origin = manga.origin
last = Manga.last_chapter(manga.id)
results = []
last_chapter = ''
last_url = ''
if last is not None:
last_chapter = ' '.join([str(last.chapter), last.title])
last_url = '/'.join([manga.slug, last.slug])
manga.updated_viewed()
chapters = Chapter.query.filter_by(
tb_manga_id=manga.id).order_by(desc(
Chapter.sortorder)).all()
for chapter in chapters:
results.append(
dict(name=' '.join([
'Ch.',
str(chapter.chapter).replace('.0', ''),
chapter.title
]),
url='/'.join([manga.slug, chapter.slug]),
time=arrow.get(
chapter.updated.replace(
tzinfo=tz.tzlocal())).humanize(present)))
return dict(origin=origin,
aka=aka,
url='/manga/{slug}'.format(slug=slug),
thumb_url='/{thumb}'.format(thumb=thumb),
artists=artists,
authors=authors,
description=description,
name=name,
tags=tags,
status=status,
time=time,
last_chapter=last_chapter,
last_url=last_url,
chapters=results)
return None
def test_init_manga(self):
manga = Manga(u'Skip Beat!', 2006)
manga.alt = u'Skip Beat!'
manga.description = u'It is the story of Kyoko Mogami (Mogami Kyoko) a 16-year-old girl \
who discovers that her childhood friend and romantic goal, Sho Fuwa, only keeps her around to act\
as a maid and earn his living expenses, as he works his way to become the top pop idol in Japan...'
DBSession.db.add(manga)
manga.genre.append(self.find_genre(u'Comedy'))
manga.genre.append(self.find_genre(u'Drama'))
manga.genre.append(self.find_genre(u'Romance'))
manga.genre.append(self.find_genre(u'Shoujo'))
manga.author.append(self.find_author(u'Nakamura Yoshiki'))
# request.db.add(manga)
transaction.commit()
manga = Manga(u'Fairy tail', 2006)
manga.alt = u'Ekor Peri; Fairy Tail;'
manga.description = u'A young celestial mage, Lucy Heartfilia,\
runs away from home and travels to the land of Fiore to join\
the magical Fairy Tail Guild. Along the way,\
she meets Natsu Dragneel, a teenage boy looking for his foster parent,\
a dragon named Igneel, with his best friend, Happy the cat. Shortly after their meeting,\
Lucy is abducted by Bora of Prominence, who was posing as Salamander of Fairy Tail, to be sold as a slave.\
Natsu rescues her and reveals that he is the real Salamander of fairy tail and has the skills of a Dragon Slayer,\
an ancient fire magic. He offers her membership into the guild, which she gladly accepts, they along with\
Erza Scarlet -Armored maiden, Gray Fullbuster -Ice Maker, and Happy become a team performing various\
missions offered to the Fairy Tail guild. Fairy Tail won the 2009 Kodansha Manga Award for shounen manga.\
It has also won The Society for the Promotion of Japanese Animation \'s Industry Awards for best comedy manga.'
DBSession.add(manga)
manga.genre.append(self.find_genre(u'Comedy'))
manga.genre.append(self.find_genre(u'Drama'))
manga.genre.append(self.find_genre(u'Action'))
manga.genre.append(self.find_genre(u'Shounen'))
manga.genre.append(self.find_genre(u'Fantasy'))
manga.genre.append(self.find_genre(u'Adventure'))
manga.genre.append(self.find_genre(u'Ecchi'))
manga.author.append(self.find_author(u'Mashima Hiro'))
# request.db.add(manga)
transaction.commit()
manga = Manga(u'Nanatsu no taizai', 2012)
manga.alt = u'Nanatsu no Daizai; The Seven Deadly Sins;'
manga.description = u'The "Seven Deadly Sins", a group of evil knights who\
conspired to overthrow the kingdom of Britannia, were said to have been eradicated by the Holy Knights,\
although some claim that they still live. Ten years later, the Holy Knights have staged a Coup d\'etat and\
assassinated the king, becoming the new, tyrannical rulers of the kingdom. Elizabeth, daughter of the king,\
sets out on a journey to find the "Seven Deadly Sins", and to enlist their help in taking back the kingdom.'
DBSession.add(manga)
manga.genre.append(self.find_genre(u'Comedy'))
manga.genre.append(self.find_genre(u'Drama'))
manga.genre.append(self.find_genre(u'Action'))
manga.genre.append(self.find_genre(u'Shounen'))
manga.genre.append(self.find_genre(u'Fantasy'))
manga.genre.append(self.find_genre(u'Adventure'))
manga.genre.append(self.find_genre(u'Ecchi'))
manga.author.append(self.find_author(u'Suzuki Nakaba'))
# request.db.add(manga)
transaction.commit()
def build_to_sys(site, source):
try:
lt = LocalDateTime.now()
"""
dict(
thumb=self.netlocs[3] + "/".join([image_thumb.split('/')[-2], image_thumb.split('/')[-1]]),
origin=origin_url,
name=title,
# time=self.parseDate.human_to_date_stamp(time),
time=time,
last_chapter=last_title,
last_url=last_url,
site=self.netlocs[1]
)
"""
# list latest
# scrap series info
# url = "/".join([site.netlocs[2], source.get('origin')])
url = source.get('origin')
# print(url)
respcontent = site.get_html(url)
series_info = site.series_info(respcontent)
# series == manga
qry = Manga.query
manga = qry.filter(Manga.slug == utils.slugist(
"-".join([site.netlocs[4], source.get('name', None)])
)).first()
if manga is None:
with transaction.manager:
manga = Manga(
site.netlocs[4],
series_info.get('name', []),
0,
", ".join(series_info.get('tags', [])),
", ".join(series_info.get('authors', [])),
", ".join(series_info.get('artists', [])),
', '.join(series_info.get('aka', [])),
",".join(series_info.get('description', None)),
1 if 'ongoing' in series_info.get('status', '').lower()
else 2 if 'completed' in series_info.get('status', '').lower() else 0
)
# manga.id = utils.guid()
manga.origin = source.get('origin', '')
manga.chapter_updated = lt.from_time_stamp(source.get('time', 'now'))
ext = series_info.get('thumb_url', '').lower().rsplit('.', 1)[-1]
manga.thumb = '.'.join(['cover', ext])
manga.category = 'ja'
DBSession.add(manga)
DBSession.flush()
manga = qry.filter(Manga.slug == utils.slugist(
"-".join([site.netlocs[4], source.get('name', None)])
)).first()
manga_id, manga_thumb, manga_slug = manga.id, manga.thumb, manga.slug
ini_path = path.join(
path.dirname(
path.dirname(__file__)
),
'/'.join(['rak', 'manga', manga_id])
)
r = requests.get(source.get('thumb'))
path_img = '/'.join([ini_path, manga_thumb])
print(path_img)
if not path.exists(ini_path):
makedirs(ini_path)
with open(path_img, "wb") as code:
code.write(r.content)
chapters_info = series_info.get('chapters', [])
for i, ch in enumerate(chapters_info[0:2]):
print(ch.get('name', ''))
# batoto slug
slug_bt = ch.get('name', '')
if ':' in slug_bt:
slug_bt = slug_bt.split(':')
slug_bt.pop(0)
slug_bt = '-'.join(slug_bt)
slug_chapter = ' '.join([manga_slug, slug_bt])
# cek chapter sudah didownload
chapter = Chapter.query.filter(Chapter.slug == utils.slugist(slug_chapter)).first()
if chapter is None:
v = utils.parse_number(ch.get('name', ''), "Vol")
v = 0 if v is None else v
c = utils.parse_number(ch.get('name', ''), "Ch")
c = 0 if c is None else c
with transaction.manager:
chapter = Chapter(
slug_bt,
c,
v
)
time = lt.human_to_date(ch.get('time', 'now'))
# chapter.id = utils.guid()
ch_manga = Manga.query.get(manga_id)
ch_manga.chapter_count += 1
chapter.lang = ISOLang.query.filter(ISOLang.iso == 'en').first()
chapter.updated = time
chapter.manga = ch_manga
# s = 1000v + c
# chapter.sortorder = (1000*float(v)) + float(c)
chapter.sortorder = float(c)
chapter.slug = slug_chapter
DBSession.add(chapter)
DBSession.flush()
chapter = Chapter.query.filter(Chapter.slug == utils.slugist(slug_chapter)).first()
# batoto
html = site.get_html(ch.get('url'))
# # ambil image dan download locally di folder chapter.id
chapter_info = site.chapter_info(html)
try:
# series info
# chapter info and images
session = FuturesSession(executor=ThreadPoolExecutor(max_workers=10))
for page in chapter_info.get('pages', []):
ini_chapter = '/'.join([ini_path, chapter.id])
print(page)
r = session.get(page).result()
if r.status_code != 200:
raise HtmlError('cannot fetch')
path_img = '/'.join([ini_chapter, page.split('/')[-1]])
print(path_img)
if not path.exists(ini_chapter):
makedirs(ini_chapter)
with open(path_img, "wb") as code:
code.write(r.content)
except ConnectionError as Conn:
print(Conn)
chapter = Chapter.query.get(chapter.id)
DBSession.delete(chapter)
shutil.rmtree(ini_chapter)
except AttributeError as e:
print(e.message)
except KeyError as e:
print(e.message)
except ValueError as e:
print(e.message)
