def get_series_from_backlog(language):
"""Find a file in the database by his slug
:param language: Ide of the language in the database
"""
"""Find in database"""
_session = app.apps.get("db_sqlalchemy")()
_items = _session.query(Serie).\
filter(Serie.language == language, Serie.status == constants.STATUS['airing']).\
order_by(asc(Serie.series_id_reference)).\
all()
_session.close()
"""Check if the file exists"""
if not _items:
return error_response(msg="Items not found.")
"""Transform data"""
_items_response = [{
**_item.to_dict(), 'series_id':
_item.series_id_reference
} for _item in _items]
_data_response = {
u"animes": _items_response,
}
return success_response(data=_data_response, msg="Items found.")
def get_episodes_from_website(series_id, lang, headers):
"""Prepare the payload"""
_payload = {
u"lang": lang,
u"series_id": series_id,
}
"""Define url"""
_url = "{0}/{1}/episodes".format(URL_ANIME, series_id)
"""Get all novels from website"""
logging.warning('*****************************************')
logging.warning('_url')
logging.warning(_url)
logging.warning('*****************************************')
logging.warning('_payload')
logging.warning(_payload)
_episodes = requests.get(_url, params=_payload, headers=headers)
logging.warning('*****************************************')
logging.warning('_episodes.text')
logging.warning(_episodes)
"""Check if the response is valid"""
if _episodes.status_code != 200:
"""Return error if the response is invalid"""
return error_response(msg=_episodes.text)
"""Get json response"""
_episodes_json = _episodes.json()
logging.warning('*****************************************')
logging.warning('_episodes_json')
logging.warning(_episodes_json)
"""Return novels"""
return success_response(_episodes_json.get('data'))
def get_episodes_by_serie_db(serie):
"""Find episodes by serie id
:param serie: Identificator for serie
"""
"""Find in database"""
_session = app.apps.get("db_sqlalchemy")()
_episodes = _session.query(
Media.media,
Media.cover,
Media.created_at,
Media.number,
Media.title,
).\
filter(
Media.serie == serie,
Media.is_deleted == False,
Media.is_active == True,
Media.status == constants.STATUS['uploaded'],
).\
order_by(desc(Media.media_id_reference)).\
all()
_session.close()
"""Check if the file exists"""
if not _episodes:
return error_response(msg="Episodes not found.")
"""Transform data"""
_episodes_json = [{
u"episode": _episode[0],
u"cover": _episode[1],
u"created_at": _episode[2].strftime("%c"),
u"number": _episode[3],
u"title": _episode[4],
} for _episode in _episodes]
return success_response(data=_episodes_json, msg="Episodes found.")
def publish_items(
items,
language,
credential,
headers,
):
"""Get chapters from all novels"""
_media_items = get_items_by_slug(
items,
headers=headers,
language=language,
)
"""Check if it hasn't novels, response to client"""
if not _media_items:
return error_response(msg="New items not found.")
"""Publish or update on website"""
_published = publish_items_db(
_media_items,
language,
headers=headers,
# watermark_code=watermark_code,
)
"""Transform data response"""
_data_response = {"anime": _published}
return success_response(_data_response)
def get_items_from_backlog(limit):
"""Find a file in the database by his slug
"""
"""Find in database"""
_session = app.apps.get("db_sqlalchemy")()
_items = _session.query(Media, Serie).\
join(Serie, Media.serie == Serie.serie, isouter=True).\
filter(Media.status == constants.STATUS['initial']).\
order_by(asc(Media.media)).\
limit(limit).\
all()
_session.close()
"""Check if the file exists"""
if not _items:
return error_response(msg="Items not found.")
"""Transform data"""
_items_response = [{
**_item.Serie.to_dict(),
**_item.Media.to_dict(),
} for _item in _items]
_data_response = {
u"items": _items_response,
}
return success_response(data=_data_response, msg="Items found.")
def publish_episodes_mirrors(req: Request, res: Response, next: Next):
"""Publish hentai"""
_result = anime.publish_episodes_mirrors(data=req.all_params(), )
"""Check if exist an error"""
if _result['valid'] is False:
return res.bad_request(_result)
"""Response the data to client"""
res.ok(success_response(data=_result['data'], msg="Items added."))
def get_episode_by_id(req: Request, res: Response, next: Next):
"""Get episode from db"""
_requests_db = media.get_episode_by_id_db(req.param('media'))
"""Check if exist an error"""
if _requests_db['valid'] is False:
return res.bad_request(_requests_db)
"""Transform data response"""
_data_response = {**_requests_db['data']}
"""Response the data to client"""
res.ok(success_response(data=_data_response, msg=_requests_db['msg']))
def update_status_media(items, body={}):
"""Create session"""
_session = app.apps.get("db_sqlalchemy")()
_items = [{**_item, **body} for _item in items]
"""For each novel deactive without results"""
_session.bulk_update_mappings(Media, _items)
"""Session commit"""
_session.commit()
"""Close session"""
_session.close()
"""Response"""
return success_response(msg="Items updated.")
def publish_episodes_mirrors(data, ):
"""Upload status"""
logging.warning('*****************************************')
logging.warning("--- Change status to uploaded ---")
media.update_status_media(
[{
u'media': data['media']
}], {
'status': constants.STATUS['uploaded'],
'cloud': data['success'][-1]['code']
})
return success_response(data)
def get_episode_by_id_db(media):
"""Find media by id
:param media: Identificator for media
"""
"""Find in database"""
_session = app.apps.get("db_sqlalchemy")()
_episode = _session.query(Media).\
filter(Media.media == media, Media.is_deleted == False, Media.is_active == True).\
first()
"""Check if the file exists"""
if not _episode:
_session.close()
return error_response(msg="Media not found.")
_servers = _session.query(Server).\
filter(Server.media == _episode.media, Server.is_deleted == False, Server.is_active == True, Server.status == constants.STATUS['uploaded']).\
all()
"""Transform data"""
_episode_json = _episode.to_dict()
_episode_json['servers'] = [_server.to_dict() for _server in _servers]
"""Pagination"""
_episode_next = _session.query(Media).\
filter(
Media.media > media,
Media.serie == _episode_json['serie'],
Media.is_deleted == False,
Media.is_active == True
).\
first()
if _episode_next:
_episode_json['next'] = _episode_next.to_dict()
else:
_episode_json['next'] = None
_episode_prev = _session.query(Media).\
filter(
Media.media < media,
Media.serie == _episode_json['serie'],
Media.is_deleted == False,
Media.is_active == True
).\
order_by(desc(Media.media)).\
first()
if _episode_prev:
_episode_json['prev'] = _episode_prev.to_dict()
else:
_episode_json['prev'] = None
"""Close session"""
_session.close()
"""Response data"""
return success_response(data=_episode_json, msg="Media found.")
def get_language_code_db(language):
"""Create session"""
_session = app.apps.get("db_sqlalchemy")()
"""Find in database"""
_language = _session.query(Language).\
filter(Language.language == language).\
first()
if not _language:
return error_response(msg="Language not found")
"""Requests to json"""
_language_json = _language.to_dict()
"""Close session"""
_session.close()
"""Response"""
return success_response(data=_language_json, msg="Language found.")
def publish_items_anime(
items,
language,
limit_publish,
headers,
):
_new_animes = []
_new_episodes_all = []
for _item in items:
if len(_new_episodes_all) >= limit_publish:
break
_new_episodes = []
"""Validate if item exists"""
_old_item = get_anime_by_anime_id_db(_item['series_id'],
language['language'])
"""Validate if it exists"""
if not _old_item['valid']:
continue
"""Validate if it has new episodes"""
_old_media = get_media_by_anime_id_db(_old_item['data']['serie'],
language['language'])
"""Get episodes"""
_episodes = get_episodes_from_website(
_item['series_id'],
language['hreflang'],
headers,
)
"""Get new episodes"""
for _episode in _episodes['data']['episodes']:
_old_episode = None
for _episode_db in _old_media['data']['items']:
if _episode_db['media_id_reference'] == _episode['media_id']:
_old_episode = _episode_db
break
if not _old_episode:
_new_episodes.append(_episode)
_new_episodes_all.append(_episode)
if len(_new_episodes_all) >= limit_publish:
break
"""Valid it has new episodes"""
if not _new_episodes:
continue
"""Define anime and new episodes"""
_new_anime = {**_item, **_old_item['data'], u'episodes': _new_episodes}
"""Add anime"""
_new_animes.append(_new_anime)
_data_response = {u'items': _new_animes}
return success_response(_data_response)
def get_items_from_db(lang, limit, headers):
"""Prepare the payload"""
_payload = {
u"lang": lang,
u"limit": limit,
}
"""Get all novels from website"""
_anime = requests.get(URL_ANIME, params=_payload, headers=headers)
"""Check if the response is valid"""
if _anime.status_code != 200:
"""Return error if the response is invalid"""
return error_response(msg=_anime.text)
"""Get json response"""
_anime_json = _anime.json()
"""Return novels"""
return success_response(_anime_json.get('data'))
def publish_latest(req: Request, res: Response, next: Next):
"""Validate obligate params"""
_validate = validate_obligate_fields({
u'username':
req.headers.get('username'),
u'password':
req.headers.get('password'),
})
# """Check if has errors return a error response"""
if _validate["valid"] is False:
return res.bad_request(
error_response("{} is necesary.".format(_validate["error"])))
_headers = {
u'username': req.headers.get('username'),
u'password': req.headers.get('password'),
}
_description_upload = req.headers.get('description_upload',
ITEM_DESCRIPTION_UPLOAD)
_storage_credential = req.headers.get('credential',
STORAGE_CREDENTIALS_DEFAULT)
_items = anime.get_items_from_backlog(
# language=_lang['data']['language'],
limit=req.param('limit', WEBSITE_LIMIT_LATEST, callback=int))
if _items['valid'] is False:
return res.bad_request(_items)
"""Publish hentai"""
_result = anime.upload_items(
_items['data']['items'],
headers=_headers,
description_upload=_description_upload,
credential=_storage_credential,
resolution=req.param('resolution',
MEDIA_RESOLUTION_DEFAULT,
callback=int),
)
"""Check if exist an error"""
if _result['valid'] is False:
return res.bad_request(_result)
"""Transform data response"""
_data_response = {"items": _result['data']}
"""Response the data to client"""
res.ok(success_response(data=_data_response, msg="Items uploaded."))
def get_episode_by_media_id_db(media_id, serie):
"""Find a file in the database by his slug
:param media_id_reference: Media Id of item in the database
:param serie: Serie of reference
"""
"""Find in database"""
_session = app.apps.get("db_sqlalchemy")()
_item = _session.query(Media).\
filter(Media.media_id_reference == media_id, Media.serie == serie).\
first()
_session.close()
"""Check if the file exists"""
if not _item:
return error_response(msg="Item not found.")
"""Transform data"""
_data_response = {
**_item.to_dict(),
}
return success_response(data=_data_response, msg="Media found.")
def get_anime_by_anime_id_db(series_id_reference, language):
"""Find a file in the database by his slug
:param series_id_reference: Media Id of item in the database
:param language: Id of the language in the database
"""
"""Find in database"""
_session = app.apps.get("db_sqlalchemy")()
_item = _session.query(Serie).\
filter(Serie.series_id_reference == series_id_reference, Serie.language == language).\
first()
_session.close()
"""Check if the file exists"""
if not _item:
return error_response(msg="Serie not found.")
"""Transform data"""
_data_response = {
**_item.to_dict(),
}
return success_response(data=_data_response, msg="Serie found.")
def get_media_by_anime_id_db(serie, language):
"""Find a file in the database by his slug
:param series_id_reference: Media Id of item in the database
:param language: Id of the language in the database
"""
"""Find in database"""
_session = app.apps.get("db_sqlalchemy")()
_items = _session.query(Media).\
filter(Media.serie == serie).\
all()
_session.close()
"""Check if the file exists"""
"""Transform data"""
_items_response = [{
**_item.to_dict(),
} for _item in _items]
"""Transform data"""
_data_response = {'items': _items_response}
return success_response(data=_data_response, msg="Items found.")
def publish_latest_episodes(req: Request, res: Response, next: Next):
"""Validate obligate params"""
_validate = validate_obligate_fields({
u'username':
req.headers.get('username'),
u'password':
req.headers.get('password'),
})
# """Check if has errors return a error response"""
if _validate["valid"] is False:
return res.bad_request(
error_response("{} is necesary.".format(_validate["error"])))
_headers = {
u'username': req.headers.get('username'),
u'password': req.headers.get('password'),
}
_storage_credential = req.headers.get('credential',
STORAGE_CREDENTIALS_DEFAULT)
"""Get the langauge"""
_lang = languages.get_language_slug_db(
req.param('lang', WEBSITE_LANG_LATEST))
"""Check if exist an error"""
if _lang['valid'] is False:
return res.bad_request(_lang)
_upgrade = req.param('upgrade')
if not _upgrade:
_items = anime.get_items_from_website(lang=_lang['data']['hreflang'],
limit=req.param(
'limit',
WEBSITE_LIMIT_LATEST,
callback=int),
headers=_headers)
else:
"""Search in db"""
_items = anime.get_series_from_backlog(
language=_lang['data']['language'])
if _items['valid'] is False:
return res.bad_request(_items)
"""Publish hentai"""
_result = anime.publish_items_anime(
_items['data']['animes'],
_lang.get('data'),
limit_publish=req.param('limit_publish',
WEBSITE_LIMIT_PUBLISH,
callback=int),
headers=_headers,
)
"""Check if exist an error"""
if _result['valid'] is False:
return res.bad_request(_result)
"""Publish hentai"""
_result_publish = anime.publish_items(
_result['data']['items'],
_lang.get('data'),
credential=_storage_credential,
headers=_headers,
)
"""Check if exist an error"""
if _result_publish['valid'] is False:
return res.bad_request(_result_publish)
"""Response the data to client"""
res.ok(success_response(data=_result_publish['data'], msg="Items added."))
def upload_items(items, headers, description_upload, credential, resolution):
"""For each item to do the following"""
_published_items = []
for _item in items:
logging.warning('*****************************************')
logging.warning('Get episode')
_url = "{0}/{1}/episodes/{2}/download".format(
URL_ANIME, _item['series_id_reference'],
_item['media_id_reference'])
logging.warning('*****************************************')
logging.warning('get_download_by_params()')
"""Get the langauge"""
_lang = languages.get_language_code_db(_item['language'])
"""Check if exist an error"""
if _lang['valid'] is False:
break
_download_mirrors = media.get_download_by_params(
headers=headers, url=_url, lang=_lang['data']['hreflang'])
"""Check if it hasn't novels, response to client"""
logging.warning('*****************************************')
logging.warning('_download_mirrors')
if not _download_mirrors:
"""Upload status"""
logging.warning('*****************************************')
logging.warning("--- Change status to error ---")
media.update_status_media([{
u'media': _item['media']
}], {'status': constants.STATUS['error']})
continue
else:
logging.warning('*****************************************')
logging.warning("--- Change status to uploading ---")
"""Upload status"""
media.update_status_media([{
u'media': _item['media']
}], {'status': constants.STATUS['uploading']})
_available_resolutions = []
for _playlist in _download_mirrors['playlists']:
_available_resolutions.append(
_playlist['stream_info']['resolution']['height'])
_available_resolutions = list(dict.fromkeys(_available_resolutions))
_available_resolutions.sort()
if not resolution in _available_resolutions:
resolution = _available_resolutions[-1]
_output = '{0} {1}-{2}-{3}p.mp4'.format(_lang['data']['prefix'],
_item['slug'], _item['number'],
resolution)
logging.warning('*****************************************')
logging.warning(_output)
_downloaded = None
_parent = "anime{0}{1}".format(_lang['data']['hreflang'],
_item['series_id_reference'])
logging.warning('*****************************************')
logging.warning("---Downloading episode as {0} ---".format(_output))
for _playlist in _download_mirrors['playlists']:
if (_playlist['stream_info']['resolution']['height'] == resolution
):
logging.warning('*****************************************')
logging.warning('Downloading stream from: {0}'.format(
_playlist['uri']))
"""Change the credential"""
_credential = _lang['data']['credential']
"""Set description"""
description_upload += _lang['data']['prefix']
"""Download files"""
_downloaded = media.download_episode_proxi(
_playlist['uri'],
_output,
_credential,
description_upload,
parent=_parent,
data={'media': _item['media']})
# _downloaded = media.download_episode(
# _playlist['uri'], _output, credential, description_upload, parent=_parent)
break
logging.warning('*****************************************')
logging.warning(_downloaded)
# if not _downloaded:
# continue
"""Get URL"""
# _remote_upload = servers.remote_upload_episode(
# _downloaded['success'][0])
# _remote_upload = servers.remote_upload_episode(
# 'a06959f895a811eb86700242ac120002')
# print("--- Save serve into db ---")
"""Publish or update on website"""
# _published = servers.publish_servers_db(
# _item,
# _remote_upload['servers'],
# )
# _item['servers'] = _published
"""Upload status"""
# logging.warning('*****************************************')
# logging.warning("--- Change status to uploaded ---")
# media.update_status_media(
# [{u'media': _item['media']}], {'status': constants.STATUS['uploaded'], 'cloud': _downloaded['success'][-1]['code']})
_published_items.append(_item)
return success_response(_published_items)