def get_series_from_backlog(language): """Find a file in the database by his slug :param language: Ide of the language in the database """ """Find in database""" _session = app.apps.get("db_sqlalchemy")() _items = _session.query(Serie).\ filter(Serie.language == language, Serie.status == constants.STATUS['airing']).\ order_by(asc(Serie.series_id_reference)).\ all() _session.close() """Check if the file exists""" if not _items: return error_response(msg="Items not found.") """Transform data""" _items_response = [{ **_item.to_dict(), 'series_id': _item.series_id_reference } for _item in _items] _data_response = { u"animes": _items_response, } return success_response(data=_data_response, msg="Items found.")
def get_episodes_from_website(series_id, lang, headers): """Prepare the payload""" _payload = { u"lang": lang, u"series_id": series_id, } """Define url""" _url = "{0}/{1}/episodes".format(URL_ANIME, series_id) """Get all novels from website""" logging.warning('*****************************************') logging.warning('_url') logging.warning(_url) logging.warning('*****************************************') logging.warning('_payload') logging.warning(_payload) _episodes = requests.get(_url, params=_payload, headers=headers) logging.warning('*****************************************') logging.warning('_episodes.text') logging.warning(_episodes) """Check if the response is valid""" if _episodes.status_code != 200: """Return error if the response is invalid""" return error_response(msg=_episodes.text) """Get json response""" _episodes_json = _episodes.json() logging.warning('*****************************************') logging.warning('_episodes_json') logging.warning(_episodes_json) """Return novels""" return success_response(_episodes_json.get('data'))
def get_episodes_by_serie_db(serie): """Find episodes by serie id :param serie: Identificator for serie """ """Find in database""" _session = app.apps.get("db_sqlalchemy")() _episodes = _session.query( Media.media, Media.cover, Media.created_at, Media.number, Media.title, ).\ filter( Media.serie == serie, Media.is_deleted == False, Media.is_active == True, Media.status == constants.STATUS['uploaded'], ).\ order_by(desc(Media.media_id_reference)).\ all() _session.close() """Check if the file exists""" if not _episodes: return error_response(msg="Episodes not found.") """Transform data""" _episodes_json = [{ u"episode": _episode[0], u"cover": _episode[1], u"created_at": _episode[2].strftime("%c"), u"number": _episode[3], u"title": _episode[4], } for _episode in _episodes] return success_response(data=_episodes_json, msg="Episodes found.")
def publish_items( items, language, credential, headers, ): """Get chapters from all novels""" _media_items = get_items_by_slug( items, headers=headers, language=language, ) """Check if it hasn't novels, response to client""" if not _media_items: return error_response(msg="New items not found.") """Publish or update on website""" _published = publish_items_db( _media_items, language, headers=headers, # watermark_code=watermark_code, ) """Transform data response""" _data_response = {"anime": _published} return success_response(_data_response)
def get_items_from_backlog(limit): """Find a file in the database by his slug """ """Find in database""" _session = app.apps.get("db_sqlalchemy")() _items = _session.query(Media, Serie).\ join(Serie, Media.serie == Serie.serie, isouter=True).\ filter(Media.status == constants.STATUS['initial']).\ order_by(asc(Media.media)).\ limit(limit).\ all() _session.close() """Check if the file exists""" if not _items: return error_response(msg="Items not found.") """Transform data""" _items_response = [{ **_item.Serie.to_dict(), **_item.Media.to_dict(), } for _item in _items] _data_response = { u"items": _items_response, } return success_response(data=_data_response, msg="Items found.")
def publish_episodes_mirrors(req: Request, res: Response, next: Next): """Publish hentai""" _result = anime.publish_episodes_mirrors(data=req.all_params(), ) """Check if exist an error""" if _result['valid'] is False: return res.bad_request(_result) """Response the data to client""" res.ok(success_response(data=_result['data'], msg="Items added."))
def get_episode_by_id(req: Request, res: Response, next: Next): """Get episode from db""" _requests_db = media.get_episode_by_id_db(req.param('media')) """Check if exist an error""" if _requests_db['valid'] is False: return res.bad_request(_requests_db) """Transform data response""" _data_response = {**_requests_db['data']} """Response the data to client""" res.ok(success_response(data=_data_response, msg=_requests_db['msg']))
def update_status_media(items, body={}): """Create session""" _session = app.apps.get("db_sqlalchemy")() _items = [{**_item, **body} for _item in items] """For each novel deactive without results""" _session.bulk_update_mappings(Media, _items) """Session commit""" _session.commit() """Close session""" _session.close() """Response""" return success_response(msg="Items updated.")
def publish_episodes_mirrors(data, ): """Upload status""" logging.warning('*****************************************') logging.warning("--- Change status to uploaded ---") media.update_status_media( [{ u'media': data['media'] }], { 'status': constants.STATUS['uploaded'], 'cloud': data['success'][-1]['code'] }) return success_response(data)
def get_episode_by_id_db(media): """Find media by id :param media: Identificator for media """ """Find in database""" _session = app.apps.get("db_sqlalchemy")() _episode = _session.query(Media).\ filter(Media.media == media, Media.is_deleted == False, Media.is_active == True).\ first() """Check if the file exists""" if not _episode: _session.close() return error_response(msg="Media not found.") _servers = _session.query(Server).\ filter(Server.media == _episode.media, Server.is_deleted == False, Server.is_active == True, Server.status == constants.STATUS['uploaded']).\ all() """Transform data""" _episode_json = _episode.to_dict() _episode_json['servers'] = [_server.to_dict() for _server in _servers] """Pagination""" _episode_next = _session.query(Media).\ filter( Media.media > media, Media.serie == _episode_json['serie'], Media.is_deleted == False, Media.is_active == True ).\ first() if _episode_next: _episode_json['next'] = _episode_next.to_dict() else: _episode_json['next'] = None _episode_prev = _session.query(Media).\ filter( Media.media < media, Media.serie == _episode_json['serie'], Media.is_deleted == False, Media.is_active == True ).\ order_by(desc(Media.media)).\ first() if _episode_prev: _episode_json['prev'] = _episode_prev.to_dict() else: _episode_json['prev'] = None """Close session""" _session.close() """Response data""" return success_response(data=_episode_json, msg="Media found.")
def get_language_code_db(language): """Create session""" _session = app.apps.get("db_sqlalchemy")() """Find in database""" _language = _session.query(Language).\ filter(Language.language == language).\ first() if not _language: return error_response(msg="Language not found") """Requests to json""" _language_json = _language.to_dict() """Close session""" _session.close() """Response""" return success_response(data=_language_json, msg="Language found.")
def publish_items_anime( items, language, limit_publish, headers, ): _new_animes = [] _new_episodes_all = [] for _item in items: if len(_new_episodes_all) >= limit_publish: break _new_episodes = [] """Validate if item exists""" _old_item = get_anime_by_anime_id_db(_item['series_id'], language['language']) """Validate if it exists""" if not _old_item['valid']: continue """Validate if it has new episodes""" _old_media = get_media_by_anime_id_db(_old_item['data']['serie'], language['language']) """Get episodes""" _episodes = get_episodes_from_website( _item['series_id'], language['hreflang'], headers, ) """Get new episodes""" for _episode in _episodes['data']['episodes']: _old_episode = None for _episode_db in _old_media['data']['items']: if _episode_db['media_id_reference'] == _episode['media_id']: _old_episode = _episode_db break if not _old_episode: _new_episodes.append(_episode) _new_episodes_all.append(_episode) if len(_new_episodes_all) >= limit_publish: break """Valid it has new episodes""" if not _new_episodes: continue """Define anime and new episodes""" _new_anime = {**_item, **_old_item['data'], u'episodes': _new_episodes} """Add anime""" _new_animes.append(_new_anime) _data_response = {u'items': _new_animes} return success_response(_data_response)
def get_items_from_db(lang, limit, headers): """Prepare the payload""" _payload = { u"lang": lang, u"limit": limit, } """Get all novels from website""" _anime = requests.get(URL_ANIME, params=_payload, headers=headers) """Check if the response is valid""" if _anime.status_code != 200: """Return error if the response is invalid""" return error_response(msg=_anime.text) """Get json response""" _anime_json = _anime.json() """Return novels""" return success_response(_anime_json.get('data'))
def publish_latest(req: Request, res: Response, next: Next): """Validate obligate params""" _validate = validate_obligate_fields({ u'username': req.headers.get('username'), u'password': req.headers.get('password'), }) # """Check if has errors return a error response""" if _validate["valid"] is False: return res.bad_request( error_response("{} is necesary.".format(_validate["error"]))) _headers = { u'username': req.headers.get('username'), u'password': req.headers.get('password'), } _description_upload = req.headers.get('description_upload', ITEM_DESCRIPTION_UPLOAD) _storage_credential = req.headers.get('credential', STORAGE_CREDENTIALS_DEFAULT) _items = anime.get_items_from_backlog( # language=_lang['data']['language'], limit=req.param('limit', WEBSITE_LIMIT_LATEST, callback=int)) if _items['valid'] is False: return res.bad_request(_items) """Publish hentai""" _result = anime.upload_items( _items['data']['items'], headers=_headers, description_upload=_description_upload, credential=_storage_credential, resolution=req.param('resolution', MEDIA_RESOLUTION_DEFAULT, callback=int), ) """Check if exist an error""" if _result['valid'] is False: return res.bad_request(_result) """Transform data response""" _data_response = {"items": _result['data']} """Response the data to client""" res.ok(success_response(data=_data_response, msg="Items uploaded."))
def get_episode_by_media_id_db(media_id, serie): """Find a file in the database by his slug :param media_id_reference: Media Id of item in the database :param serie: Serie of reference """ """Find in database""" _session = app.apps.get("db_sqlalchemy")() _item = _session.query(Media).\ filter(Media.media_id_reference == media_id, Media.serie == serie).\ first() _session.close() """Check if the file exists""" if not _item: return error_response(msg="Item not found.") """Transform data""" _data_response = { **_item.to_dict(), } return success_response(data=_data_response, msg="Media found.")
def get_anime_by_anime_id_db(series_id_reference, language): """Find a file in the database by his slug :param series_id_reference: Media Id of item in the database :param language: Id of the language in the database """ """Find in database""" _session = app.apps.get("db_sqlalchemy")() _item = _session.query(Serie).\ filter(Serie.series_id_reference == series_id_reference, Serie.language == language).\ first() _session.close() """Check if the file exists""" if not _item: return error_response(msg="Serie not found.") """Transform data""" _data_response = { **_item.to_dict(), } return success_response(data=_data_response, msg="Serie found.")
def get_media_by_anime_id_db(serie, language): """Find a file in the database by his slug :param series_id_reference: Media Id of item in the database :param language: Id of the language in the database """ """Find in database""" _session = app.apps.get("db_sqlalchemy")() _items = _session.query(Media).\ filter(Media.serie == serie).\ all() _session.close() """Check if the file exists""" """Transform data""" _items_response = [{ **_item.to_dict(), } for _item in _items] """Transform data""" _data_response = {'items': _items_response} return success_response(data=_data_response, msg="Items found.")
def publish_latest_episodes(req: Request, res: Response, next: Next): """Validate obligate params""" _validate = validate_obligate_fields({ u'username': req.headers.get('username'), u'password': req.headers.get('password'), }) # """Check if has errors return a error response""" if _validate["valid"] is False: return res.bad_request( error_response("{} is necesary.".format(_validate["error"]))) _headers = { u'username': req.headers.get('username'), u'password': req.headers.get('password'), } _storage_credential = req.headers.get('credential', STORAGE_CREDENTIALS_DEFAULT) """Get the langauge""" _lang = languages.get_language_slug_db( req.param('lang', WEBSITE_LANG_LATEST)) """Check if exist an error""" if _lang['valid'] is False: return res.bad_request(_lang) _upgrade = req.param('upgrade') if not _upgrade: _items = anime.get_items_from_website(lang=_lang['data']['hreflang'], limit=req.param( 'limit', WEBSITE_LIMIT_LATEST, callback=int), headers=_headers) else: """Search in db""" _items = anime.get_series_from_backlog( language=_lang['data']['language']) if _items['valid'] is False: return res.bad_request(_items) """Publish hentai""" _result = anime.publish_items_anime( _items['data']['animes'], _lang.get('data'), limit_publish=req.param('limit_publish', WEBSITE_LIMIT_PUBLISH, callback=int), headers=_headers, ) """Check if exist an error""" if _result['valid'] is False: return res.bad_request(_result) """Publish hentai""" _result_publish = anime.publish_items( _result['data']['items'], _lang.get('data'), credential=_storage_credential, headers=_headers, ) """Check if exist an error""" if _result_publish['valid'] is False: return res.bad_request(_result_publish) """Response the data to client""" res.ok(success_response(data=_result_publish['data'], msg="Items added."))
def upload_items(items, headers, description_upload, credential, resolution): """For each item to do the following""" _published_items = [] for _item in items: logging.warning('*****************************************') logging.warning('Get episode') _url = "{0}/{1}/episodes/{2}/download".format( URL_ANIME, _item['series_id_reference'], _item['media_id_reference']) logging.warning('*****************************************') logging.warning('get_download_by_params()') """Get the langauge""" _lang = languages.get_language_code_db(_item['language']) """Check if exist an error""" if _lang['valid'] is False: break _download_mirrors = media.get_download_by_params( headers=headers, url=_url, lang=_lang['data']['hreflang']) """Check if it hasn't novels, response to client""" logging.warning('*****************************************') logging.warning('_download_mirrors') if not _download_mirrors: """Upload status""" logging.warning('*****************************************') logging.warning("--- Change status to error ---") media.update_status_media([{ u'media': _item['media'] }], {'status': constants.STATUS['error']}) continue else: logging.warning('*****************************************') logging.warning("--- Change status to uploading ---") """Upload status""" media.update_status_media([{ u'media': _item['media'] }], {'status': constants.STATUS['uploading']}) _available_resolutions = [] for _playlist in _download_mirrors['playlists']: _available_resolutions.append( _playlist['stream_info']['resolution']['height']) _available_resolutions = list(dict.fromkeys(_available_resolutions)) _available_resolutions.sort() if not resolution in _available_resolutions: resolution = _available_resolutions[-1] _output = '{0} {1}-{2}-{3}p.mp4'.format(_lang['data']['prefix'], _item['slug'], _item['number'], resolution) logging.warning('*****************************************') logging.warning(_output) _downloaded = None _parent = "anime{0}{1}".format(_lang['data']['hreflang'], _item['series_id_reference']) logging.warning('*****************************************') logging.warning("---Downloading episode as {0} ---".format(_output)) for _playlist in _download_mirrors['playlists']: if (_playlist['stream_info']['resolution']['height'] == resolution ): logging.warning('*****************************************') logging.warning('Downloading stream from: {0}'.format( _playlist['uri'])) """Change the credential""" _credential = _lang['data']['credential'] """Set description""" description_upload += _lang['data']['prefix'] """Download files""" _downloaded = media.download_episode_proxi( _playlist['uri'], _output, _credential, description_upload, parent=_parent, data={'media': _item['media']}) # _downloaded = media.download_episode( # _playlist['uri'], _output, credential, description_upload, parent=_parent) break logging.warning('*****************************************') logging.warning(_downloaded) # if not _downloaded: # continue """Get URL""" # _remote_upload = servers.remote_upload_episode( # _downloaded['success'][0]) # _remote_upload = servers.remote_upload_episode( # 'a06959f895a811eb86700242ac120002') # print("--- Save serve into db ---") """Publish or update on website""" # _published = servers.publish_servers_db( # _item, # _remote_upload['servers'], # ) # _item['servers'] = _published """Upload status""" # logging.warning('*****************************************') # logging.warning("--- Change status to uploaded ---") # media.update_status_media( # [{u'media': _item['media']}], {'status': constants.STATUS['uploaded'], 'cloud': _downloaded['success'][-1]['code']}) _published_items.append(_item) return success_response(_published_items)