def get_metadata( self, link ) :
try :
link = clear_url(link)
cookie = self.get_cookie(self = self) if hasattr(self, 'get_cookie') else None
page = requests.get( link, headers = self.HEADERS, cookies = cookie )
if page.status_code == 200 :
tree = html.fromstring( page.text )
return self.run( self = self, content = page.text, xpath = tree, link = link, update_video_detail = False )
else :
return makeResponseFailed({'status_code': page.status_code})
except Exception as ex :
return makeResponseFailed({'exception': str(ex)})
async def get_metadata_async( self, link, update_video_detail = False ) :
try :
link = clear_url(link)
async with aiohttp.ClientSession() as session:
cookie = self.get_cookie(self = self) if hasattr(self, 'get_cookie') else None
async with session.get(link, headers = self.HEADERS_NO_UTF8, cookies = cookie) as resp:
if resp.status == 200 :
page_content = await resp.text()
tree = html.fromstring(page_content)
return await self.run_async( self = self, content = page_content, xpath = tree, link = link, update_video_detail = update_video_detail )
else :
return makeResponseFailed({'status_code': resp.status_code})
except Exception as ex :
import traceback
print(traceback.format_exc(), file = sys.stderr)
return makeResponseFailed({'exception': str(ex)})
async def get_unique_id_async(self, link):
link = clear_url(link)
return await self.unique_id_async(self=self, link=link)
def get_unique_id(self, link):
link = clear_url(link)
return self.unique_id(self=self, link=link)
def dispatch(url):
url = clear_url(url.strip())
for cralwer in _dispatch_map:
if cralwer.test(self=cralwer, url=url):
return cralwer, cralwer.normalize_url(self=cralwer, url=url)
return None, None