def yandex(url):
try:
cookie = client.request(url, output='cookie')
r = client.request(url, cookie=cookie)
r = re.sub(r'[^\x00-\x7F]+', ' ', r)
sk = re.findall('"sk"\s*:\s*"([^"]+)', r)[0]
idstring = re.findall('"id"\s*:\s*"([^"]+)', r)[0]
idclient = binascii.b2a_hex(os.urandom(16))
post = {
'idClient': idclient,
'version': '3.9.2',
'sk': sk,
'_model.0': 'do-get-resource-url',
'id.0': idstring
}
post = urllib.urlencode(post)
r = client.request('https://yadi.sk/models/?_m=do-get-resource-url',
post=post,
cookie=cookie)
r = json.loads(r)
url = r['models'][0]['data']['file']
return url
except:
return
def episodeAbsoluteNumber(self, thetvdb, season, episode):
try:
url = 'http://thetvdb.com/api/%s/series/%s/default/%01d/%01d' % ('MUQ2MkYyRjkwMDMwQzQ0NA=='.decode('base64'), thetvdb, int(season), int(episode))
return int(client.parseDOM(client.request(url), 'absolute_number')[0])
except:
pass
return episode
def get_size(url):
try:
size = client.request(url, output='file_size')
if size == '0': size = False
size = convert_size(size)
return size
except:
return False
def getTVShowTranslation(self, thetvdb, lang):
try:
url = 'http://thetvdb.com/api/%s/series/%s/%s.xml' % ('MUQ2MkYyRjkwMDMwQzQ0NA=='.decode('base64'), thetvdb, lang)
r = client.request(url)
title = client.parseDOM(r, 'SeriesName')[0]
title = client.replaceHTMLCodes(title)
title = title.encode('utf-8')
return title
except:
pass
def geturl(url):
try:
r = client.request(url, output='geturl')
if r is None: return r
host1 = re.findall('([\w]+)[.][\w]+$',
urlparse.urlparse(url.strip().lower()).netloc)[0]
host2 = re.findall('([\w]+)[.][\w]+$',
urlparse.urlparse(r.strip().lower()).netloc)[0]
if host1 == host2: return r
proxies = sorted(get(), key=lambda x: random.random())
proxies = sorted(proxies, key=lambda x: random.random())
proxies = proxies[:3]
for p in proxies:
p += urllib.quote_plus(url)
r = client.request(p, output='geturl')
if r is not None: return parse(r)
except:
pass
def vk(url):
try:
query = urlparse.parse_qs(urlparse.urlparse(url).query)
try:
oid, video_id = query['oid'][0], query['id'][0]
except:
oid, video_id = re.findall('\/video(.*)_(.*)', url)[0]
sources_url = 'http://vk.com/al_video.php?act=show_inline&al=1&video=%s_%s' % (
oid, video_id)
html = client.request(sources_url)
html = re.sub(r'[^\x00-\x7F]+', ' ', html)
sources = re.findall('(\d+)x\d+.+?(http.+?\.m3u8.+?)n', html)
if not sources:
sources = re.findall('"url(\d+)"\s*:\s*"(.+?)"', html)
sources = [(i[0], i[1].replace('\\', '')) for i in sources]
sources = dict(sources)
url = []
try:
url += [{'quality': 'HD', 'url': sources['720']}]
except:
pass
try:
url += [{'quality': 'SD', 'url': sources['540']}]
except:
pass
try:
url += [{'quality': 'SD', 'url': sources['480']}]
except:
pass
if not url == []: return url
try:
url += [{'quality': 'SD', 'url': sources['360']}]
except:
pass
if not url == []: return url
try:
url += [{'quality': 'SD', 'url': sources['240']}]
except:
pass
if not url == []: return url
except:
return
def cldmailru(url):
try:
v = url.split('public')[-1]
r = client.request(url)
r = re.sub(r'[^\x00-\x7F]+', ' ', r)
tok = re.findall('"tokens"\s*:\s*{\s*"download"\s*:\s*"([^"]+)', r)[0]
url = re.findall('"weblink_get"\s*:\s*\[.+?"url"\s*:\s*"([^"]+)', r)[0]
url = '%s%s?key=%s' % (url, v, tok)
return url
except:
return
def googlepass(url):
try:
try:
headers = dict(urlparse.parse_qsl(url.rsplit('|', 1)[1]))
except:
headers = None
url = url.split('|')[0].replace('\\', '')
url = client.request(url, headers=headers, output='geturl')
if 'requiressl=yes' in url:
url = url.replace('http://', 'https://')
else:
url = url.replace('https://', 'http://')
if headers: url += '|%s' % urllib.urlencode(headers)
return url
except:
return
def cdnImport(uri, name):
import imp
from civitasscrapers.modules import client
path = os.path.join(dataPath, 'py' + name)
path = path.decode('utf-8')
deleteDir(os.path.join(path, ''), force=True)
makeFile(dataPath) ; makeFile(path)
r = client.request(uri)
p = os.path.join(path, name + '.py')
f = openFile(p, 'w') ; f.write(r) ; f.close()
m = imp.load_source(name, p)
deleteDir(os.path.join(path, ''), force=True)
return m
def odnoklassniki(url):
try:
media_id = re.compile('//.+?/.+?/([\w]+)').findall(url)[0]
result = client.request('http://ok.ru/dk', post={'cmd': 'videoPlayerMetadata', 'mid': media_id})
result = re.sub(r'[^\x00-\x7F]+', ' ', result)
result = json.loads(result).get('videos', [])
hd = []
for name, quali in {'ultra': '4K', 'quad': '1440p', 'full': '1080p', 'hd': 'HD'}.items():
hd += [{'quality': quali, 'url': i.get('url')} for i in result if i.get('name').lower() == name]
sd = []
for name, quali in {'sd': 'SD', 'low': 'SD', 'lowest': 'SD', 'mobile': 'SD'}.items():
sd += [{'quality': quali, 'url': i.get('url')} for i in result if i.get('name').lower() == name]
url = hd + sd[:1]
if not url == []: return url
except:
return
def __getTrakt(url, post=None):
try:
url = urlparse.urljoin(BASE_URL, url)
post = json.dumps(post) if post else None
headers = {
'Content-Type': 'application/json',
'trakt-api-key': V2_API_KEY,
'trakt-api-version': 2
}
if getTraktCredentialsInfo():
headers.update({
'Authorization':
'Bearer %s' % control.setting('trakt.token')
})
result = client.request(url,
post=post,
headers=headers,
output='extended',
error=True)
resp_code = result[1]
resp_header = result[2]
result = result[0]
if resp_code in [
'500', '502', '503', '504', '520', '521', '522', '524'
]:
log_utils.log('Temporary Trakt Error: %s' % resp_code,
log_utils.LOGWARNING)
return
elif resp_code in ['404']:
log_utils.log('Object Not Found : %s' % resp_code,
log_utils.LOGWARNING)
return
# elif resp_code in ['429']:
# log_utils.log('Trakt Rate Limit Reached: %s' % resp_code, log_utils.LOGWARNING)
# return
if resp_code not in ['401', '405']:
return result, resp_header
oauth = urlparse.urljoin(BASE_URL, '/oauth/token')
opost = {
'client_id': V2_API_KEY,
'client_secret': CLIENT_SECRET,
'redirect_uri': REDIRECT_URI,
'grant_type': 'refresh_token',
'refresh_token': control.setting('trakt.refresh')
}
result = client.request(oauth, post=json.dumps(opost), headers=headers)
result = utils.json_loads_as_str(result)
token, refresh = result['access_token'], result['refresh_token']
control.setSetting(id='trakt.token', value=token)
control.setSetting(id='trakt.refresh', value=refresh)
headers['Authorization'] = 'Bearer %s' % token
result = client.request(url,
post=post,
headers=headers,
output='extended',
error=True)
return result[0], result[2]
except Exception as e:
log_utils.log('Unknown Trakt Error: %s' % e, log_utils.LOGWARNING)
pass
def authTrakt():
try:
if getTraktCredentialsInfo() == True:
if control.yesnoDialog(
control.lang(32511).encode('utf-8'),
control.lang(32512).encode('utf-8'), '', 'Trakt'):
control.setSetting(id='trakt.user', value='')
control.setSetting(id='trakt.token', value='')
control.setSetting(id='trakt.refresh', value='')
raise Exception()
result = getTraktAsJson('/oauth/device/code',
{'client_id': V2_API_KEY})
verification_url = (control.lang(32513) %
result['verification_url']).encode('utf-8')
user_code = (control.lang(32514) % result['user_code']).encode('utf-8')
expires_in = int(result['expires_in'])
device_code = result['device_code']
interval = result['interval']
progressDialog = control.progressDialog
progressDialog.create('Trakt', verification_url, user_code)
for i in range(0, expires_in):
try:
if progressDialog.iscanceled(): break
time.sleep(1)
if not float(i) % interval == 0: raise Exception()
r = getTraktAsJson(
'/oauth/device/token', {
'client_id': V2_API_KEY,
'client_secret': CLIENT_SECRET,
'code': device_code
})
if 'access_token' in r: break
except:
pass
try:
progressDialog.close()
except:
pass
token, refresh = r['access_token'], r['refresh_token']
headers = {
'Content-Type': 'application/json',
'trakt-api-key': V2_API_KEY,
'trakt-api-version': 2,
'Authorization': 'Bearer %s' % token
}
result = client.request(urlparse.urljoin(BASE_URL, '/users/me'),
headers=headers)
result = utils.json_loads_as_str(result)
user = result['username']
control.setSetting(id='trakt.user', value=user)
control.setSetting(id='trakt.token', value=token)
control.setSetting(id='trakt.refresh', value=refresh)
raise Exception()
except:
control.openSettings('4.1')
def google(url):
try:
if any(x in url for x in ['youtube.', 'docid=']): url = 'https://drive.google.com/file/d/%s/view' % re.compile('docid=([\w-]+)').findall(url)[0]
netloc = urlparse.urlparse(url.strip().lower()).netloc
netloc = netloc.split('.google')[0]
if netloc == 'docs' or netloc == 'drive':
url = url.split('/preview', 1)[0]
url = url.replace('drive.google.com', 'docs.google.com')
headers = {'User-Agent': client.agent()}
result = client.request(url, output='extended', headers=headers)
try:
headers['Cookie'] = result[2]['Set-Cookie']
except:
pass
result = result[0]
if netloc == 'docs' or netloc == 'drive':
result = re.compile('"fmt_stream_map",(".+?")').findall(result)[0]
result = json.loads(result)
result = [i.split('|')[-1] for i in result.split(',')]
result = sum([googletag(i, append_height=True) for i in result], [])
elif netloc == 'photos':
result = result.replace('\r', '').replace('\n', '').replace('\t', '')
result = re.compile('"\d*/\d*x\d*.+?","(.+?)"').findall(result)[0]
result = result.replace('\\u003d', '=').replace('\\u0026', '&')
result = re.compile('url=(.+?)&').findall(result)
result = [urllib.unquote(i) for i in result]
result = sum([googletag(i, append_height=True) for i in result], [])
elif netloc == 'picasaweb':
id = re.compile('#(\d*)').findall(url)[0]
result = re.search('feedPreload:\s*(.*}]}})},', result, re.DOTALL).group(1)
result = json.loads(result)['feed']['entry']
if len(result) > 1:
result = [i for i in result if str(id) in i['link'][0]['href']][0]
elif len(result) == 1:
result = result[0]
result = result['media']['content']
result = [i['url'] for i in result if 'video' in i['type']]
result = sum([googletag(i, append_height=True) for i in result], [])
elif netloc == 'plus':
id = (urlparse.urlparse(url).path).split('/')[-1]
result = result.replace('\r', '').replace('\n', '').replace('\t', '')
result = result.split('"%s"' % id)[-1].split(']]')[0]
result = result.replace('\\u003d', '=').replace('\\u0026', '&')
result = re.compile('url=(.+?)&').findall(result)
result = [urllib.unquote(i) for i in result]
result = sum([googletag(i, append_height=True) for i in result], [])
result = sorted(result, key=lambda i: i.get('height', 0), reverse=True)
url = []
for q in ['4K', '1440p', '1080p', 'HD', 'SD']:
try:
url += [[i for i in result if i.get('quality') == q][0]]
except:
pass
for i in url:
i.pop('height', None)
i.update({'url': i['url'] + '|%s' % urllib.urlencode(headers)})
if not url: return
return url
except:
return
def request(url,
check,
close=True,
redirect=True,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
XHR=False,
limit=None,
referer=None,
cookie=None,
compression=True,
output='',
timeout='30'):
try:
r = client.request(url,
close=close,
redirect=redirect,
proxy=proxy,
post=post,
headers=headers,
mobile=mobile,
XHR=XHR,
limit=limit,
referer=referer,
cookie=cookie,
compression=compression,
output=output,
timeout=timeout)
if r is not None and error is not False: return r
if check in str(r) or str(r) == '': return r
proxies = sorted(get(), key=lambda x: random.random())
proxies = sorted(proxies, key=lambda x: random.random())
proxies = proxies[:3]
for p in proxies:
p += urllib.quote_plus(url)
if post is not None:
if isinstance(post, dict):
post = utils.byteify(post)
post = urllib.urlencode(post)
p += urllib.quote_plus('?%s' % post)
r = client.request(p,
close=close,
redirect=redirect,
proxy=proxy,
headers=headers,
mobile=mobile,
XHR=XHR,
limit=limit,
referer=referer,
cookie=cookie,
compression=compression,
output=output,
timeout='20')
if check in str(r) or str(r) == '': return r
except:
pass
