def process_page(self, url):
request = self._s.get(real_url(url))
self.soup = BeautifulSoup(clean_html(request.text), 'html.parser')
debug(url)
tbl = self.soup.find('table', class_='tablesorter')
if tbl:
tbody = tbl.find('tbody')
if tbody:
for tr in tbody.find_all('tr'):
item = {}
cat_a = tr.find('a', class_='gen')
if cat_a:
item['category'] = cat_a['href']
topic_a = tr.find('a', class_='topictitle')
if topic_a:
item['a'] = topic_a
dl_a = tr.find('a', attrs={'rel': "nofollow"})
if dl_a:
item['dl_link'] = dl_a['href']
seeds_td = tr.find('td', attrs={'title': "Seeders"})
if seeds_td:
item['seeds'] = seeds_td.get_text()
self._items.append(item.copy())
def process_page(self, url):
try:
request = self._s.get(real_url(url, self.settings))
except requests.exceptions.ConnectionError:
return
self.soup = BeautifulSoup(clean_html(request.text), 'html.parser')
debug(url)
indx = self.soup.find('div', attrs={'id': 'index'})
if indx:
bgnd = indx.find('tr', class_='backgr')
if bgnd:
for row in bgnd.next_siblings:
td2 = row.contents[1]
td5 = row.contents[-1]
item = {}
topic_a = td2.find_all('a')
if topic_a:
topic_a = topic_a[-1]
item['a'] = topic_a
dl_a = td2.find('a', class_='downgif')
if dl_a:
item['dl_link'] = dl_a['href']
span_green = td5.find('span', class_='green')
if span_green:
item['seeds'] = span_green.get_text().strip()
self._items.append(item.copy())
def process_page(self, url):
request = self._s.get(url)
self.soup = BeautifulSoup(clean_html(request.text), 'html.parser')
debug(url)
for tbl in self.soup.select('table.pline'):
self._items.append(tbl)
def process_page(self, url):
request = self._s.get(url)
self.soup = BeautifulSoup(clean_html(request.text), 'html.parser')
debug(url)
tbl = self.soup.find('table', class_='tablesorter')
if tbl:
tbody = tbl.find('tbody')
if tbody:
for tr in tbody.find_all('tr'):
item = {}
cat_a = tr.find('a', class_='gen')
if cat_a:
item['category'] = cat_a['href']
topic_a = tr.find('a', class_='topictitle')
if topic_a:
item['a'] = topic_a
dl_a = tr.find('a', attrs={'rel': "nofollow"})
if dl_a:
item['dl_link'] = dl_a['href']
seeds_td = tr.find('td', attrs={'title': "Seeders"})
if seeds_td:
item['seeds'] = seeds_td.get_text()
self._items.append(item.copy())
def process_page(self, url):
request = self._s.get(real_url(url))
self.soup = BeautifulSoup(clean_html(request.text), 'html.parser')
debug(url)
for tbl in self.soup.select('table.pline'):
self._items.append(tbl)
def download_torrent(url, path, settings):
from base import save_hashes
save_hashes(path)
import shutil
url = urllib2.unquote(url)
debug('download_torrent:' + url)
href = None
link = None # find_direct_link(url, settings)
if link is None:
s = create_session(settings)
page = s.get(url)
# debug(page.text.encode('cp1251'))
soup = BeautifulSoup(clean_html(page.text), 'html.parser')
a = soup.select('td.gensmall > span.genmed > b > a')
if len(a) > 0:
href = 'http://nnm-club.me/forum/' + a[0]['href']
else:
href = linkd
response = urllib2.urlopen(real_url(link, settings))
#CHUNK = 256 * 1024
with filesystem.fopen(path, 'wb') as f:
shutil.copyfileobj(response, f)
save_hashes(path)
return True
#r = requests.head(link)
#debug(r.headers)
#return False
if href:
def make_req():
if link:
return requests.get(real_url(link, settings), verify=False)
else:
return s.get(href,
headers={'Referer': real_url(url, settings)})
try:
r = make_req()
if not r.ok and r.status_code == 502:
import time
time.sleep(1)
r = make_req()
if 'Content-Type' in r.headers:
if not 'torrent' in r.headers['Content-Type']:
return False
with filesystem.fopen(path, 'wb') as torr:
for chunk in r.iter_content(100000):
torr.write(chunk)
save_hashes(path)
return True
except:
pass
return False
def Actors(self):
if self.actors is not None:
return self.actors
self.actors = []
if self.kinopoisk_url:
cast_url = self.kinopoisk_url + 'cast/'
r = self._http_get(cast_url)
if r.status_code == requests.codes.ok:
soup = BeautifulSoup(base.clean_html(r.text), 'html.parser')
for a in soup.select('a[name="actor"]'):
for sibling in a.next_siblings:
if not hasattr(sibling, 'tag'):
continue
if sibling.tag == 'a':
return self.actors
for actorInfo in sibling.select('.actorInfo'):
photo = actorInfo.select('div.photo a')[0]['href']
#http://st.kp.yandex.net/images/actor_iphone/iphone360_30098.jpg
#/name/7627/
photo = photo.replace('/', '').replace('name', '')
photo = 'http://st.kp.yandex.net/images/actor_iphone/iphone360_' + photo + '.jpg'
ru_name = actorInfo.select('div.info .name a')[0].get_text()
en_name = actorInfo.select('div.info .name span')[0].get_text()
role = actorInfo.select('div.info .role')[0].get_text().replace('... ', '')
role = role.split(',')[0]
self.actors.append({'photo': photo,'ru_name': ru_name,'en_name': en_name,'role': role})
return self.actors
def get_google_cache(self, url):
import urllib
search_url = "http://www.google.com/search?q=" + urllib.quote_plus(url)
headers = {'user-agent': user_agent}
r = self.session.get(search_url, headers=headers, timeout=2.0)
try:
soup = BeautifulSoup(base.clean_html(r.text), 'html.parser')
a = soup.find('a', class_='fl')
if a:
cache_url = a['href']
import urlparse
res = urlparse.urlparse(cache_url)
res = urlparse.ParseResult(
res.scheme if res.scheme else 'https', res.netloc
if res.netloc else 'webcache.googleusercontent.com',
res.path, res.params, res.query, res.fragment)
cache_url = urlparse.urlunparse(res)
#print cache_url
r = self.session.get(cache_url, headers=headers, timeout=2.0)
indx = r.text.find('<html')
resp = Object()
resp.status_code = r.status_code
resp.text = r.text[indx:]
return resp
except BaseException as e:
debug(str(e))
return requests.Response()
def download_torrent(url, path, settings):
url = urllib2.unquote(url)
debug('download_torrent:' + url)
page = requests.get(real_url(url, settings))
# debug(page.text.encode('cp1251'))
soup = BeautifulSoup(clean_html(page.text), 'html.parser')
a = soup.select('#download > a')
if len(a) > 1:
link = a[1]['href']
else:
link = None
if link:
r = requests.get(real_url(link, settings))
debug(r.headers)
if 'Content-Type' in r.headers:
if not 'torrent' in r.headers['Content-Type']:
return False
try:
with filesystem.fopen(path, 'wb') as torr:
for chunk in r.iter_content(100000):
torr.write(chunk)
return True
except:
pass
return False
def get_magnet_link(url):
r = requests.get(real_url(url))
if r.status_code == requests.codes.ok:
soup = BeautifulSoup(clean_html(r.text), 'html.parser')
for a in soup.select('a[href*="magnet:"]'):
debug(a['href'])
return a['href']
return None
def get_magnet_link(url):
r = requests.get(real_url(url, settings))
if r.status_code == requests.codes.ok:
soup = BeautifulSoup(clean_html(r.text), 'html.parser')
for a in soup.select('a[href*="magnet:"]'):
debug(a['href'])
return a['href']
return None
def __init__(self, imdb_id):
headers = {'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7'}
resp = requests.get('http://www.imdb.com/title/' + imdb_id + '/',
headers=headers)
if resp.status_code == requests.codes.ok:
text = base.clean_html(resp.content)
self.page = BeautifulSoup(text, 'html.parser')
def parse_description(self, html_text):
self.soup = BeautifulSoup(clean_html(html_text), 'html.parser')
tag = u''
self._dict['gold'] = False
for a in self.soup.select('img[src="images/gold.gif"]'):
self._dict['gold'] = True
debug('gold')
for span in self.soup.select('span.postbody span'):
try:
text = span.get_text()
tag = self.get_tag(text)
if tag != '':
if tag != u'plot':
self._dict[tag] = base.striphtml(unicode(span.next_sibling).strip())
else:
self._dict[tag] = base.striphtml(unicode(span.next_sibling.next_sibling).strip())
debug('%s (%s): %s' % (text.encode('utf-8'), tag.encode('utf-8'), self._dict[tag].encode('utf-8')))
except:
pass
if 'genre' in self._dict:
self._dict['genre'] = self._dict['genre'].lower().replace('.', '')
count_id = 0
for a in self.soup.select('a[href*="www.imdb.com/title/"]'):
try:
href = a['href']
components = href.split('/')
if components[2] == u'www.imdb.com' and components[3] == u'title':
self._dict['imdb_id'] = components[4]
count_id += 1
except:
pass
if count_id > 1:
return False
for img in self.soup.select('var.postImg'): # ('img.postImg'):
try:
self._dict['thumbnail'] = img['title']
debug('!!!!!!!!!!!!!!thumbnail: ' + self._dict['thumbnail'])
break
except:
pass
self.parse_country_studio()
if self.settings:
if self.settings.use_kinopoisk:
for kp_id in self.soup.select('#kp_id'):
self._dict['kp_id'] = kp_id['href']
self.make_movie_api(self.get_value('imdb_id'), self.get_value('kp_id'))
return True
def Trailer(self):
if self.kinopoisk_url:
trailer_page = self.kinopoisk_url + 'video/type/1/'
r = self._http_get(trailer_page)
if r.status_code == requests.codes.ok:
soup = BeautifulSoup(base.clean_html(r.text), 'html.parser')
for div in soup.select('tr td div div.flag2'):
trailer = self.__trailer(div)
if trailer:
return trailer
for a in soup.select('a.all'):
return self.__trailer(a)
return None
def getTitle(self):
title = None
if self.kinopoisk_url and self.soup is None:
r = self._http_get(self.kinopoisk_url)
if r.status_code == requests.codes.ok:
self.soup = BeautifulSoup(base.clean_html(r.text), 'html.parser')
if self.soup:
h = self.soup.find('h1', class_ = 'moviename-big')
if h:
title = h.contents[0].strip()
return title
def download_torrent(url, path, settings):
import shutil
url = urllib2.unquote(url)
debug('download_torrent:' + url)
href = None
link = find_direct_link(url, settings)
if link is None:
s = create_session(settings)
page = s.get(real_url(url))
# debug(page.text.encode('cp1251'))
soup = BeautifulSoup(clean_html(page.text), 'html.parser')
a = soup.select('td.gensmall > span.genmed > b > a')
if len(a) > 0:
href = 'http://nnm-club.me/forum/' + a[0]['href']
debug(s.headers)
else:
href = link
response = urllib2.urlopen(real_url(link))
#CHUNK = 256 * 1024
with filesystem.fopen(path, 'wb') as f:
shutil.copyfileobj(response, f)
return True
#r = requests.head(link)
#debug(r.headers)
#return False
if href:
if link:
r = requests.get(real_url(link))
else:
r = s.get(real_url(href), headers={'Referer': real_url(url)}, verify=False)
debug(r.headers)
# 'Content-Type': 'application/x-bittorrent'
if 'Content-Type' in r.headers:
if not 'torrent' in r.headers['Content-Type']:
return False
try:
with filesystem.fopen(path, 'wb') as torr:
for chunk in r.iter_content(100000):
torr.write(chunk)
return True
except:
pass
return False
def create_session(settings):
s = requests.Session()
r = s.get(real_url("http://nnm-club.me/forum/login.php"))
soup = BeautifulSoup(clean_html(r.text), 'html.parser')
for inp in soup.select('input[name="code"]'):
code = inp['value']
# debug(code)
data = {"username": settings.nnmclub_login, "password": settings.nnmclub_password,
"autologin": "******", "code": code, "redirect": "", "login": ""}
login = s.post(real_url("http://nnm-club.me/forum/login.php"), data=data,
headers={'Referer': real_url("http://nnm-club.me/forum/login.php")})
debug('Login status: %d' % login.status_code)
return s
def trailer(self):
if self.kinopoisk_url:
trailer_page = self.kinopoisk_url + 'video/type/1/'
r = self._http_get(trailer_page)
if r.status_code == requests.codes.ok:
text = base.clean_html(r.text)
soup = BeautifulSoup(text, 'html.parser')
if not soup:
return None
for div in soup.select('tr td div div.flag2'):
trailer = self.__trailer(div)
if trailer:
return trailer
for a in soup.select('a.all'):
return self.__trailer(a)
return None
def get_passkey(settings=None, session=None):
if session is None and settings is None:
return None
if session is None:
session = create_session(settings)
page = session.get(real_url('http://nnm-club.me/forum/profile.php?mode=editprofile'))
soup = BeautifulSoup(clean_html(page.text), 'html.parser')
next = False
for span in soup.select('span.gen'):
if next:
return span.get_text()
if span.get_text() == u'Текущий passkey:':
next = True
return None
def get_passkey(settings=None, session=None):
if session is None and settings is None:
return None
if session is None:
session = create_session(settings)
page = session.get('http://nnm-club.me/forum/profile.php?mode=editprofile')
soup = BeautifulSoup(clean_html(page.text), 'html.parser')
next = False
for span in soup.select('span.gen'):
if next:
return span.get_text()
if span.get_text() == u'Текущий passkey:':
next = True
return None
def get_uid(settings, session=None):
if session is None:
session = create_session(settings)
try:
page = session.get(real_url('http://nnm-club.me/'))
if page.status_code == requests.codes.ok:
soup = BeautifulSoup(clean_html(page.text), 'html.parser')
'''
a = soup.select_one('a[href*="profile.php"]')
if a is None:
return None
'''
for a in soup.select('a.mainmenu'):
m = re.search('profile.php.+?u=(\d+)', a['href'])
if m:
return m.group(1)
else:
debug('page.status_code: ' + str(page.status_code))
except BaseException as e:
log.print_tb(e)
pass
return None
def get_uid(settings, session=None):
if session is None:
session = create_session(settings)
try:
page = session.get('http://nnm-club.me/')
if page.status_code == requests.codes.ok:
soup = BeautifulSoup(clean_html(page.text), 'html.parser')
'''
a = soup.select_one('a[href*="profile.php"]')
if a is None:
return None
'''
for a in soup.select('a.mainmenu'):
m = re.search('profile.php.+?u=(\d+)', a['href'])
if m:
return m.group(1)
else:
debug('page.status_code: ' + str(page.status_code))
except BaseException as e:
log.print_tb(e)
pass
return None
def download_torrent(url, path, settings):
from base import save_hashes
save_hashes(path)
url = urllib2.unquote(url)
debug('download_torrent:' + url)
page = requests.get(real_url(url, settings))
soup = BeautifulSoup(clean_html(page.text), 'html.parser')
a = soup.select('#download > a')
if len(a) > 1:
link = a[1]['href']
else:
link = None
if link:
r = requests.get(real_url(link, settings))
debug(r.headers)
if 'Content-Type' in r.headers:
if not 'torrent' in r.headers['Content-Type']:
return False
try:
with filesystem.fopen(path, 'wb') as torr:
for chunk in r.iter_content(100000):
torr.write(chunk)
save_hashes(path)
return True
except:
pass
return False
def actors(self):
if self._actors is not None:
return self._actors
self._actors = []
if self.kinopoisk_url:
cast_url = self.kinopoisk_url + 'cast/'
r = self._http_get(cast_url)
if r.status_code == requests.codes.ok:
text = base.clean_html(r.text)
soup = BeautifulSoup(text, 'html.parser')
if not soup:
return []
for actorInfo in soup.find_all('div', class_='actorInfo'):
photo = actorInfo.select('div.photo a')[0]['href']
#http://st.kp.yandex.net/images/actor_iphone/iphone360_30098.jpg
#/name/7627/
photo = photo.replace('/', '').replace('name', '')
photo = 'http://st.kp.yandex.net/images/actor_iphone/iphone360_' + photo + '.jpg'
ru_name = actorInfo.select(
'div.info .name a')[0].get_text()
en_name = actorInfo.select(
'div.info .name span')[0].get_text()
role = actorInfo.select(
'div.info .role')[0].get_text().replace('... ', '')
role = role.split(',')[0]
self._actors.append({
'photo': photo,
'ru_name': ru_name,
'en_name': en_name,
'role': role
})
return self._actors
def parse_description(self, html_text):
from HTMLParser import HTMLParseError
html_text = clean_html(html_text)
try:
self.soup = BeautifulSoup(html_text, 'html.parser')
except HTMLParseError as e:
log.print_tb(e)
log.debug(html_text)
return False
tag = u''
for b in self.soup.select('#details b'):
try:
text = b.get_text()
tag = self.get_tag(text)
if tag == 'plot':
self._dict[tag] = base.striphtml(unicode(b.next_sibling.next_sibling).strip())
debug('%s (%s): %s' % (text.encode('utf-8'), tag.encode('utf-8'), self._dict[tag].encode('utf-8')))
elif tag == 'genre':
genres = []
elements = b.findNextSiblings('a')
for a in elements:
if '/tag/' in a['href']:
genres.append(a.get_text())
self._dict[tag] = u', '.join(genres)
elif tag != '':
self._dict[tag] = base.striphtml(unicode(b.next_sibling).strip())
debug('%s (%s): %s' % (text.encode('utf-8'), tag.encode('utf-8'), self._dict[tag].encode('utf-8')))
except:
pass
if 'genre' in self._dict:
self._dict['genre'] = self._dict['genre'].lower().replace('.', '')
for tag in [u'title', u'year', u'genre', u'director', u'actor', u'plot']:
if tag not in self._dict:
return False
count_id = 0
for a in self.soup.select('a[href*="www.imdb.com/title/"]'):
try:
href = a['href']
components = href.split('/')
if components[2] == u'www.imdb.com' and components[3] == u'title':
self._dict['imdb_id'] = components[4]
count_id += 1
except:
pass
if count_id == 0:
div_index = self.soup.select('#index')
if div_index:
for a in div_index[0].findAll('a', recursive=True):
if '/torrent/' in a['href']:
parts = a['href'].split('/')
href = parts[0] + '/' + parts[1] + '/' + parts[2]
html = urllib2.urlopen(real_url(href, settings))
soup = BeautifulSoup(clean_html(html.read()), 'html.parser')
for a in soup.select('a[href*="www.imdb.com/title/"]'):
try:
href = a['href']
components = href.split('/')
if components[2] == u'www.imdb.com' and components[3] == u'title':
self._dict['imdb_id'] = components[4]
count_id += 1
except:
pass
if 'imdb_id' in self._dict:
break
if count_id > 1:
return False
if 'imdb_id' not in self._dict:
return False
for det in self.soup.select('#details'):
tr = det.find('tr', recursive=False)
if tr:
tds = tr.findAll('td', recursive=False)
if len(tds) > 1:
td = tds[1]
img = td.find('img')
try:
self._dict['thumbnail'] = img['src']
debug('!!!!!!!!!!!!!!thumbnail: ' + self._dict['thumbnail'])
break
except:
pass
if self.settings:
if self.settings.use_kinopoisk:
for kp_id in self.soup.select('a[href*="www.kinopoisk.ru/"]'):
self._dict['kp_id'] = kp_id['href']
self.make_movie_api(self.get_value('imdb_id'), self.get_value('kp_id'))
return True
def makeSoup(self):
if self.kinopoisk_url and self.soup is None:
r = self._http_get(self.kinopoisk_url)
if r.status_code == requests.codes.ok:
text = base.clean_html(r.text)
self.soup = BeautifulSoup(text, 'html.parser')
def parse_description(self, html_text):
self.soup = BeautifulSoup(clean_html(html_text), 'html.parser')
tag = u''
self._dict['gold'] = False
for a in self.soup.select('img[src="images/gold.gif"]'):
self._dict['gold'] = True
debug('gold')
for span in self.soup.select('.postbody span'):
try:
text = span.get_text()
tag = self.get_tag(text)
if tag != '':
if tag != u'plot':
self._dict[tag] = base.striphtml(
unicode(span.next_sibling).strip())
else:
self._dict[tag] = base.striphtml(
unicode(span.next_sibling.next_sibling).strip())
debug('%s (%s): %s' %
(text.encode('utf-8'), tag.encode('utf-8'),
self._dict[tag].encode('utf-8')))
except:
pass
if 'genre' in self._dict:
self._dict['genre'] = self._dict['genre'].replace('.', '')
count_id = 0
for a in self.soup.select('a[href*="www.imdb.com/title/"]'):
try:
href = a['href']
components = href.split('/')
if components[2] == u'www.imdb.com' and components[
3] == u'title':
self._dict['imdb_id'] = components[4]
count_id += 1
except:
pass
if count_id > 1:
return False
img = self.soup.find('var', class_='postImg')
if img:
try:
self._dict['thumbnail'] = img['title'].split('?link=')[-1]
debug('!!!!!!!!!!!!!!thumbnail: ' + self._dict['thumbnail'])
except:
pass
if 'thumbnail' not in self._dict:
imgs = self.soup.select('span.postbody > img')
try:
self._dict['thumbnail'] = imgs[0]['src'].split('?link=')[-1]
debug('!!!!!!!!!!!!!!thumbnail: ' + self._dict['thumbnail'])
except BaseException as e:
pass
self.parse_country_studio()
try:
kp = self.soup.select_one('div.kpi a')
except TypeError:
kp = None
if not kp:
try:
kp = self.soup.select_one('#kp_id')
except TypeError:
kp = None
if kp:
self._dict['kp_id'] = kp['href']
self.make_movie_api(self.get_value('imdb_id'), self.get_value('kp_id'),
self.settings)
return True
def parse_description(self, html_text):
from HTMLParser import HTMLParseError
html_text = clean_html(html_text)
try:
self.soup = BeautifulSoup(html_text, 'html.parser')
except HTMLParseError as e:
log.print_tb(e)
log.debug(html_text)
return False
tag = u''
for b in self.soup.select('#details b'):
try:
text = b.get_text()
tag = self.get_tag(text)
if tag == 'plot':
plot = base.striphtml(
unicode(b.next_sibling.next_sibling).strip())
if plot:
self._dict[tag] = plot
debug('%s (%s): %s' %
(text.encode('utf-8'), tag.encode('utf-8'),
self._dict[tag].encode('utf-8')))
elif tag == 'genre':
genres = []
elements = b.findNextSiblings('a')
for a in elements:
if '/tag/' in a['href']:
genres.append(a.get_text())
self._dict[tag] = u', '.join(genres)
elif tag != '':
self._dict[tag] = base.striphtml(
unicode(b.next_sibling).strip())
debug('%s (%s): %s' %
(text.encode('utf-8'), tag.encode('utf-8'),
self._dict[tag].encode('utf-8')))
except:
pass
tags = []
for tag in [
u'title', u'year', u'genre', u'director', u'actor', u'plot'
]:
if tag not in self._dict:
tags.append(tag)
if tags:
try:
details = self.soup.select_one('#details').get_text()
lines = details.split('\n')
for l in lines:
if ':' in l:
key, desc = l.split(':', 1)
key = key.strip(u' \r\n\t✦═')
desc = desc.strip(u' \r\n\t')
tag = self.get_tag(key + ':')
if tag and desc and tag not in self._dict:
self._dict[tag] = desc
except BaseException as e:
debug('No parse #details')
debug(e)
pass
if 'genre' in self._dict:
self._dict['genre'] = self._dict['genre'].lower().replace('.', '')
if 'video' in self._dict:
self._dict['video'] = self._dict['video'].replace('|', ',')
if self.settings.rutor_nosd:
video = self._dict['video']
parts = video.split(',')
for part in parts:
part = part.strip()
if 'XviD' in part:
return False
m = re.search(ur'(\d+)[xXхХ](\d+)', part)
if m:
w = int(m.group(1))
#h = int(m.group(2))
if w < 1280:
return False
else:
pass
count_id = 0
for a in self.soup.select('a[href*="www.imdb.com/title/"]'):
try:
href = a['href']
components = href.split('/')
if components[2] == u'www.imdb.com' and components[
3] == u'title':
self._dict['imdb_id'] = components[4]
count_id += 1
except:
pass
if count_id == 0:
div_index = self.soup.select('#index')
if div_index:
for a in div_index[0].findAll('a', recursive=True):
if '/torrent/' in a['href']:
parts = a['href'].split('/')
href = parts[0] + '/' + parts[1] + '/' + parts[2]
html = urllib2.urlopen(real_url(href, self.settings))
soup = BeautifulSoup(clean_html(html.read()),
'html.parser')
for a in soup.select('a[href*="www.imdb.com/title/"]'):
try:
href = a['href']
components = href.split('/')
if components[
2] == u'www.imdb.com' and components[
3] == u'title':
self._dict['imdb_id'] = components[4]
count_id += 1
except:
pass
if 'imdb_id' in self._dict:
break
if count_id > 1:
return False
if 'imdb_id' not in self._dict:
if not hasattr(self.settings, 'no_skip_by_imdb'):
return False
for det in self.soup.select('#details'):
tr = det.find('tr', recursive=False)
if tr:
tds = tr.findAll('td', recursive=False)
if len(tds) > 1:
td = tds[1]
img = td.find('img')
try:
self._dict['thumbnail'] = img['src']
debug('!!!!!!!!!!!!!!thumbnail: ' +
self._dict['thumbnail'])
break
except:
pass
for kp_id in self.soup.select('a[href*="www.kinopoisk.ru/"]'):
self._dict['kp_id'] = kp_id['href']
self.make_movie_api(self.get_value('imdb_id'), self.get_value('kp_id'),
self.settings)
return True
def create_session(settings):
try:
return settings.session
except AttributeError:
s = requests.Session()
cookies = None
if settings.nnmclub_use_ssl:
cookies = dict(ssl='enable_ssl')
r = s.get(real_url("http://nnm-club.me/forum/login.php", settings),
verify=False)
soup = BeautifulSoup(clean_html(r.text), 'html.parser')
code = ''
for inp in soup.select('input[name="code"]'):
code = inp['value']
# debug(code)
data = {
"username": settings.nnmclub_login,
"password": settings.nnmclub_password,
"autologin": "******",
"code": code,
"redirect": "",
"login": ""
}
login = s.post(real_url("http://nnm-club.me/forum/login.php",
settings),
data=data,
verify=False,
cookies=cookies,
headers={
'Referer':
real_url("http://nnm-club.me/forum/login.php",
settings)
})
debug('Login status: %d' % login.status_code)
class MySession():
def __init__(self, session, settings):
self.session = session
self.settings = settings
def _prepare(self, kwargs):
if settings.nnmclub_use_ssl:
kwargs['verify'] = False
kwargs['cookies'] = cookies
def get(self, url, **kwargs):
self._prepare(kwargs)
return self.session.get(real_url(url, self.settings), **kwargs)
def post(self, url, **kwargs):
self._prepare(kwargs)
return self.session.post(real_url(url, self.settings),
**kwargs)
s = MySession(s, settings)
settings.session = s
return s
