def load(self): url_from_name = form_url_from_name(self.film.name_orig) if not url_from_name is None: reqobj = simple_tor_get_page(url_from_name) html_with_type = HTML_with_type(reqobj.decode('utf-8')) len(html_with_type) html_with_type.page_type='film_page' return {'html':html_with_type, 'url_view':url_from_name, 'url':url_from_name} else: html_with_type =HTML_with_type(simple_tor_get_page(form_search_url(self.film.name))) html_with_type.page_type = 'search_page' return {'html':html_with_type, 'url_view':url_from_name, 'url':url_from_name}
def get_film_data(self): search_film_url = '/hbo/api/v1/films.json?' site_name = 'www.amediateka.ru' filter_film_search = 'limit=1000&offset=0&expand=genres&client_id=amediateka&platform=desktop' url = "http://{}{}{}".format(site_name, search_film_url, filter_film_search) response = simple_tor_get_page(url) data_site = json.loads(response)['films'] film = Films.objects.values_list('id', 'name') data = film.values('name', 'id') locations = { 'info': [], 'type': 'amediateka_ru' } for f in data_site: for film in data: if f['name'] == film['name']: film_data = Films.objects.filter(id=film['id']) for dict_film in film_data: d = self.film_dict(dict_film, f) one_loc_res = save_location(**d) save_existed_location_to_locs_dict(locations, one_loc_res) break fill_log_table_for_not_schema_corresponded_robots(locations) robot_is_banned = MultiLocationRobotsBunCheck.is_result_looks_like_robot_banned(locations) if not robot_is_banned: LocationRobotsCorrector.correct_locations(locations, 'amediateka') return locations
def update_kinopoisk_persone(pid): try: response = simple_tor_get_page('http://www.kinopoisk.ru/name/{}/view_info/ok/#trivia'.format(pid), True) soup = BeautifulSoup(response) tag = soup.find('span', attrs={'itemprop': 'alternativeHeadline'}) orig_name = tag.text.strip() p = Persons.objects.get(kinopoisk_id=pid) tag_birthdate = soup.find('td', attrs={'class': 'birth'}) birthdate = '' print "ID = ", p.id if not (tag_birthdate is None): birthdate = tag_birthdate.get('birthdate') else: print 'No data birthdate for this person id = {}'.format(pid) tags_bio = soup.findAll('li', attrs={'class': 'trivia'}) bio = '' if len(tags_bio): for li in tags_bio: bio = bio + ' ' + li.text else: print 'No biography for this person id = {}'.format(pid) p.name_orig = orig_name p.birthdate = birthdate p.bio = bio p.kinopoisk_id = pid if p.photo == '' and p.kinopoisk_id != 0: p.photo.save('profile.jpg', File(get_photo(p.kinopoisk_id))) p.save() except Exception, e: import traceback traceback.print_exc()
def parse(self, response, dict_gen, film, url): d = dict_gen(film) content = simple_tor_get_page(url) value = '' isFilm = False try: soup = BeautifulSoup(content) tag = soup.find('meta', attrs={'property': 'og:type'}) if (not tag is None and film.type == APP_FILM_FULL_FILM)\ or (film.type == APP_FILM_SERIAL and tag is None): tag = soup.find('div', attrs={'class': 'big_rating'}) id = tag.get('id') value = self.parse_value + id isFilm = True except: pass if isFilm: d['url_view'] = url d['value'] = value d['price_type'] = 0 d['price'] = self.get_price() d['type'] = 'zoomby' return [d] return []
def parse(self, response, dict_gen, film, url): d = dict_gen(film) content = simple_tor_get_page(url) value = '' isFilm = False try: soup = BeautifulSoup(content) tag = soup.find('meta', attrs={'property': 'og:type'}) if (not tag is None and film.type == APP_FILM_FULL_FILM)\ or (film.type == APP_FILM_SERIAL and tag is None): tag = soup.find('div', attrs={'class': 'big_rating'}) id = tag.get('id') value = self.parse_value + id isFilm = True except: pass if isFilm: d['url_view'] = url d['value'] = value d['price_type'] = 0 d['price'] = self.get_price() d['type'] = 'zoomby' return [d] return []
def get_film_data(self): url = 'http://my.mail.ru/video/catalog/movies' content = simple_tor_get_page(url) soup = BeautifulSoup(content) locations = { 'info': [], 'type': 'mail_ru' } items = soup.findAll('a', {'class': 'link-default'}) for item in items: film_name = item.text film_link = item.get('href') film_url = HOST + film_link film_dict = self.get_film_dict(film_name) if not film_dict is None: film_dict['type'] = 'mail_ru' film_dict['url_view'] = film_url film_dict['price'] = 0 film_dict['price_type'] = APP_CONTENTS_PRICE_TYPE_FREE one_loc_res = save_location(**film_dict) save_existed_location_to_locs_dict(locations, one_loc_res) fill_log_table_for_not_schema_corresponded_robots(locations) return locations
def parse_search(response, film_name, year): film_link = None try: soup = BeautifulSoup(response) if not (soup.find('div', {'class': 'empty-search'}) is None): return None search_div = soup.find('div', {'class': 'card-list'}) film_divs = search_div.find_all( 'div', {'class': ['card', 'no-rationale', 'tall-cover', 'movies tiny']}) for film in film_divs: film_tag = film.find('a', {'class': 'title'}) if film_name == film_tag.get('title'): film_link = 'http://play.google.com' + film_tag.get('href') if film_link: page = simple_tor_get_page(film_link) soup_page = BeautifulSoup(page) film_year = soup_page.find('div', { 'itemprop': 'datePublished' }).text film_year = re.search(ur'\d+', film_year) if not str(year) in film_year.group(): return None except IndexError: film_link = None return film_link
def get_data(self): all_film_url = 'http://viaplay.ru/filmy/vse/5/alphabetical' locations = { 'info': [], 'type': 'viaplay' } content = simple_tor_get_page(all_film_url) soup_films = BeautifulSoup(content).find('ul', {'class': 'atoz-list'}).li.ul.find_all('li') films = Films.objects.values('name', 'id') for li_film in soup_films: for film in films: if li_film.a.text.lower().strip().encode('utf-8').translate(None, string.punctuation) == film['name'].lower().strip().encode('utf-8').translate(None, string.punctuation): link = 'http://viaplay.ru' + li_film.a.get('href') film_query_set = Films.objects.filter(id=film['id']) for obj in film_query_set: d = self.film_dict(obj, link) one_loc_res = save_location(**d) save_existed_location_to_locs_dict(locations, one_loc_res) break fill_log_table_for_not_schema_corresponded_robots(locations) robot_is_banned = MultiLocationRobotsBunCheck.is_result_looks_like_robot_banned(locations) if not robot_is_banned: LocationRobotsCorrector.correct_locations(locations, 'viaplay') return locations
def get_soup(): ''' Getting data from playfamily site and parsing its xml into BeautfulSoup object ''' xmldata = simple_tor_get_page(PLAYFAMILY_XML) soup = BeautifulSoup(xmldata,'xml') return soup
def get_image(template, actor_id): try: r = simple_tor_get_page(template.format(actor_id), tor_flag=False) return convert_file(r) except Exception as e: import traceback traceback.print_exc() return None
def parse_translation_live_russia_tv(): translation_list = [] # Get current date current_date = timezone.now() # Get page translation_page = simple_tor_get_page(TRANSLATION_URL) soup = BeautifulSoup(translation_page) translation_bloc = soup.find('div', {'class': ['broadcasts', 'tab-non-active', 'tab-broadcasts']}) for trans in translation_bloc.find_all('li'): try: # Get date and time time_tag = trans.find('div', {'class': 'time'}) time = re.findall(ur'\d+', time_tag.text) date_tag = trans.find('div', {'class': 'label'}) date_str = date_tag.text.lower().strip().split() if len(date_str) == 1: if date_str[0] == u'сегодня': date = timezone.datetime(year=current_date.year, month=current_date.month, day=current_date.day, hour=int(time[0]), minute=int(time[1])) elif len(date_str) == 3: date = timezone.datetime(year=int(date_str[2]), month=MONTHS[date_str[1]], day=int(date_str[0]), hour=int(time[0]), minute=int(time[1])) else: continue # Get title title_tag = trans.h2 title = title_tag.text #Get link href = trans.a.get('href') link = TRANSLATION_URL + href #Get video id video_id = href.split('/')[-1] #Create dict with information about translation translation_data = { 'title': title, 'date': date, 'price': float(0), 'link': link, 'meta': {}, 'embed_code': None, 'value': video_id, 'player': PLAYER_LINK % video_id } translation_list.append(translation_data) except Exception, e: print e.message
def __init__(self, film_id): self.film = Films.objects.get(id=film_id) search_film = urllib.urlencode( {'text': (self.film.name.encode('utf-8'))}) search_url = URL_SEARCH.format(search_film) url = "https://%s/%s" % ( HOST, search_url, ) self.response = simple_tor_get_page(url)
def kinopoisk_news(): data = BeautifulSoup( simple_tor_get_page(KINOPOISK_PREMIERES_URL).content.decode('cp1251')) big_names = data.select('span.name_big') names = data.select('span.name') for name in big_names + names: if name.a: kinopoisk_id = int(name.a.attrs['href'].split('/')[2]) kinopoisk_name = name.a.text yield kinopoisk_name, kinopoisk_id
def parse_search(response, film_name, film_year): film_link = None try: flag = False soup = BeautifulSoup(response) if soup.find(attrs={'data-href': 'video_search'}) is None: return None class_tag = soup.find('aside', {'role': 'complementary'}) if class_tag: li_list = class_tag.find_all('li') for li in li_list: if u'Видео' in li.text: flag = True break if flag: search_divs = soup.find_all('div', {'class': 'catalog-list search'}) film_div = None for div in search_divs: if div.figure: film_div = div break if film_div: films = film_div.find_all('figure') for film in films: if film_name.lower().strip().encode('utf-8').translate(None, string.punctuation) == film.figcaption.div.header.strong.text.lower().strip().encode('utf-8').translate(None, string.punctuation): film_link = 'http://www.zabava.ru' + film.a.get('href') break else: return None else: return None if film_link: film_page = simple_tor_get_page(film_link) film_soup = BeautifulSoup(film_page) year_bloc = film_soup.find('div', {'class': 'mbottom10'}) reg = re.compile(ur'Год издания') year_tag = None for e in year_bloc.find_all('em'): if reg.match(e.text): year_tag = e.parent break if year_tag: year = re.search(ur'\d+', year_tag.text) year = int(year.group()) if film_year.year != year: film_link = None else: return None except: film_link = None return film_link
def kinopoisk_news(): data = BeautifulSoup(simple_tor_get_page(KINOPOISK_PREMIERES_URL).content.decode('cp1251')) big_names = data.select('span.name_big') names = data.select('span.name') for name in big_names + names: if name.a: kinopoisk_id = int(name.a.attrs['href'].split('/')[2]) kinopoisk_name = name.a.text yield kinopoisk_name, kinopoisk_id
def load(self): url_from_name = form_url_from_name(self.film.name_orig) if not url_from_name is None: reqobj = simple_tor_get_page(url_from_name) html_with_type = HTML_with_type(reqobj.decode('utf-8')) len(html_with_type) html_with_type.page_type = 'film_page' return { 'html': html_with_type, 'url_view': url_from_name, 'url': url_from_name } else: html_with_type = HTML_with_type( simple_tor_get_page(form_search_url(self.film.name))) html_with_type.page_type = 'search_page' return { 'html': html_with_type, 'url_view': url_from_name, 'url': url_from_name }
def get_vote(soup): csslink = [lnk.attrs['href'] for lnk in soup.find_all('link') if 'votes' in lnk.attrs['href']][0] # TODO implement caching css = simple_tor_get_page(csslink, tor_flag=True) m = re.search('[.]starbar[ ]{width[:][ ](?P<width>[0-9]+)px', css) parent_width = float(m.groupdict()['width']) starbar_div = soup.select('div.starbar_w') child_width = float(dict([i.split(':') for i in starbar_div[0].attrs['style'].split(';')])['width'].replace('px', '')) return round(child_width / (parent_width * 10), 2)
def get_data(self): all_film_url = 'http://viaplay.ru/filmy/vse/5/alphabetical' content = simple_tor_get_page(all_film_url) soup_films = BeautifulSoup(content).find('ul', { 'class': 'atoz-list' }).li.ul.find_all('li') films = Films.objects.values('name', 'id') for li_film in soup_films: for film in films: if li_film.a.text == film['name']: link = 'http://viaplay.ru' + li_film.a.get('href') film_query_set = Films.objects.filter(id=film['id']) for obj in film_query_set: d = self.film_dict(obj, link) save_location(**d) break
def parse_news(robot_name): #Get site site = SITE_DICT[robot_name] # Get page page = simple_tor_get_page(site['news_url']) soup = BeautifulSoup(page) # Constant for punctuation punctuation = string.punctuation trans = string.maketrans(punctuation, ' ' * len(punctuation)) # String constant pattern = '!"#$%&\'()*+,-./:;<=>–№«»?@[\\]^_`{|}~ ' # ascii: 160 cursor = connection.cursor() #Get films from page films = soup.find_all(*site['films_tag_args']) news = [] for film in films: try: # Find name from tag name_tag = film.find(*site['name_tag_args']) name = name_tag.text.lower().strip().encode('utf-8').translate(trans) # Find year from tag year_tag = film.find(*site['year_tag_args']) year = int(re.search(ur'\d+', year_tag.text).group()) query = """SELECT * FROM (SELECT films.id, films.name, EXTRACT(YEAR FROM films.release_date) AS year, regexp_split_to_array(trim(both lower(translate(films.name, E%s, %s))), E'\\s+') AS new_name FROM films) AS t WHERE t.year=%s AND t.new_name=regexp_split_to_array(%s, E'\\s+')""" cursor.execute(query, [pattern, ' ' * len(pattern), year, name]) result = dict_fetch_all_without_gen(cursor) if len(result) == 1: link_tag = name_tag if robot_name == ROBOT_STREAM else name_tag.a link = site['host'] + link_tag.get('href') news.append({'film_id': result[0]['id'], 'url': link}) except Exception, e: print e.message
def get_film_data(self): search_film_url = '/hbo/api/v1/films.json?' filter_film_search = 'limit=1000&offset=0&expand=genres&client_id=amediateka&platform=desktop' url = "http://{}{}{}".format('www.amediateka.ru', search_film_url, filter_film_search) response = simple_tor_get_page(url) data_site = json.loads(response)['films'] film = Films.objects.values_list('id', 'name') data = film.values('name', 'id') for f in data_site: for film in data: if f['name'] == film['name']: film_data = Films.objects.filter(id=film['id']) for dict_film in film_data: d = self.film_dict(dict_film, f) save_location(**d) break
def get_serials_data(self): search_serials_url = '/hbo/api/v1/serials.json?' filter_serials_search = 'limit=1000&offset=0&expand=seasons,genres&client_id=amediateka&platform=desktop' url = "http://{}{}{}".format('www.amediateka.ru', search_serials_url, filter_serials_search) response = simple_tor_get_page(url) data_site = json.loads(response)['serials'] data = Films.objects.values_list('id', 'name').values('name', 'id') for s in data_site: for serials in data: if s['name'] == serials['name']: serials_data = Films.objects.filter(id=serials['id']) for dict_serials in serials_data: if dict_serials.type == APP_FILM_SERIAL: list_serial = self.serial_dict(dict_serials, s) for ser in list_serial: save_location(**ser) break
def parse_translation_championat_com(): translation_list = [] champ_dict = {} # Get page list_translations_page = simple_tor_get_page(TRANSLATION_URL+'/broadcast/') list_translations_soup = BeautifulSoup(list_translations_page) championship_bloc = list_translations_soup.find('div', {'class': 'broadcast__menu'}) #Create championship map for champ in championship_bloc.find_all('div', {'class': 'broadcast__menu__i'}): try: img = champ.find('div', {'class': 'broadcast__tournament'}) if img: champ_dict[img.get('class')[-1]] = champ.p.text except Exception, e: print e.message
def get_serials_data(self): search_serials_url = '/hbo/api/v1/serials.json?' filter_serials_search = 'limit=1000&offset=0&expand=seasons,genres&client_id=amediateka&platform=desktop' url = "http://{}{}{}".format('www.amediateka.ru', search_serials_url, filter_serials_search) response = simple_tor_get_page(url) data_site = json.loads(response)['serials'] data = Films.objects.values_list('id', 'name').values('name', 'id') for s in data_site: for serials in data: if s['name'] == serials['name']: serials_data = Films.objects.filter(id=serials['id']) for dict_serials in serials_data: if dict_serials.type == APP_FILM_SERIAL: list_serial = self.serial_dict(dict_serials, s) for ser in list_serial: save_location(**ser) break
def acquire_page(page_id): if not os.path.exists(PAGE_ARCHIVE): os.mkdir(PAGE_ARCHIVE) dump_path = os.path.join(PAGE_ARCHIVE, str(page_id)) page_dump = '' if os.path.exists(dump_path): with open(dump_path) as fd: page_dump = fd.read().decode('utf-8') if not page_dump: url = u"http://www.kinopoisk.ru/film/%d/cast/" % page_id res = simple_tor_get_page(url, tor_flag=True) page_dump = res.decode('cp1251') with open(dump_path, 'w') as fdw: fdw.write(page_dump.encode('utf-8')) return page_dump
def get_vote(soup): csslink = [ lnk.attrs['href'] for lnk in soup.find_all('link') if 'votes' in lnk.attrs['href'] ][0] # TODO implement caching r = simple_tor_get_page(csslink, tor_flag=True) css = r m = re.search('[.]starbar[ ]{width[:][ ](?P<width>[0-9]+)px', css) parent_width = float(m.groupdict()['width']) starbar_div = soup.select('div.starbar_w') child_width = float( dict([i.split(':') for i in starbar_div[0].attrs['style'].split(';') ])['width'].replace('px', '')) return round(child_width / parent_width * 10, 2)
def acquire_page(page_id, force_reload=False): if not os.path.exists(PAGE_ARCHIVE): os.mkdir(PAGE_ARCHIVE) page_dump = '' dump_path = os.path.join(PAGE_ARCHIVE, str(page_id)) if os.path.exists(dump_path): with open(dump_path) as fd: page_dump = fd.read().decode('utf-8') if not page_dump or force_reload: url = u"http://www.kinopoisk.ru/film/%d/cast/" % page_id res = simple_tor_get_page(url, tor_flag=True) page_dump = res.decode('cp1251') with open(dump_path, 'w') as fdw: fdw.write(page_dump.encode('utf-8')) return page_dump
def get_data(self): try: films = json.loads( self.response)['live_search']['search_movies_result'] for film in films: if film['rus_title'] == self.film.name: film_link = 'https://www.ayyo.ru/movies/%s/' % ( film['slug']) ayyo_film_id = film['movie'] break film_url = 'https://www.ayyo.ru/api/movies/?{}'.format( urllib.urlencode({'id__in': ayyo_film_id})) film_response = simple_tor_get_page(film_url) price = float( json.loads(film_response)['movies']['data'][str(ayyo_film_id)] ['streaming_price']) d = self.film_dict(self.film, film_link, price) save_location(**d) except Exception, e: pass
def get_data(self): locations = { 'info': [], 'type': 'www.ayyo.ru' } try: films = json.loads(self.response)['live_search']['search_movies_result'] print films for film in films: if film['rus_title'].lower().strip().encode('utf-8').translate(None, string.punctuation) == self.film.name.lower().strip().encode('utf-8').translate(None, string.punctuation): film_link = 'https://www.ayyo.ru/movies/%s/' % (film['slug']) ayyo_film_id = film['movie'] break film_url = 'https://www.ayyo.ru/api/movies/?{}'.format(urllib.urlencode({'id__in': ayyo_film_id})) film_response = simple_tor_get_page(film_url) price = float(json.loads(film_response)['movies']['data'][str(ayyo_film_id)]['streaming_price']) d = self.film_dict(self.film, film_link, price) one_loc_res = save_location(**d) save_existed_location_to_locs_dict(locations, one_loc_res) except Exception, e: pass
def kinopoisk_films(pages): try: for page in range(1, pages+1): print u"Page number: {0} of {1}".format(page, pages) html = simple_tor_get_page(KINOPOISK_LIST_FILMS_URL.format(page), tor_flag=True) soup = BeautifulSoup(html) films_list = soup.findAll('div', attrs={'class': 'name'}) for film in films_list: name = film.a.text print u"Film name: {0}".format(name) kinopoisk_id = int(film.a.get('href').split('/')[4]) if u'(сериал)' in name: name = name.replace(u'(сериал)', u'') film, flag = Films.objects.get_or_create(kinopoisk_id=kinopoisk_id, defaults={'type': '', 'name':name}) print u"Film: {0} {1}".format(film.name, film.kinopoisk_id) kinopoisk_parse_one_film.apply_async((film.kinopoisk_id, film.name)) persons_films_update_with_indexes.apply_async((film.kinopoisk_id,)) except Exception, e: import traceback traceback.print_exc()
def update_kinopoisk_persone(pid): try: response = simple_tor_get_page('http://www.kinopoisk.ru/name/{}/view_info/ok/#trivia'.format(pid), True) soup = BeautifulSoup(response) tag = soup.find('span', attrs={'itemprop': 'alternativeHeadline'}) orig_name = tag.text.strip() p = Persons.objects.get(kinopoisk_id=pid) tag_birthdate = soup.find('td', attrs={'class': 'birth'}) birthdate = '' print "ID = ", p.id if not (tag_birthdate is None): birthdate = tag_birthdate.get('birthdate') else: print 'No data birthdate for this person id = {}'.format(pid) bio = '' tags_bio = soup.findAll('li', attrs={'class': 'trivia'}) if len(tags_bio): for li in tags_bio: bio = bio + ' ' + li.text else: print 'No biography for this person id = {}'.format(pid) p.bio = bio p.kinopoisk_id = pid p.name_orig = orig_name p.birthdate = birthdate if p.photo == '' and p.kinopoisk_id != 0: p.photo.save('profile.jpg', File(get_photo(p.kinopoisk_id))) p.save() except Exception, e: traceback_own(e)
def parse_tvzor_news(): # Get page page = simple_tor_get_page(NEW_URL) # Constant for punctuation punctuation = string.punctuation trans = string.maketrans(punctuation, ' ' * len(punctuation)) # String constant pattern = '!"#$%&\'()*+,-./:;<=>–№«»?@[\\]^_`{|}~ ' # ascii: 160 cursor = connection.cursor() #Get films from page in json format films = json.loads(page) news = [] for film in films: try: # Film name name = film['name'].lower().strip().encode('utf-8').translate(trans) # Film year year = int(film['releaseDate']) query = """SELECT * FROM (SELECT films.name, films.id, EXTRACT(YEAR FROM films.release_date) AS year, regexp_split_to_array(trim(both lower(translate(films.name, E%s, %s))), E'\\s+') AS new_name FROM films) AS t WHERE t.year=%s AND t.new_name=regexp_split_to_array(%s, E'\\s+')""" cursor.execute(query, [pattern, ' ' * len(pattern), year, name]) result = dict_fetch_all(cursor) if len(result) == 1: link = HOST + '/movie/' + film['assetId'] news.append({'film_id': result[0]['id'], 'url': link}) except Exception, e: print e.message
def kinopoisk_films(pages): try: for page in range(1, pages+1): print u"Page number: {0} of {1}".format(page, pages) html = simple_tor_get_page(KINOPOISK_LIST_FILMS_URL.format(page), tor_flag=True) soup = BeautifulSoup(html) films_list = soup.findAll('div', attrs={'class': 'name'}) for film in films_list: kinopoisk_id = int(film.a.get('href').split('/')[4]) name = film.a.text print u"Film name: {0}".format(name) if u'(сериал)' in name: name = name.replace(u'(сериал)', u'') film, flag = Films.objects.get_or_create(kinopoisk_id=kinopoisk_id, defaults={'type': '', 'name':name}) print u"Film: {0} {1}".format(film.name, film.kinopoisk_id) kinopoisk_parse_one_film.apply_async((film.kinopoisk_id, film.name)) persons_films_update_with_indexes.apply_async((film.kinopoisk_id,)) except Exception, e: traceback_own(e)
def __init__(self, film_id): self.film = Films.objects.get(id=film_id) search_film = urllib.urlencode({'text': (self.film.name.encode('utf-8'))}) search_url = URL_SEARCH.format(search_film) url = "https://%s/%s" % (HOST, search_url, ) self.response = simple_tor_get_page(url)
def get_data_dict(kinopoisk_id): url = 'http://www.kinopoisk.ru/handler_trailer_popup.php?ids={id}'.format(id=kinopoisk_id) content = simple_tor_get_page(url) return json.loads(content)
def get_data_dict(kinopoisk_id): content = simple_tor_get_page( 'http://www.kinopoisk.ru/handler_trailer_popup.php?ids={}'.format( kinopoisk_id)) return json.loads(content)
time = re.findall(ur'\d+', dates[1].text) #Get price price_tag = trans.find('div', {'class': '_paid'}) price = re.search(ur'\d+', price_tag.text).group() #Get link link_tag = trans.find('a', {'class': 'broadcast__table__link'}) link = TRANSLATION_URL + link_tag.get('href') #Get championship championship_img = trans.find('td', {'class': '_icon'}).div.get('class') championship = champ_dict[championship_img[-1]] #Get value from translation page trans_page = simple_tor_get_page(link) trans_soup = BeautifulSoup(trans_page) value_div = trans_soup.find('div', {'class': 'broadcast'}) value = value_div.iframe.get('src') #Create dict with information about translation translation_data = { 'title': title, 'date': timezone.datetime(year=current_year, month=int(date[1]), day=int(date[0]), hour=int(time[0]), minute=int(time[1])), 'price': float(price), 'link': link, 'meta': {'championship': championship if championship else None}, 'value': value, } translation_list.append(translation_data)
def get_image(template, actor_id): try: result = simple_tor_get_page(template.format(actor_id), tor_flag=False) return convert_file(result) except Exception, e: traceback_own(e)