def get_episodes(self, channel):
url = 'http://www.youtube.com/user/CanalACulturaActiva/feed'
j = 1
shows = {}
html = urllib.urlopen(url).read()
dom = lxml.html.document_fromstring(html)
for item in dom.cssselect('.feed-item-main'):
p = [x.strip() for x in item.cssselect('h4')[0].text_content().split('-')]
show_title = p[0]
episode_title = '-'.join(p[1:])
try:
serie = Serie.objects.get(name=show_title)
except Serie.DoesNotExist:
serie = Serie(name=show_title)
print ">> SERIE: %s" % show_title.encode('utf8')
serie.channel = channel
serie.save()
serie.genres.add(Genre.objects.get_or_create(code='CULTURA',defaults={'name':'Cultura'})[0])
if Episode.objects.filter(name=episode_title).count() > 0:
continue
episode = Episode(serie=serie, name=episode_title, number=0)
print "%s" % episode_title
episode.description = item.cssselect('.description')[0].text_content() + "\n" + \
item.cssselect('.video-time')[0].text_content()
episode.thumbnail = urllib.basejoin(self.BASE_URL, item.cssselect('.video-thumb img')[0].get('src'))
episode.save()
url2 = item.cssselect('a')[0].get('href')
video_id = re.findall('v=([^&]+)', url2)[0]
video_url = get_youtube_url(video_id)
media = HttpMediaFile(width=640, height=480, mimetype='video/mp4', url=video_url)
media.episode = episode
media.save()
serie.episode_set.add(episode)
def scrap_episode(self, serie, url, thumbnail):
if Episode.objects.filter(url=url).count() > 0:
print "EXISTS"
return
print url
html = urllib.urlopen(url).read()
soup = BeautifulSoup(html, from_encoding='utf-8')
info = soup.find('article','info')
p = soup.find('div','description').text
season,number,duration = re.findall('Temporada ([0-9]+) \| Ep. ([0-9]+) \(([0-9:]+)\)', p)[0]
if Episode.objects.filter(serie=serie, season=int(season), number=int(number)).count() > 0:
print "EXISTS"
return
episode = Episode()
episode.serie = serie
episode.season = int(season)
episode.number = int(number)
episode.duration = time(0,*map(int,duration.split(':')))
episode.description = soup.find('div','description').p.children.next()
episode.thumbnail = thumbnail
episode.save()
media = MundoFoxMediaFile()
smil_url = re.findall('player.releaseUrl = "([^"]+)"', html)[0]
smil_url += "&manifest=m3u&format=SMIL&Tracking=true&Embedd=true"
media._url = smil_url
media.episode = episode
media.save()
def scrap_serie(self, serie):
html = urllib.urlopen(serie.url).read()
html = re.sub('<\?.*?\?>','',html)
soup = BeautifulSoup(html, from_encoding='utf8')
for cnt,article in enumerate(soup('td','listado-mediateca-menu')):
episode = Episode()
episode.serie = serie
episode.thumbnail = urllib.basejoin(self.base_url, article.find('div','imagen-mediateca').img.get('src'))
episode.name = article.find('h5','titulo-mediateca').a.text
print episode.name.encode('utf8')
episode.description = article.find('span','texto-mediateca').text
episode_url = urllib.basejoin(self.base_url, article.find('h5','titulo-mediateca').a.get('href'))
episode.number = cnt+1
episode.season = 1
episode.save()
self.scrap_episode(episode, episode_url)
def scrap_serie(self, serie):
html = urllib.urlopen(serie.url).read()
soup = BeautifulSoup(html, from_encoding='utf8')
for cnt,article in enumerate(soup.find('div','primary-pane').ol('li')):
episode = Episode()
episode.thumbnail = article.img.get('src')
episode.name = article.find('span','video-overview').span.text
print episode.name.encode('utf8')
dur = article.find('span','video-time').text.split(':')
episode.duration = time(0, int(dur[0]), int(dur[1]))
episode.serie = serie
episode.number = cnt+1
episode.season = 1
episode.save()
video_id = re.findall('v=([^&]+)', article.a.get('href'))[0]
media = HttpMediaFile()
media.url = self.get_real_url(video_id)
media.episode = episode
media.save()
def get_episode(self, serie, url):
cnt = serie.episode_set.count()+1
html = urllib.urlopen(url).read()
dom = lxml.html.document_fromstring(html)
for elem in dom.cssselect("#ms-player-thumb-videos ul li"):
episode = Episode()
episode.serie = serie
episode.number = cnt
cnt += 1
episode.name = elem.cssselect(".ms-thumb-titulo")[0].text_content()
print episode.name
episode.thumbnail = elem.cssselect(".ms-thumb-img img")[0].get('src')
episode.save()
num = re.findall("\(([0-9])\)", elem.cssselect("a")[0].get('onclick'))[0]
elemscript = dom.cssselect("#ms-player2-%s" % num)[0].getnext()
sig = re.findall('"(.*?)"', elemscript.text_content())[1]
media = HttpMediaFile()
media.episode = episode
media.url = "http://api.kewego.com/video/getHTML5Stream?playerKey=%s&sig=%s&format=normal" % (self.player_key,sig)
media.save()
def scrap_serie(self, serie):
#serie_url = "http://www.conectate.gob.ar/educar-portal-video-web/module/detalleRecurso/DetalleRecurso.do?canalId=1&modulo=menu&temaCanalId=1&tipoEmisionId=3&idRecurso=50123"
try:
html = urllib.urlopen(serie.url).read()
soup = BeautifulSoup(html, from_encoding='latin-1')
r = re.compile("nero:\t +(\w+)", re.DOTALL|re.UNICODE).findall(soup.find('div','titSerieEncabezado').text)
if r:
serie.genres.add(Genre.objects.get_or_create(name=r[0])[0])
for seasonelem in soup('ul','serieCap'):
c = seasonelem.get('id')[-1]
season = int(c) if c.isdigit() else 1
for cnt,episodeelem in enumerate(seasonelem("li")):
episode = Episode()
episode.serie = serie
episode.name = episodeelem.a.text
episode.number = cnt+1
episode_url = urllib.basejoin(self.BASE_URL, episodeelem.a.get("href"))
episode.save()
self.get_episode(episode, episode_url)
print "EPISODE S%02dE%02d: %s" % (episode.season, episode.number, episode.name.encode('utf8'))
except Exception,e:
print e
