def handle_starttag(self, tag, attrs) :
if tag == 'a' :
for attr in attrs :
if attr[0] == 'href' :
joined = urlparse.urljoin(self.baseurl, attr[1])
#print joined
if not joined.startswith('http') :
break
if joined.endswith('.torrent') :
# ok, this should be torrent
self.href = 1
self.cur_link = (joined, None)
else :
try :
link = urlopen(joined,referer=self.baseurl)
except Exception:
break
if is_torrent(link):
self.href = 1
m = None
if link.info().has_key('Content-Disposition') :
m = content_dispo_re.search(link.info()['Content-Disposition'])
if m :
self.cur_link = (joined, m.group('filename'))
else :
self.cur_link = (joined, None)
break
def prefetch(self):
TrackerLoader.prefetch(self)
try:
fd = urlopen(self.login_url,urlencode({'username': self.user,
'password': self.passwd}))
except Exception,why:
self.log.error('prefetch failed: %s' % str(why))
def _process(self):
media_list = List('MediaList')
m = self.table_item_cre.search(self.content)
while m:
self.log.debug('%s %s %s\n' % \
(m.group('title'),
m.group('publisher'),
m.group('category')))
try:
fd = urlopen(urlparse.urljoin(self.baseurl,m.group('link')))
detail = fd.read()
except Exception,why:
self.log.warn('unexpected error: %s\n' % str(why))
detail = ''
d = re.search(r'<a href="([^\?]+\?act=bt&func=download&id=\d+)">',detail,re.I)
if d:
link = urlparse.urljoin(self.baseurl,d.group(1))
attrs = {'title': m.group('title'),
'publisher': m.group('publisher'),
'link': link,
'type': m.group('category')}
media = GenericMedia('Media',attrs=attrs)
if self.filter(media):
media_list.append(media)
m = self.table_item_cre.search(self.content,
m.start(0)+len(m.group(0)))
def fetch(self):
try:
urlopener = AURLOpener(self.user,self.passwd)
urlopener.set_default()
web = urlopen(self.url)
meta = web.read()
except Exception,why:
self.log.error('torrent failed: %s\n' % str(why))
return
def is_movie(url):
try:
if type(url) in types.StringTypes:
doc = urlopen(url)
else:
doc = url
return doc.geturl().endswith(".avi") or doc.geturl().endswith(".ogm")
except Exception:
return 0
def prefetch(self):
TrackerLoader.prefetch(self)
try:
fd = urlopen(self.login_url,urlencode({'act': 'Login',
'CODE': '01',
'CookieDate': '1',
'UserName': self.user,
'PassWord': self.passwd}),
method='post')
except Exception,why:
self.log.error('prefetch failed: %s' % str(why))
def _process(self):
media_list = List('MediaList')
url_list = self.tracker.find_elements('Url')
media_name = self.tracker.media or 'GenericMedia'
media_tag = self.tracker.tag or 'Media'
for url in url_list:
content = url.urlopen(referer=self.tracker.url).read()
filter_list = url.find_elements('Filter')
if len(filter_list) == 0:
continue
self.log.finer(content+'\n')
self.log.finest(filter_list[0].content+'\n')
cre = re.compile(filter_list[0].content,filter_list[0].flags)
m = cre.search(content)
while m:
self.attributes = {}
self._update_attributes(m,url.url)
referer = url.url
for filter in filter_list[1:]:
fcre = re.compile(filter.content,filter.flags)
link = self.attributes.get('link','')
if not link:
break
temp_content = urlopen(link,referer=url.url).read()
self.log.finer(temp_content+'\n')
self.log.finest(filter.content+'\n')
n = fcre.search(temp_content)
if not n:
break
self._update_attributes(n,link)
if 'download' in n.groupdict().keys():
referer = link
keys = self.attributes.keys()
self.log.fine('filter: %s\n' % str(self.attributes))
if 'title' in keys and \
'download' in keys:
self.log.debug('filter: %s %s %s\n' % \
(misc.string(self.attributes.get('title','')),
misc.string(self.attributes.get('publisher','')),
misc.string(self.attributes.get('download',''))))
self.attributes['type'] = self.attributes.get('category',
self.attributes.get('type',''))
self.attributes['link'] = self.attributes.get('download','')
media = factory.create(media_name,media_tag,attrs=self.attributes)
if self.filter(media):
media.fetch(referer=referer)
media_list.append(media)
m = cre.search(content,
m.start(0)+len(m.group(0)))
return media_list
def urlopen(self,url=None,referer=None):
url = url or self.url
method = self.method or 'get'
data = {}
params = self.find_elements('Param')
for param in params:
data[param.name] = param.value
return urlopen(self.url,
data=urlencode(data),
user=self.user,
passwd=self.password,
method=method,
referer=referer)
def download_response(uri, title):
# adress, loc, path, param, query, frag = urlparse(uri)
torrent_path = policy(qpolicy.TORRENT_PATH)
torrent_file = os.path.join(torrent_path, asciiquote(title) + ".torrent")
if os.path.exists(torrent_file):
return
try:
fo = open(torrent_file, "w")
web = urlopen(uri, user, passwd)
fo.write(web.read())
fo.close()
except Exception:
pass
def process(self,tracker):
self.tracker = tracker
self.baseurl = tracker.url
self.content = ''
try:
if self.preload:
webfd = urlopen(self.baseurl)
self.content = webfd.read()
media_list = self._process()
except Exception,why:
self.log.error('tracker failed: %s\n' % str(why))
import traceback
traceback.print_exc()
media_list = List('MediaList')
def is_torrent(url):
try:
if type(url) in types.StringTypes:
doc = urlopen(url)
else:
doc = url
# basic check
doc_type = doc.info().getsubtype()
if doc_type.lower() == bt_mimetype.lower():
return 1
elif doc.geturl().endswith(".torrent"):
return 1
else:
return 0
except Exception:
return 0