def gotLoggedInPage(result): data,headers = result if re.search(r'(?i)<form[^>]* name="loginForm"', data) is not None: print 'WARNING: unable to log in: bad username or password' return print "logged in in Youtube" # Confirm age age_form = { 'next_url': '/', 'action_confirm': 'Confirm', } postdata = urlencode(age_form) d = getPage(self._AGE_URL, postdata=postdata, headers=std_headers) d.addCallback(gotAgeConfirmedPage)
def parse_description(self): def gotPage(x): self.debug("got device description from %r" % self.location) data, headers = x xml_data = None try: xml_data = utils.parse_xml(data, 'utf-8') except: self.warning("Invalid device description received from %r", self.location) import traceback self.debug(traceback.format_exc()) if xml_data is not None: tree = xml_data.getroot() major = tree.findtext('./{%s}specVersion/{%s}major' % (ns,ns)) minor = tree.findtext('./{%s}specVersion/{%s}minor' % (ns,ns)) try: self.upnp_version = '.'.join((major,minor)) except: self.upnp_version = 'n/a' try: self.urlbase = tree.findtext('./{%s}URLBase' % ns) except: import traceback self.debug(traceback.format_exc()) d = tree.find('./{%s}device' % ns) if d is not None: self.parse_device(d) # root device def gotError(failure, url): self.warning("error getting device description from %r", url) self.info(failure) utils.getPage(self.location).addCallbacks(gotPage, gotError, None, None, [self.location], None)
def gotLanguageSet(result): data,headers = result # No authentication to be performed if username is None: return # Log in login_form = { 'current_form': 'loginForm', 'next': '/', 'action_login': '******', 'username': username, 'password': password, } postdata = urlencode(login_form) d = getPage(self._LOGIN_URL, method='POST', postdata=postdata, headers=std_headers) d.addCallbacks(gotLoggedInPage, gotLoginError)
def update_data(self,rss_url,container=None,encoding="utf-8"): """ creates a deferred chain to retrieve the rdf file, parse and extract the metadata and reschedule itself """ def fail(f): self.info("fail %r", f) self.debug(f.getTraceback()) return f dfr = getPage(rss_url) dfr.addCallback(parse_xml, encoding=encoding) dfr.addErrback(fail) dfr.addCallback(self.parse_data,container) dfr.addErrback(fail) dfr.addBoth(self.queue_update,rss_url,container) return dfr
def callRemote(self, soapmethod, arguments): soapaction = soapmethod or self.soapaction if '#' not in soapaction: soapaction = '#'.join((self.namespace[1],soapaction)) self.action = soapaction.split('#')[1] self.info("callRemote %r %r %r %r", self.soapaction, soapmethod, self.namespace, self.action) headers = { 'content-type': 'text/xml ;charset="utf-8"', 'SOAPACTION': '"%s"' % soapaction,} if arguments.has_key('headers'): headers.update(arguments['headers']) del arguments['headers'] payload = soap_lite.build_soap_call("{%s}%s" % (self.namespace[1], self.action), arguments, encoding=None) self.info("callRemote soapaction: ", self.action,self.url) self.debug("callRemote payload: ", payload) def gotError(error, url): self.warning("error requesting url %r" % url) self.debug(error) try: tree = parse_xml(error.value.response) body = tree.find('{http://schemas.xmlsoap.org/soap/envelope/}Body') return failure.Failure(Exception("%s - %s" % (body.find('.//{urn:schemas-upnp-org:control-1-0}errorCode').text, body.find('.//{urn:schemas-upnp-org:control-1-0}errorDescription').text))) except: import traceback self.debug(traceback.format_exc()) return error return getPage(self.url, postdata=payload, method="POST", headers=headers ).addCallbacks(self._cbGotResult, gotError, None, None, [self.url], None)
def _real_extract(self, url): # Extract video id from URL mobj = re.match(self._VALID_URL, url) if mobj is None: self._downloader.trouble(u'ERROR: invalid URL: %s' % url) return video_id = mobj.group(2) # Downloader parameters best_quality = False format_param = None video_extension = None quality_index = 0 if self._downloader is not None: params = self._downloader.params format_param = params.get('format', None) if format_param == '0': format_param = self._available_formats[quality_index] best_quality = True video_extension = self._video_extensions.get(format_param, 'flv') # video info video_info_url = 'http://www.youtube.com/get_video_info?&video_id=%s&el=detailpage&ps=default&eurl=&gl=US&hl=en' % video_id if format_param is not None: video_info_url = '%s&fmt=%s' % (video_info_url, format_param) def gotPage(result, format_param, video_extension): video_info_webpage,headers = result # check format if (format_param == '22'): print "Check if HD video exists..." mobj = re.search(r'var isHDAvailable = true;', video_info_webpage) if mobj is None: print "No HD video -> switch back to SD" format_param = '18' else: print "...HD video OK!" # "t" param mobj = re.search(r'(?m)&token=([^&]+)(?:&|$)', video_info_webpage) if mobj is None: # Attempt to see if YouTube has issued an error message mobj = re.search(r'(?m)&reason=([^&]+)(?:&|$)', video_info_webpage) if mobj is None: self.to_stderr(u'ERROR: unable to extract "t" parameter') print video_info_webpage return [None] else: reason = unquote_plus(mobj.group(1)) self.to_stderr(u'ERROR: YouTube said: %s' % reason.decode('utf-8')) token = unquote(mobj.group(1)) video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token) if format_param is not None: video_real_url = '%s&fmt=%s' % (video_real_url, format_param) # uploader mobj = re.search(r'(?m)&author=([^&]+)(?:&|$)', video_info_webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract uploader nickname') return video_uploader = unquote(mobj.group(1)) # title mobj = re.search(r'(?m)&title=([^&]+)(?:&|$)', video_info_webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract video title') return video_title = unquote(mobj.group(1)) video_title = video_title.decode('utf-8') video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title) video_title = video_title.replace(os.sep, u'%') # simplified title simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title) simple_title = simple_title.strip(ur'_') # Return information return [{ 'id': video_id.decode('utf-8'), 'url': video_real_url.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'), 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), }] def gotError(error): print "Unable to process Youtube request: %s" % url print "Error: %s" % error return [None] d = getPage(video_info_url, headers=std_headers) d.addCallback(gotPage, format_param, video_extension) d.addErrback(gotError) return d
def parse_actions(self): def gotPage(x): #print "gotPage" #print x self.scpdXML, headers = x tree = utils.parse_xml(self.scpdXML, 'utf-8').getroot() ns = "urn:schemas-upnp-org:service-1-0" for action_node in tree.findall('.//{%s}action' % ns): name = action_node.findtext('{%s}name' % ns) arguments = [] for argument in action_node.findall('.//{%s}argument' % ns): arg_name = argument.findtext('{%s}name' % ns) arg_direction = argument.findtext('{%s}direction' % ns) arg_state_var = argument.findtext('{%s}relatedStateVariable' % ns) arguments.append(action.Argument(arg_name, arg_direction, arg_state_var)) self._actions[name] = action.Action(self, name, 'n/a', arguments) for var_node in tree.findall('.//{%s}stateVariable' % ns): send_events = var_node.attrib.get('sendEvents','yes') name = var_node.findtext('{%s}name' % ns) data_type = var_node.findtext('{%s}dataType' % ns) values = [] """ we need to ignore this, as there we don't get there our {urn:schemas-beebits-net:service-1-0}X_withVendorDefines attibute there """ for allowed in var_node.findall('.//{%s}allowedValue' % ns): values.append(allowed.text) instance = 0 self._variables.get(instance)[name] = variable.StateVariable(self, name, 'n/a', instance, send_events, data_type, values) """ we need to do this here, as there we don't get there our {urn:schemas-beebits-net:service-1-0}X_withVendorDefines attibute there """ self._variables.get(instance)[name].has_vendor_values = True #print 'service parse:', self, self.device self.detection_completed = True louie.send('Coherence.UPnP.Service.detection_completed', sender=self.device, device=self.device) self.info("send signal Coherence.UPnP.Service.detection_completed for %r" % self) """ if (self.last_time_updated == None): if( self.id.endswith('AVTransport') or self.id.endswith('RenderingControl')): louie.send('Coherence.UPnP.DeviceClient.Service.notified', sender=self.device, service=self) self.last_time_updated = time.time() """ def gotError(failure, url): self.warning('error requesting', url) self.info('failure', failure) louie.send('Coherence.UPnP.Service.detection_failed', self.device, device=self.device) #print 'getPage', self.get_scpd_url() utils.getPage(self.get_scpd_url()).addCallbacks(gotPage, gotError, None, None, [self.get_scpd_url()], None)