def get_items(self, url, count=20): """ return return items from stream by url """ logging.info('start fetching url %s ', url_unquote(url)) req_param = {'r': 'n', 'n': count, 'client': 'scroll'} continuation = None while True: if continuation: req_param['c'] = continuation req_data = url_encode(req_param) feed_url = "{url}?{req_data}".format(url=url, req_data=req_data) req = urllib2.Request(feed_url, headers=self._header) try: resp = urllib2.urlopen(req).read() except (urllib2.HTTPError, urllib2.URLError) as exc: logging.error("Failed getting stream (%s) items: %s", url_unquote(url), exc) break feed_posts = json.loads(resp) for post in feed_posts['items']: yield post continuation = feed_posts.get('continuation', None) if not continuation: logging.info('end fetching url %s ', url_unquote(url)) break
def uri_params(uri): args = {} allstr = uri.split('?') if len(allstr) > 1: uri, argstr = allstr argstr = argstr.split('&') for arg in argstr: key, val = arg.split('=') # Do not encode the already-encoded attachments if key == 'attachment': key = url_unquote(key) val = url_unquote(val) # Attachment strings are encoded, # although Moin gives them as unicode -> # decode to unicode. This should be actually # done in the inline-bit in ViewDot-action, # but is done here to assure functionality # of existing wiki pages val = val.encode('raw_unicode_escape') val = unicode(val, config.charset) args.setdefault(key, []).append(val) else: args.setdefault(encode(url_unquote(key)), []).append(encode(url_unquote(val))) return uri, args
def get_items(self, url, count=20): """ return return items from stream by url """ logging.info('start fetching url %s ', url_unquote(url)) req_param = {'r': 'n', 'n': count, 'client': 'scroll'} continuation = None while True: if continuation: req_param['c'] = continuation req_data = url_encode(req_param) feed_url = "{url}?{req_data}".format(url=url, req_data=req_data) req = urllib2.Request(feed_url, headers=self._header) try: resp = urllib2.urlopen(req).read() except (urllib2.HTTPError, urllib2.URLError) as exc: logging.error("Failed getting stream items: %s", exc) break feed_posts = json.loads(resp) for post in feed_posts['items']: yield post continuation = feed_posts.get('continuation', None) if not continuation: logging.info('end fetching url %s ', url_unquote(url)) break
def get_file_info(self, shareinfo, fsid, secret=None): fi = FileInfo() extra_params = dict(bdstoken=shareinfo.bdstoken, sign=shareinfo.sign, timestamp=shareinfo.timestamp) post_form = { 'encrypt': '0', 'product': 'share', 'uk': shareinfo.uk, 'primaryid': shareinfo.share_id, 'fid_list': json.dumps([fsid]), } if self.session.cookies.get('BDCLND'): post_form['extra'] = '{"sekey":"%s"}' % (url_unquote( self.session.cookies['BDCLND'])), logger.debug(post_form, extra={'type': 'form', 'method': 'POST'}) url = BAIDUPAN_SERVER + 'sharedownload' while True: response = self._request('POST', url, extra_params=extra_params, post_data=post_form) if not response.ok: raise UnknownError _json = response.json() errno = _json['errno'] logger.debug(_json, extra={'type': 'json', 'method': 'POST'}) if errno == 0: fi.filename = _json['list'][0]['server_filename'] fi.path = os.path.dirname(_json['list'][0]['path']) fi.dlink = _json['list'][0]['dlink'] fi.parent_path = url_unquote( shareinfo.fileinfo[0]['parent_path'].encode('utf8')) break elif errno == -20: verify_params = self._handle_captcha(shareinfo.bdstoken) post_form.update(verify_params) response = self._request('POST', url, extra_params=extra_params, post_data=post_form) _json = response.json() errno = _json['errno'] continue elif errno == 116: raise DownloadError("The share file does not exist") else: raise UnknownError return fi
def get_json_for_policy(iam, policy_arn): """ Returns a policy in JSON format. """ policy = get_policy(iam, policy_arn).policy version = policy.get("default_version_id") if version: pv_response = get_policy_version( iam, policy_arn, version).policy_version pv = { version: { "is_default_version": pv_response.is_default_version, "document": url_unquote(pv_response.document), "create_date": pv_response.create_date, } } else: pv = {} return { "arn": policy_arn, "name": policy.policy_name, "description": policy.description, "create_date": policy.create_date, "update_date": policy.update_date, "policy_id": policy.policy_id, "path": policy.path, "default_version_id": version, "policy_versions": pv, }
def signed_headers(self): """ An ordered dictionary containing the signed header names and values. """ # See if the signed headers are listed in the query string signed_headers = self.query_parameters.get(_x_amz_signedheaders) if signed_headers is not None: signed_headers = url_unquote(signed_headers[0]) else: # Get this from the authentication header signed_headers = self.authorization_header_parameters[ _signedheaders] # Header names are separated by semicolons. parts = signed_headers.split(";") # Make sure the signed headers list is canonicalized. For security # reasons, we consider it an error if it isn't. canonicalized = sorted([sh.lower() for sh in parts]) if parts != canonicalized: raise AttributeError("SignedHeaders is not canonicalized: %r" % (signed_headers,)) # Allow iteration in-order. return OrderedDict([(header, self.headers[header]) for header in signed_headers.split(";")])
def access_key(self): """ The access key id used to sign the request. If the access key is not in the same credential scope as this request, an AttributeError exception is raised. """ credential = self.query_parameters.get(_x_amz_credential) if credential is not None: credential = url_unquote(credential[0]) else: credential = self.authorization_header_parameters.get(_credential) if credential is None: raise AttributeError("Credential was not passed in the request") try: key, scope = credential.split("/", 1) except ValueError: raise AttributeError("Invalid request credential: %r" % credential) if scope != self.credential_scope: raise AttributeError("Incorrect credential scope: %r (wanted %r)" % (scope, self.credential_scope)) return key
def get_file_info(self, shareinfo, fsid, secret=None): fi = FileInfo() extra_params = dict(bdstoken=shareinfo.bdstoken, sign=shareinfo.sign, timestamp=shareinfo.timestamp) post_form = { 'encrypt': '0', 'product': 'share', 'uk': shareinfo.uk, 'primaryid': shareinfo.share_id, 'fid_list': json.dumps([fsid]), } if self.session.cookies.get('BDCLND'): post_form['extra'] = '{"sekey":"%s"}' % ( url_unquote(self.session.cookies['BDCLND'])), logger.debug(post_form, extra={'type': 'form', 'method': 'POST'}) url = BAIDUPAN_SERVER + 'sharedownload' while True: response = self._request( 'POST', url, extra_params=extra_params, post_data=post_form) if not response.ok: raise UnknownError _json = response.json() errno = _json['errno'] logger.debug(_json, extra={'type': 'json', 'method': 'POST'}) if errno == 0: fi.filename = _json['list'][0]['server_filename'] fi.path = os.path.dirname(_json['list'][0]['path']) fi.dlink = _json['list'][0]['dlink'] fi.parent_path = url_unquote(shareinfo.fileinfo[0]['parent_path'].encode('utf8')) break elif errno == -20: verify_params = self._handle_captcha(shareinfo.bdstoken) post_form.update(verify_params) response = self._request( 'POST', url, extra_params=extra_params, post_data=post_form) _json = response.json() errno = _json['errno'] continue elif errno == 116: raise DownloadError("The share file does not exist") else: raise UnknownError return fi
def download_mp3_by_id(id, q=urllib2.urlopen): """download .mp3 of the given id""" r = re.search( '<a href="/music/top100/url\?q=(http%3A%2F%2Ffile.*\.mp3)', q('http://www.google.cn/music/top100/musicdownload?id=' + id).read() ) if r: return url_unquote(r.groups()[0]) else: return ''
def get_dlink(self, link, secret=None, fsid=None): info = FileInfo() js = self._get_js(link, secret) # Fix #15 self.session.get('http://d.pcs.baidu.com/rest/2.0/pcs/file?method=plantcookie&type=ett') self.pcsett = self.session.cookies.get('pcsett') logger.debug(self.pcsett, extra={'type': 'cookies', 'method': 'SetCookies'}) if info.match(js): # Fix #17 # Need to update fsid if we download it by providing fsid way if fsid: info.fid_list = fsid extra_params = dict(bdstoken=info.bdstoken, sign=info.sign, timestamp=info.timestamp) post_form = { 'encrypt': '0', 'product': 'share', 'uk': info.uk, 'primaryid': info.share_id, 'fid_list': '[{0}]'.format(info.fid_list), } if self.session.cookies.get('BDCLND'): post_form['extra'] = '{"sekey":"%s"}' % (url_unquote(self.session.cookies['BDCLND'])), logger.debug(post_form, extra={'type': 'form', 'method': 'POST'}) url = BAIDUPAN_SERVER + 'sharedownload' while True: response = self._request('POST', url, extra_params=extra_params, post_data=post_form) if not response.ok: raise UnknownError _json = response.json() errno = _json['errno'] logger.debug(_json, extra={'type': 'json', 'method': 'POST'}) if errno == 0: # FIXME: only support single file for now dlink = _json['list'][0]['dlink'] setattr(info, 'dlink', dlink) # Fix #17 # Need to update filename if we download it by providing fsid way if fsid: info.filename = _json['list'][0]['server_filename'] break elif errno == -20: verify_params = self._handle_captcha(info.bdstoken) post_form.update(verify_params) response = self._request('POST', url, extra_params=extra_params, post_data=post_form) _json = response.json() errno = _json['errno'] continue elif errno == 116: raise DownloadError("The share file does not exist") else: raise UnknownError return info
def unquote_url(request): form = URLUnquoteForm(request.POST) if form.is_valid(): cd = form.cleaned_data jdata = json.dumps({'result': url_unquote(cd['value'])}) return JSONResponse(jdata) else: jdata = json.dumps({'error': form.errors}) return JSONResponseBadRequest(jdata)
def _parse_attributes_v3(self, s): attributes = {} for pair_string in s.split(";"): try: tag, value = pair_string.split("=") attributes[url_unquote(tag)] = map(url_unquote, value.split(",")) except ValueError: raise FormatError("Invalid attributes string: " + s) return attributes
def _parse_record_v3(self, line): self._references_resolved = False fields = line[:-1].split('\t') if len(fields) != 9: raise FormatError, "Invalid number of fields (should be 9):\n" + line try: return GFF(seqid=url_unquote(fields[0]), source=url_unquote(fields[1]), type=url_unquote(fields[2]), start=int(fields[3]), end=int(fields[4]), score=parse_maybe_empty(fields[5], float), strand=parse_maybe_empty(fields[6]), phase=parse_maybe_empty(fields[7], int), attributes=self._parse_attributes_v3(fields[8])) except ValueError, e: raise FormatError, "GFF field format error: " + e.message
def main(argv): if len(argv) != 2: print("Usage: {0} <URL>".format(sys.argv[0])) sys.exit(1) url = argv[1] tracker_list = url_unquote(url).split('&tr=')[1:] tracker_list = [tracker.strip() for tracker in tracker_list] with open('tracker.txt', 'a') as fp: fp.write('# Date: {0}\n'.format(time.strftime('%Y-%m-%d %H:%M:%S')) + '\n'.join(tracker_list) + '\n')
def execute(macro, args): formatter = macro.formatter macro.request.page.formatter = formatter request = macro.request _ = request.getText # Import the plugin action to print out the graph html form dotviewer = wikiutil.importPlugin(request.cfg, 'action', 'ViewDot', 'execute') arglist = [x.strip() for x in args.split(',') if x] kw = {} for arg in arglist: data = arg.split('=') key = data[0] val = '='.join(data[1:]) if key in ['height', 'width']: kw[str(key)] = str(val) if not arglist: return "" uri, args = uri_params(arglist[0]) if not args: return "" pagename = url_unquote(uri) old_page = request.page old_values = request.values old_url = getattr(request, 'url', '') request.page = Page(request, pagename) req_url = url_construct(request, args, pagename) request.values = CombinedMultiDict([MultiDict(args)]) request.url = req_url request.write(u'<div class="viewdot">') dotviewer(pagename, request, **kw) request.page = old_page request.values = old_values del request.url if old_url: request.url = old_url return '<a href="%s&view=View" class="graph-link">[%s]</a>\n</div>' % \ (req_url, _('View'))
def _parse_attributes_v3(self, s): attributes = {} for pair_string in s.split(";"): if len(pair_string) == 0: continue try: tag, value = pair_string.split("=") attributes[url_unquote(tag)] = map(url_unquote, value.split(",")) except ValueError: print >> sys.stderr, "WARNING: Invalid attributes string: ", s # raise FormatError("Invalid attributes string: " + s) return attributes
def get(self): url=self.request.url r=re.search(r'^http://(.*)$', url) is_dev = os.environ['SERVER_SOFTWARE'][:3] if r and is_dev!='Dev': self.redirect('https://'+r.groups()[0], permanent=True) else: r=re.search(r'http.*(http.?://.*$)', url_unquote(url)) if r: self.redirect(r.groups()[0], permanent=True) else: self.response.set_status(400) self.response.out.write(url)
def get_dlink(self, link, secret=None): info = FileInfo() js = self._get_js(link, secret) if info.match(js): extra_params = dict(bdstoken=info.bdstoken, sign=info.sign, timestamp=info.timestamp) post_form = { 'encrypt': '0', 'product': 'share', 'uk': info.uk, 'primaryid': info.share_id, 'fid_list': '[{0}]'.format(info.fid_list), } if self.session.cookies.get('BDCLND'): post_form['extra'] = '{"sekey":"%s"}' % (url_unquote( self.session.cookies['BDCLND'])), logger.debug(post_form, extra={'type': 'form', 'method': 'POST'}) url = BAIDUPAN_SERVER + 'sharedownload' while True: response = self._request('POST', url, extra_params=extra_params, post_data=post_form) if not response.ok: raise UnknownError _json = response.json() errno = _json['errno'] logger.debug(_json, extra={'type': 'json', 'method': 'POST'}) if errno == 0: # FIXME: only support single file for now dlink = _json['list'][0]['dlink'] setattr(info, 'dlink', dlink) break elif errno == -20: verify_params = self._handle_captcha(info.bdstoken) post_form.update(verify_params) response = self._request('POST', url, extra_params=extra_params, post_data=post_form) _json = response.json() errno = _json['errno'] continue elif errno == 116: raise DownloadError("The share file does not exist") else: raise UnknownError return info
def _parse_attributes_v3(self, s): attributes = {} for pair_string in s.split(";"): if (len (pair_string) == 0): continue try: tag, value = pair_string.split("=") attributes[url_unquote(tag)] = map(url_unquote, value.split(",")) except ValueError: print >>sys.stderr, "WARNING: Invalid attributes string: ", s # raise FormatError("Invalid attributes string: " + s) return attributes
def get_links(text): query = SEARCH_QUERY.format(url_quote(text)) #print('Query: ', query) html = pg(requests.get(query).text) links = [a.attrib['href'] for a in html('.r')('a')] result = [] for link in links: # q={url} match_result = RE_LINK.match(link) if not match_result: continue link = match_result.groups()[0] result.append(url_unquote(link)) return result
def parse_ticket(cls, secret, ticket, ip, session): '''Parse cookie and check its signature. :var secret: shared secret used between multiple trusted peers to verify signature of cookie :var ticket: signed cookie content :var ip: originating client IP address - extracted from X Forwarded or Remote address iterms in HTTP header :var session: AuthKit session object content :return: tuple of parsed cookie content ''' if session is not None: if not session.has_key('authkit.cookie.user'): raise BadTicket('No authkit.cookie.user key exists in the ' 'session') if not session.has_key('authkit.cookie.user_data'): raise BadTicket('No authkit.cookie.user_data key exists in the ' 'session') decrypted_ticket = Encoder().decode_msg(ticket, secret) try: timestamp = int(decrypted_ticket[:8], 16) except ValueError as e: raise BadTicket('Timestamp is not a hex integer: %s' % e) try: userid, data = decrypted_ticket[8:].split('!', 1) except ValueError: raise BadTicket('userid is not followed by !') userid = url_unquote(userid) if '!' in data: tokens, user_data = data.split('!', 1) else: # @@: Is this the right order? tokens = '' user_data = data tokens = tokens.split(',') return timestamp, userid, tokens, user_data
def parse_ticket(cls, secret, ticket, ip, session): """Parse cookie and check its signature. :var secret: shared secret used between multiple trusted peers to verify signature of cookie :var ticket: signed cookie content :var ip: originating client IP address - extracted from X Forwarded or Remote address iterms in HTTP header :var session: AuthKit session object content :return: tuple of parsed cookie content """ if session is not None: if not session.has_key("authkit.cookie.user"): raise BadTicket("No authkit.cookie.user key exists in the " "session") if not session.has_key("authkit.cookie.user_data"): raise BadTicket("No authkit.cookie.user_data key exists in the " "session") decrypted_ticket = Encoder().decode_msg(ticket, secret) try: timestamp = int(decrypted_ticket[:8], 16) except ValueError as e: raise BadTicket("Timestamp is not a hex integer: %s" % e) try: userid, data = decrypted_ticket[8:].split("!", 1) except ValueError: raise BadTicket("userid is not followed by !") userid = url_unquote(userid) if "!" in data: tokens, user_data = data.split("!", 1) else: # @@: Is this the right order? tokens = "" user_data = data tokens = tokens.split(",") return timestamp, userid, tokens, user_data
def get_dlink(self, link, secret=None): info = FileInfo() js = self._get_js(link, secret) if info.match(js): extra_params = dict(bdstoken=info.bdstoken, sign=info.sign, timestamp=info.timestamp) post_form = { 'encrypt': '0', 'product': 'share', 'uk': info.uk, 'primaryid': info.share_id, 'fid_list': '[{0}]'.format(info.fid_list), } if self.session.cookies.get('BDCLND'): post_form['extra'] = '{"sekey":"%s"}' % (url_unquote(self.session.cookies['BDCLND'])), logger.debug(post_form, extra={'type': 'form', 'method': 'POST'}) url = BAIDUPAN_SERVER + 'sharedownload' while True: response = self._request('POST', url, extra_params=extra_params, post_data=post_form) if not response.ok: raise UnknownError _json = response.json() errno = _json['errno'] logger.debug(_json, extra={'type': 'json', 'method': 'POST'}) if errno == 0: # FIXME: only support single file for now dlink = _json['list'][0]['dlink'] setattr(info, 'dlink', dlink) break elif errno == -20: verify_params = self._handle_captcha(info.bdstoken) post_form.update(verify_params) response = self._request('POST', url, extra_params=extra_params, post_data=post_form) _json = response.json() errno = _json['errno'] continue elif errno == 116: raise DownloadError("The share file does not exist") else: raise UnknownError return info
def parse_ticket(secret, ticket, ip, digest_algo=DEFAULT_DIGEST): """ Parse the ticket, returning (timestamp, userid, tokens, user_data). If the ticket cannot be parsed, ``BadTicket`` will be raised with an explanation. """ if isinstance(digest_algo, str): # correct specification of digest from hashlib or fail digest_algo = getattr(hashlib, digest_algo) digest_hexa_size = digest_algo().digest_size * 2 ticket = ticket.strip('"') digest = ticket[:digest_hexa_size] try: timestamp = int(ticket[digest_hexa_size:digest_hexa_size + 8], 16) except ValueError as e: raise BadTicket('Timestamp is not a hex integer: %s' % e) try: userid, data = ticket[digest_hexa_size + 8:].split('!', 1) except ValueError: raise BadTicket('userid is not followed by !') userid = url_unquote(userid) if '!' in data: tokens, user_data = data.split('!', 1) else: # @@: Is this the right order? tokens = '' user_data = data expected = calculate_digest(ip, timestamp, secret, userid, tokens, user_data, digest_algo) if expected != digest: raise BadTicket('Digest signature is not correct', expected=(expected, digest)) tokens = tokens.split(',') return (timestamp, userid, tokens, user_data)
def parse_ticket(secret, ticket, ip, digest_algo=DEFAULT_DIGEST): """ Parse the ticket, returning (timestamp, userid, tokens, user_data). If the ticket cannot be parsed, ``BadTicket`` will be raised with an explanation. """ if isinstance(digest_algo, six.binary_type): # correct specification of digest from hashlib or fail digest_algo = getattr(hashlib, digest_algo) digest_hexa_size = digest_algo().digest_size * 2 ticket = ticket.strip('"') digest = ticket[:digest_hexa_size] try: timestamp = int(ticket[digest_hexa_size:digest_hexa_size + 8], 16) except ValueError as e: raise BadTicket('Timestamp is not a hex integer: %s' % e) try: userid, data = ticket[digest_hexa_size + 8:].split('!', 1) except ValueError: raise BadTicket('userid is not followed by !') userid = url_unquote(userid) if '!' in data: tokens, user_data = data.split('!', 1) else: # @@: Is this the right order? tokens = '' user_data = data expected = calculate_digest(ip, timestamp, secret, userid, tokens, user_data, digest_algo) if expected != digest: raise BadTicket('Digest signature is not correct', expected=(expected, digest)) tokens = tokens.split(',') return (timestamp, userid, tokens, user_data)
def __call__(self, value): try: # if the URL does not misuse the '%' character if not self.GENERIC_URL.search(value): # if the URL is only composed of valid characters if self.GENERIC_URL_VALID.match(value): # Then split up the URL into its components and check on # the scheme scheme = url_split_regex.match(value).group(2) # Clean up the scheme before we check it if scheme is not None: scheme = url_unquote(scheme).lower() # If the scheme really exists if scheme in self.allowed_schemes: # Then the URL is valid return value, None else: # else, for the possible case of abbreviated URLs with # ports, check to see if adding a valid scheme fixes # the problem (but only do this if it doesn't have # one already!) if value.find('://') < 0 and \ None in self.allowed_schemes: schemeToUse = self.prepend_scheme or 'http' prependTest = self.__call__( schemeToUse + '://' + value) # if the prepend test succeeded if prependTest[1] is None: # if prepending in the output is enabled if self.prepend_scheme: return prependTest else: # else return the original, # non-prepended value return value, None except: pass # else the URL is not valid return value, translate(self.message)
def __call__(self, value): try: # if the URL does not misuse the '%' character if not self.GENERIC_URL.search(value): # if the URL is only composed of valid characters if self.GENERIC_URL_VALID.match(value): # Then split up the URL into its components and check on # the scheme scheme = url_split_regex.match(value).group(2) # Clean up the scheme before we check it if scheme is not None: scheme = url_unquote(scheme).lower() # If the scheme really exists if scheme in self.allowed_schemes: # Then the URL is valid return value, None else: # else, for the possible case of abbreviated URLs with # ports, check to see if adding a valid scheme fixes # the problem (but only do this if it doesn't have # one already!) if value.find('://') < 0 and \ None in self.allowed_schemes: schemeToUse = self.prepend_scheme or 'http' prependTest = self.__call__(schemeToUse + '://' + value) # if the prepend test succeeded if prependTest[1] is None: # if prepending in the output is enabled if self.prepend_scheme: return prependTest else: # else return the original, # non-prepended value return value, None except: pass # else the URL is not valid return value, translate(self.message)
""" if isinstance(digest_algo, str): # correct specification of digest from hashlib or fail digest_algo = getattr(hashlib, digest_algo) digest_hexa_size = digest_algo().digest_size * 2 ticket = ticket.strip('"') digest = ticket[:digest_hexa_size] try: timestamp = int(ticket[digest_hexa_size:digest_hexa_size + 8], 16) except ValueError, e: raise BadTicket('Timestamp is not a hex integer: %s' % e) try: userid, data = ticket[digest_hexa_size + 8:].split('!', 1) except ValueError: raise BadTicket('userid is not followed by !') userid = url_unquote(userid) if '!' in data: tokens, user_data = data.split('!', 1) else: # @@: Is this the right order? tokens = '' user_data = data expected = calculate_digest(ip, timestamp, secret, userid, tokens, user_data, digest_algo) if expected != digest: raise BadTicket('Digest signature is not correct', expected=(expected, digest)) tokens = tokens.split(',')
def send(self, request, stream=False, verify=None, cert=None, proxies=None, timeout=None): """issue request""" data = url_unquote(request.url[len('data:'):]) if ',' not in data: raise InvalidURL('data URL missing comma') mime, content = data.split(',', 1) content = content.strip() base64 = False charset = None while ';' in mime: mime, encoding_spec = mime.rsplit(';', 1) encoding_spec = encoding_spec.strip() if encoding_spec == 'base64': base64 = True elif not encoding_spec.startswith('charset='): raise InvalidURL('unrecognized encoding parameter: %r' % encoding_spec) else: charset = encoding_spec[len('charset='):] try: if base64: content = a2b_base64(content) content_type = mime.strip() if charset: content_type += "; charset=" + charset response = Response() response.url = request.url response.headers['Date'] = formatdate(timeval=None, localtime=True) if request.method in ('GET', 'HEAD'): response.status_code = 200 response.headers['Content-Length'] = len(content) response.headers['Last-Modified'] = formatdate() response.headers['Content-Type'] = content_type if charset: response.encoding = charset response.raw = StringIO(str(content)) else: response.status_code = 405 response.headers['Status'] = '405 Method Not Allowed' except Exception: response.status_code = 500 response.headers['Status'] = '500 Internal Server Error' response.raw = StringIO(format_exc()) # context response.request = request response.connection = self # hooks response = dispatch_hook('response', request.hooks, response) # streaming if not stream: response.content return response
req2 = urllib2.Request(url2) #first_opener.addheaders = headers first_opener.open(req2) cookie.save(ignore_discard=True, ignore_expires=True) for item in cookie: print 'Name = ' + item.name print 'Value = ' + item.value #get SHARE_ID,SEKEY,SIGN,SHARE_UK share_id = info.share_id share_uk = info.uk sign = info.sign print 'sign is', sign sekey = info.sekey #暂时为None sekey = '{"sekey":"%s"}' % ( url_unquote(u'f9ciJq7HsoXeVjfiJjgrYn%2FppjvK7p8uETVQzEZBfxQ%3D')) fid_list = info.fid_list timestamp = info.timestamp bdstoken = info.bdstoken url5 = 'http://pan.baidu.com/api/sharedownload?' data_get_req = { 'sign': sign, 'timestamp': timestamp, 'bdstoken': bdstoken, 'type': 'batch' } data_get_req_encode = urllib.urlencode(data_get_req) print 'data_get', data_get_req url5 += data_get_req_encode print 'url5 is', url5
def decleanId(id): """Reverse cleanId.""" if id: id = id.replace('--', '\x00').replace('-', '%').replace('\x00', '-') return url_unquote(id) return ''
def send( self, request, stream=False, verify=None, cert=None, proxies=None, timeout=None ): """issue request""" fname = url_unquote(request.url[len('file://'):]) if not fname: raise InvalidURL('missing file name') if '/' not in fname: raise InvalidURL( 'hostname without filename (perhaps missing a /?)' ) host, fname = fname.split('/', 1) fname = self.resolve_host(host, fname) response = Response() response.url = request.url response.headers['Date'] = formatdate(timeval=None, localtime=True) try: if request.method in ('GET', 'HEAD'): statdata = stat(fname) etag = '"%s/%s/%s' \ % (statdata.st_dev, statdata.st_ino, statdata.st_mtime) if S_ISLNK(statdata.st_mode): # handle relative symlinks! target_file = abspath(readlink(fname)) response.status_code = 302 response.headers['Status'] = '302 Found' response.headers['Location'] = \ url_quote('file://' + target_file) elif S_ISDIR(statdata.st_mode): response.status_code = 200 response.headers['Status'] = '200 Ok' body = \ """<html><head><title>%s</title></head><body><ul>""" \ % fname for subfname in sorted(listdir(fname)): body += '<li><a href="file://' + \ url_quote(subfname) + '">' + \ html_escape(fname) + '</a></li>' body += '</body></html>' response.headers['ETag'] = 'W/' + etag response.raw = StringIO(body) elif S_ISREG(statdata.st_mode): response.status_code = 200 response.headers['Content-Length'] = statdata.st_size response.headers['Last-Modified'] = formatdate( timeval=statdata.st_mtime, localtime=True ) mt, enc = guess_mime_type(request.url, strict=False) if mt is None: mt = 'application/octet-stream' if enc is not None: response.headers['Content-Encoding'] = enc response.headers['Content-Type'] = mt response.headers['ETag'] = etag if request.method == 'GET': response.raw = open(fname, 'r') else: response.status_code = 500 response.headers['Status'] = '500 Internal Server Error' elif request.method == 'PUT': open(fname, 'w').write(request.body) # FIXME: Is this right? response.status_code = 200 response.headers['Status'] = '200 Ok' elif request.method == 'POST': if exists(fname): # FIXME: Is this right? response.status_code = 409 response.headers['Status'] = '409 Conflict' else: open(fname, 'w').write(request.body) elif request.method == 'DELETE': unlink(fname) response.status_code = 200 response.headers['Status'] = '200 Ok' else: response.status_code = 405 response.headers['Status'] = '405 Method Not Allowed' except OSError as e: if e.errno == errno.ENOENT: if request.method == 'DELETE': response.status_code = 410 response.headers['Status'] = '410 Gone' else: response.status_code = 404 response.headers['Status'] = '404 Not Found' elif e.errno == errno.EISDIR: response.status_code = 405 response.headers['Status'] = '405 Method Not Allowed' response.raw = StringIO('Cannot %r a directory...' % request.method) elif e.errno == errno.EACCES: response.status_code = 403 response.headers['Status'] = '403 Forbidden' else: response.status_code = 500 response.headers['Status'] = '500 Internal Server Error' response.raw = StringIO('OSError: ' + strerror(e.errno)) except Exception: response.status_code = 500 response.headers['Status'] = '500 Internal Server Error' response.raw = StringIO(format_exc()) # context response.request = request response.connection = self # hooks response = dispatch_hook('response', request.hooks, response) # streaming if not stream: response.content return response
def url_unquote_text(v, encoding='utf-8', errors='replace'): # pragma: no cover v = url_unquote(v) return v.decode(encoding, errors)
def construct_table(macro, pagelist, metakeys, legend='', checkAccess=True, styles={}): request = macro.request request.page.formatter = request.formatter days = dict() times = dict() orginalPage = request.page for page in pagelist: pageobj = Page(request, url_unquote(page)) request.page = pageobj request.formatter.page = pageobj metas = get_metas(request, page, metakeys, checkAccess=checkAccess) spans = days.setdefault(page, spanner.Spanner()) count = times.get(page, 0) for key in metakeys: values = metas[key] key = url_unquote(key) style = styles.get(key, {}) for value in values: value = value.strip().split() if len(value) != 2: continue start, end = value start = stringToTime(start) end = stringToTime(end) if not (start and end): continue count += 1 spans.addSpan(start, end) times[page] = count request.write(macro.formatter.linebreak() + u'<div class="metatable">' + macro.formatter.table(1)) request.write(macro.formatter.table_row(1, {'rowclass': 'meta_header'})) t_cell(macro, "User") t_cell(macro, "Used (days)") t_cell(macro, "Used (times)") request.write(macro.formatter.table_row(0)) tmp = list() for key in days: value = days[key] total = sum(map(lambda (start, end): end-start, value)) total /= float(60*60*24) count = times.get(key, 0) if count == 0: continue tmp.append((total, count, key)) tmp.sort(reverse=True) row = 0 for (days, times, user) in tmp: row = row + 1 if row % 2: request.write(macro.formatter.table_row(1, {'rowclass': 'metatable-odd-row'})) else: request.write(macro.formatter.table_row(1, {'rowclass': 'metatable-even-row'})) t_cell(macro, url_unquote(user), head=1) t_cell(macro, u"%.02f" % days) t_cell(macro, u"%d" % times) request.write(macro.formatter.table_row(0)) request.page = orginalPage request.formatter.page = orginalPage request.write(macro.formatter.table(0)) request.write(u'</div>')
#url5='http://pan.baidu.com/api/sharedownload?' url5 = 'http://eyun.baidu.com/api/sharedownload?' data_get_req5 = { 'sign': sign, 'timestamp': timestamp, 'bdstoken': bdstoken, 'cid': info.cid, 'web': 1, 'clienttype': 12 } data_get_req5_encode = urllib.urlencode(data_get_req5) print 'data_get', data_get_req5 url5 += data_get_req5_encode print 'url5 is', url5 extra = '{"sekey":"%s","cid":"%s"}' % (url_unquote(sekey), url_unquote(info.cid)) data_post5 = { 'encrypt': int('0'), 'extra': extra, 'fid_list': fid_list, 'product': "enterprise".strip('"'), 'primaryid': int(share_id), 'uk': share_uk, } data_post5_encode = urllib.urlencode(data_post5) while True: print 'data_post5', data_post5_encode req5 = urllib2.Request(url5, data_post5_encode) returned_info = first_opener.open(req5).read()
""" if isinstance(digest_algo, str): # correct specification of digest from hashlib or fail digest_algo = getattr(hashlib, digest_algo) digest_hexa_size = digest_algo().digest_size * 2 ticket = ticket.strip('"') digest = ticket[:digest_hexa_size] try: timestamp = int(ticket[digest_hexa_size : digest_hexa_size + 8], 16) except ValueError, e: raise BadTicket("Timestamp is not a hex integer: %s" % e) try: userid, data = ticket[digest_hexa_size + 8 :].split("!", 1) except ValueError: raise BadTicket("userid is not followed by !") userid = url_unquote(userid) if "!" in data: tokens, user_data = data.split("!", 1) else: # @@: Is this the right order? tokens = "" user_data = data expected = calculate_digest(ip, timestamp, secret, userid, tokens, user_data, digest_algo) if expected != digest: raise BadTicket("Digest signature is not correct", expected=(expected, digest)) tokens = tokens.split(",") return (timestamp, userid, tokens, user_data)
print 'fid_list is',fid_list #url5='http://pan.baidu.com/api/sharedownload?' url5='http://eyun.baidu.com/api/sharedownload?' data_get_req5={ 'sign':sign, 'timestamp':timestamp, 'bdstoken':bdstoken, 'cid':info.cid, 'web':1, 'clienttype':12 } data_get_req5_encode=urllib.urlencode(data_get_req5) print 'data_get',data_get_req5 url5+=data_get_req5_encode print 'url5 is',url5 extra='{"sekey":"%s","cid":"%s"}' % (url_unquote(sekey),url_unquote(info.cid)) data_post5={ 'encrypt':int('0'), 'extra':extra, 'fid_list':fid_list, 'product':"enterprise".strip('"'), 'primaryid':int(share_id), 'uk':share_uk, } data_post5_encode=urllib.urlencode(data_post5) while True: print 'data_post5',data_post5_encode req5=urllib2.Request(url5,data_post5_encode) returned_info=first_opener.open(req5).read() print returned_info
class Pan(object): headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)' ' Chrome/37.0.2062.120 Safari/537.36' } def __init__(self): self.baiduid = '' self.bduss = '' self.bdstoken = '' self.pcsett = '' self.session = requests.Session() self._load_cookies_from_file() self.cookies = self.session.cookies def _load_cookies_from_file(self): """Load cookies file if file exist.""" if os.access(global_config.cookies, os.F_OK): with open(global_config.cookies) as f: cookies = requests.utils.cookiejar_from_dict(pickle.load(f)) self.session.cookies = cookies # NOT SURE stoken is bdstoken! # self.token = self.session.cookies.get('STOKEN') self.baiduid = self.session.cookies.get('BAIDUID') self.bduss = self.session.cookies.get('BDUSS') return True return False def _save_img(self, img_url): """Download vcode image and save it to path of source code.""" r = self.session.get(img_url) data = r.content img_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), VCODE) with open(img_path, mode='wb') as fp: fp.write(data) print("Saved verification code to ", os.path.dirname(os.path.abspath(__file__))) def _try_open_img(self, vcode): _platform = platform.system().lower() if _platform == 'darwin': os.system('open ' + vcode) elif _platform == 'windows': os.system('start ' + vcode) def _handle_captcha(self, bdstoken=None): url = BAIDUPAN_SERVER + 'getcaptcha' d = {} extra_params = { 'prod': 'share', } if bdstoken: extra_params['bdstoken'] = bdstoken res = self._request(base_url=url, extra_params=extra_params) if res.ok: t = res.json() self._save_img(t['vcode_img']) self._try_open_img( os.path.join(os.path.dirname(os.path.abspath(__file__)), VCODE)) vcode_input = raw_input("Please input the captcha:\n") d['vcode_str'] = t['vcode_str'] d['vcode_input'] = vcode_input return d @staticmethod def _dict_to_utf8(dictionary): """Convert dictionary's value to utf-8""" if not isinstance(dictionary, dict): return for k, v in dictionary.items(): if isinstance(v, unicode): dictionary[k] = v.encode('utf-8') def _get_js(self, link, secret=None): """Get javascript code in html which contains share files info :param link: netdisk sharing link(publib or private). :type link: str :return str or None """ req = self.session.get(link, headers=self.headers) if 'init' in req.url: self.verify_passwd(req.url, secret) req = self.session.get(link) data = req.text js_pattern = re.compile( '<script\stype="text/javascript">!function\(\)([^<]+)</script>', re.DOTALL) js = re.findall(js_pattern, data) return js[0] or None def get_dlink(self, link, secret=None, fsid=None): info = ShareInfo() js = None try: js = self._get_js(link, secret) except IndexError, e: # Retry with new cookies js = self._get_js(link, secret) # Fix #15 self.session.get( 'http://d.pcs.baidu.com/rest/2.0/pcs/file?method=plantcookie&type=ett' ) self.pcsett = self.session.cookies.get('pcsett') logger.debug(self.pcsett, extra={ 'type': 'cookies', 'method': 'SetCookies' }) if info.match(js): # Fix #17 # Need to update fsid if we download it by providing fsid way if fsid: info.fid_list = fsid extra_params = dict(bdstoken=info.bdstoken, sign=info.sign, timestamp=info.timestamp) if info.sharepagetype == DLTYPE_MULTIPLE: extra_params['type'] = 'batch' post_form = { 'encrypt': '0', 'product': 'share', 'uk': info.uk, 'primaryid': info.share_id, 'fid_list': info.fid_list, } if self.session.cookies.get('BDCLND'): post_form['extra'] = '{"sekey":"%s"}' % (url_unquote( self.session.cookies['BDCLND'])), logger.debug(post_form, extra={'type': 'form', 'method': 'POST'}) url = BAIDUPAN_SERVER + 'sharedownload' while True: response = self._request('POST', url, extra_params=extra_params, post_data=post_form) if not response.ok: raise UnknownError _json = response.json() errno = _json['errno'] logger.debug(_json, extra={'type': 'json', 'method': 'POST'}) if errno == 0: if info.sharepagetype == DLTYPE_MULTIPLE: dlink = _json['dlink'] else: dlink = _json['list'][0]['dlink'] setattr(info, 'dlink', dlink) # Fix #17 # Need to update filename if we download it by providing # fsid way if fsid: info.filename = _json['list'][0]['server_filename'] break elif errno == -20: verify_params = self._handle_captcha(info.bdstoken) post_form.update(verify_params) response = self._request('POST', url, extra_params=extra_params, post_data=post_form) _json = response.json() errno = _json['errno'] continue elif errno == 116: raise DownloadError("The share file does not exist") else: raise UnknownError return info
def __onDownloadRequested(self, url): print "[Browser].__onDownloadRequested '%s'" %(url) filename = url_unquote(url).split("/")[-1] localfile = "%s/%s" %(config.plugins.WebBrowser.downloadpath.value, filename) downloadManager.AddJob(DownloadJob(url, localfile, filename)) self.session.open(MessageBox, _("Download started..."), type = MessageBox.TYPE_INFO, timeout = 3)
def parse_editform(request, form): r""" >>> from graphingwiki.editing import _doctest_request >>> request = _doctest_request() >>> parse_editform(request, {"Test-gwikiseparator-" : ["1", "2"], ":: " : ["a"]}) {'Test': ({}, {'a': ['1', '2']})} >>> request = _doctest_request({"Test" : {"meta" : {"a" : ["x"]}}}) >>> parse_editform(request, {"Test-gwikiseparator-a" : ["1", "2"], ":: a" : ["a"]}) {'Test': ({'a': ['x']}, {'a': ['1', '2']})} >>> request = _doctest_request({"Test" : {"meta" : {"a" : ["x"]}}}) >>> parse_editform(request, {"Test-gwikiseparator-a" : ["x"], ":: a" : [""]}) {'Test': ({'a': ['x']}, {'a': ['']})} >>> request = _doctest_request({"Test" : {"meta" : {"a" : ["1", "2"]}}}) >>> parse_editform(request, {"Test-gwikiseparator-a" : ["1"], ":: a" : ["a"]}) {'Test': ({'a': ['1', '2']}, {'a': ['1', '']})} """ _ = request.getText msgs = list() keys = dict() pages = dict() files = dict() for key in request.files: f = request.files[key] if not f.filename: continue fileobj = f.stream page, key = key.split(SEPARATOR, 1) files.setdefault(page, dict())[key] = (f.filename, fileobj) # Key changes for oldKey, newKeys in form.iteritems(): if not newKeys or not oldKey.startswith(':: '): continue newKey = newKeys[0].strip() oldKey = oldKey[3:].strip() if oldKey == newKey: continue keys[oldKey] = newKey # Value changes for pageAndKey, newValues in form.iteritems(): if pageAndKey.endswith("__filename__"): continue # At least the key 'save' may be there and should be ignored if not SEPARATOR in pageAndKey: continue # Form keys not autodecoded from utf-8 page, oldKey = pageAndKey.split(SEPARATOR, 1) # Decoding pagename page = url_unquote(page) newKey = keys.get(oldKey, oldKey) if not request.user.may.write(page): err = '%s: ' % page + _('You are not allowed to edit this page.') if not err in msgs: msgs.append(err) continue oldMeta, newMeta = pages.setdefault(page, (dict(), dict())) if oldKey: oldMetas = get_metas(request, page, [oldKey], abs_attach=False, includeGenerated=False) oldValues = oldMetas[oldKey] oldMeta.setdefault(oldKey, list()).extend(oldValues) if newKey != oldKey: # Remove the corresponding values newMeta.setdefault(oldKey, list()).extend([""] * len(oldValues)) if newKey: missing = 0 if oldKey: missing = len(oldValues) - len(newValues) newMeta.setdefault(newKey, list()).extend(newValues) newMeta.setdefault(newKey, list()).extend([""] * missing) # Prune for page, (oldMeta, newMeta) in list(pages.iteritems()): for key in list(newMeta): if key not in oldMeta: continue if oldMeta[key] != newMeta[key]: continue del oldMeta[key] del newMeta[key] if not (oldMeta or newMeta): del pages[page] for page in files: metas = pages.get(page, (dict(), dict()))[1] for key in files[page]: filename, descriptor = files[page][key] values = metas.get(key, list()) try: index = values.index(descriptor) values[index] = "[[attachment:%s]]" % filename except ValueError: values.append("[[attachment:%s]]" % filename) return pages, msgs, files
def parse_editform(request, form): r""" >>> from MoinMoin.metadata.util import doctest_request >>> request = doctest_request() >>> parse_editform(request, {"Test-gwikiseparator-" : ["1", "2"], ":: " : ["a"]}) {'Test': ({}, {'a': ['1', '2']})} >>> request = doctest_request({"Test" : {"meta" : {"a" : ["x"]}}}) >>> parse_editform(request, {"Test-gwikiseparator-a" : ["1", "2"], ":: a" : ["a"]}) {'Test': ({'a': ['x']}, {'a': ['1', '2']})} >>> request = doctest_request({"Test" : {"meta" : {"a" : ["x"]}}}) >>> parse_editform(request, {"Test-gwikiseparator-a" : ["x"], ":: a" : [""]}) {'Test': ({'a': ['x']}, {'a': ['']})} >>> request = doctest_request({"Test" : {"meta" : {"a" : ["1", "2"]}}}) >>> parse_editform(request, {"Test-gwikiseparator-a" : ["1"], ":: a" : ["a"]}) {'Test': ({'a': ['1', '2']}, {'a': ['1', '']})} """ _ = request.getText msgs = list() keys = dict() pages = dict() files = dict() for key in request.files: f = request.files[key] if not f.filename: continue fileobj = f.stream page, key = key.split(SEPARATOR, 1) files.setdefault(page, dict())[key] = (f.filename, fileobj) # Key changes for oldKey, newKeys in form.iteritems(): if not newKeys or not oldKey.startswith(':: '): continue newKey = newKeys[0].strip() oldKey = oldKey[3:].strip() if oldKey == newKey: continue keys[oldKey] = newKey # Value changes for pageAndKey, newValues in form.iteritems(): if pageAndKey.endswith("__filename__"): continue # At least the key 'save' may be there and should be ignored if not SEPARATOR in pageAndKey: continue # Form keys not autodecoded from utf-8 page, oldKey = pageAndKey.split(SEPARATOR, 1) # Decoding pagename page = url_unquote(page) newKey = keys.get(oldKey, oldKey) if not request.user.may.write(page): err = '%s: ' % page + _('You are not allowed to edit this page.') if not err in msgs: msgs.append(err) continue oldMeta, newMeta = pages.setdefault(page, (dict(), dict())) if oldKey: oldMetas = get_metas(request, page, [oldKey], abs_attach=False, includeGenerated=False) oldValues = oldMetas[oldKey] oldMeta.setdefault(oldKey, list()).extend(oldValues) if newKey != oldKey: # Remove the corresponding values newMeta.setdefault(oldKey, list()).extend([""] * len(oldValues)) if newKey: missing = 0 if oldKey: missing = len(oldValues) - len(newValues) newMeta.setdefault(newKey, list()).extend(newValues) newMeta.setdefault(newKey, list()).extend([""] * missing) # Prune for page, (oldMeta, newMeta) in list(pages.iteritems()): for key in list(newMeta): if key not in oldMeta: continue if oldMeta[key] != newMeta[key]: continue del oldMeta[key] del newMeta[key] if not (oldMeta or newMeta): del pages[page] for page in files: metas = pages.get(page, (dict(), dict()))[1] for key in files[page]: filename, descriptor = files[page][key] values = metas.get(key, list()) try: index = values.index(descriptor) values[index] = "[[attachment:%s]]" % filename except ValueError: values.append("[[attachment:%s]]" % filename) return pages, msgs, files
def send(self, request, stream=False, verify=None, cert=None, proxies=None, timeout=None): """issue request""" data = url_unquote(request.url[len('data:'):]) if ',' not in data: raise InvalidURL('data URL missing comma') mime, content = data.split(',', 1) content = content.strip() base64 = False charset = None while ';' in mime: mime, encoding_spec = mime.rsplit(';', 1) encoding_spec = encoding_spec.strip() if encoding_spec == 'base64': base64 = True elif not encoding_spec.startswith('charset='): raise InvalidURL( 'unrecognized encoding parameter: %r' % encoding_spec ) else: charset = encoding_spec[len('charset='):] try: if base64: content = a2b_base64(content) content_type = mime.strip() if charset: content_type += "; charset=" + charset response = Response() response.url = request.url response.headers['Date'] = formatdate(timeval=None, localtime=True) if request.method in ('GET', 'HEAD'): response.status_code = 200 response.headers['Content-Length'] = len(content) response.headers['Last-Modified'] = formatdate() response.headers['Content-Type'] = content_type if charset: response.encoding = charset response.raw = StringIO(str(content)) else: response.status_code = 405 response.headers['Status'] = '405 Method Not Allowed' except Exception: response.status_code = 500 response.headers['Status'] = '500 Internal Server Error' response.raw = StringIO(format_exc()) # context response.request = request response.connection = self # hooks response = dispatch_hook('response', request.hooks, response) # streaming if not stream: response.content return response
def url_unquote_text(v, encoding="utf-8", errors="replace"): # pragma: no cover v = url_unquote(v) return v.decode(encoding, errors)