def get_items(self, url, count=20):
"""
return return items from stream by url
"""
logging.info('start fetching url %s ', url_unquote(url))
req_param = {'r': 'n', 'n': count, 'client': 'scroll'}
continuation = None
while True:
if continuation:
req_param['c'] = continuation
req_data = url_encode(req_param)
feed_url = "{url}?{req_data}".format(url=url, req_data=req_data)
req = urllib2.Request(feed_url, headers=self._header)
try:
resp = urllib2.urlopen(req).read()
except (urllib2.HTTPError, urllib2.URLError) as exc:
logging.error("Failed getting stream (%s) items: %s",
url_unquote(url), exc)
break
feed_posts = json.loads(resp)
for post in feed_posts['items']:
yield post
continuation = feed_posts.get('continuation', None)
if not continuation:
logging.info('end fetching url %s ', url_unquote(url))
break
def uri_params(uri):
args = {}
allstr = uri.split('?')
if len(allstr) > 1:
uri, argstr = allstr
argstr = argstr.split('&')
for arg in argstr:
key, val = arg.split('=')
# Do not encode the already-encoded attachments
if key == 'attachment':
key = url_unquote(key)
val = url_unquote(val)
# Attachment strings are encoded,
# although Moin gives them as unicode ->
# decode to unicode. This should be actually
# done in the inline-bit in ViewDot-action,
# but is done here to assure functionality
# of existing wiki pages
val = val.encode('raw_unicode_escape')
val = unicode(val, config.charset)
args.setdefault(key,
[]).append(val)
else:
args.setdefault(encode(url_unquote(key)),
[]).append(encode(url_unquote(val)))
return uri, args
def get_items(self, url, count=20):
"""
return return items from stream by url
"""
logging.info('start fetching url %s ', url_unquote(url))
req_param = {'r': 'n', 'n': count, 'client': 'scroll'}
continuation = None
while True:
if continuation:
req_param['c'] = continuation
req_data = url_encode(req_param)
feed_url = "{url}?{req_data}".format(url=url, req_data=req_data)
req = urllib2.Request(feed_url, headers=self._header)
try:
resp = urllib2.urlopen(req).read()
except (urllib2.HTTPError, urllib2.URLError) as exc:
logging.error("Failed getting stream items: %s", exc)
break
feed_posts = json.loads(resp)
for post in feed_posts['items']:
yield post
continuation = feed_posts.get('continuation', None)
if not continuation:
logging.info('end fetching url %s ', url_unquote(url))
break
def get_file_info(self, shareinfo, fsid, secret=None):
fi = FileInfo()
extra_params = dict(bdstoken=shareinfo.bdstoken,
sign=shareinfo.sign,
timestamp=shareinfo.timestamp)
post_form = {
'encrypt': '0',
'product': 'share',
'uk': shareinfo.uk,
'primaryid': shareinfo.share_id,
'fid_list': json.dumps([fsid]),
}
if self.session.cookies.get('BDCLND'):
post_form['extra'] = '{"sekey":"%s"}' % (url_unquote(
self.session.cookies['BDCLND'])),
logger.debug(post_form, extra={'type': 'form', 'method': 'POST'})
url = BAIDUPAN_SERVER + 'sharedownload'
while True:
response = self._request('POST',
url,
extra_params=extra_params,
post_data=post_form)
if not response.ok:
raise UnknownError
_json = response.json()
errno = _json['errno']
logger.debug(_json, extra={'type': 'json', 'method': 'POST'})
if errno == 0:
fi.filename = _json['list'][0]['server_filename']
fi.path = os.path.dirname(_json['list'][0]['path'])
fi.dlink = _json['list'][0]['dlink']
fi.parent_path = url_unquote(
shareinfo.fileinfo[0]['parent_path'].encode('utf8'))
break
elif errno == -20:
verify_params = self._handle_captcha(shareinfo.bdstoken)
post_form.update(verify_params)
response = self._request('POST',
url,
extra_params=extra_params,
post_data=post_form)
_json = response.json()
errno = _json['errno']
continue
elif errno == 116:
raise DownloadError("The share file does not exist")
else:
raise UnknownError
return fi
def get_json_for_policy(iam, policy_arn):
"""
Returns a policy in JSON format.
"""
policy = get_policy(iam, policy_arn).policy
version = policy.get("default_version_id")
if version:
pv_response = get_policy_version(
iam, policy_arn, version).policy_version
pv = {
version: {
"is_default_version": pv_response.is_default_version,
"document": url_unquote(pv_response.document),
"create_date": pv_response.create_date,
}
}
else:
pv = {}
return {
"arn": policy_arn,
"name": policy.policy_name,
"description": policy.description,
"create_date": policy.create_date,
"update_date": policy.update_date,
"policy_id": policy.policy_id,
"path": policy.path,
"default_version_id": version,
"policy_versions": pv,
}
def signed_headers(self):
"""
An ordered dictionary containing the signed header names and values.
"""
# See if the signed headers are listed in the query string
signed_headers = self.query_parameters.get(_x_amz_signedheaders)
if signed_headers is not None:
signed_headers = url_unquote(signed_headers[0])
else:
# Get this from the authentication header
signed_headers = self.authorization_header_parameters[
_signedheaders]
# Header names are separated by semicolons.
parts = signed_headers.split(";")
# Make sure the signed headers list is canonicalized. For security
# reasons, we consider it an error if it isn't.
canonicalized = sorted([sh.lower() for sh in parts])
if parts != canonicalized:
raise AttributeError("SignedHeaders is not canonicalized: %r" %
(signed_headers,))
# Allow iteration in-order.
return OrderedDict([(header, self.headers[header])
for header in signed_headers.split(";")])
def access_key(self):
"""
The access key id used to sign the request.
If the access key is not in the same credential scope as this request,
an AttributeError exception is raised.
"""
credential = self.query_parameters.get(_x_amz_credential)
if credential is not None:
credential = url_unquote(credential[0])
else:
credential = self.authorization_header_parameters.get(_credential)
if credential is None:
raise AttributeError("Credential was not passed in the request")
try:
key, scope = credential.split("/", 1)
except ValueError:
raise AttributeError("Invalid request credential: %r" % credential)
if scope != self.credential_scope:
raise AttributeError("Incorrect credential scope: %r (wanted %r)" %
(scope, self.credential_scope))
return key
def get_file_info(self, shareinfo, fsid, secret=None):
fi = FileInfo()
extra_params = dict(bdstoken=shareinfo.bdstoken,
sign=shareinfo.sign, timestamp=shareinfo.timestamp)
post_form = {
'encrypt': '0',
'product': 'share',
'uk': shareinfo.uk,
'primaryid': shareinfo.share_id,
'fid_list': json.dumps([fsid]),
}
if self.session.cookies.get('BDCLND'):
post_form['extra'] = '{"sekey":"%s"}' % (
url_unquote(self.session.cookies['BDCLND'])),
logger.debug(post_form, extra={'type': 'form', 'method': 'POST'})
url = BAIDUPAN_SERVER + 'sharedownload'
while True:
response = self._request(
'POST', url, extra_params=extra_params, post_data=post_form)
if not response.ok:
raise UnknownError
_json = response.json()
errno = _json['errno']
logger.debug(_json, extra={'type': 'json', 'method': 'POST'})
if errno == 0:
fi.filename = _json['list'][0]['server_filename']
fi.path = os.path.dirname(_json['list'][0]['path'])
fi.dlink = _json['list'][0]['dlink']
fi.parent_path = url_unquote(shareinfo.fileinfo[0]['parent_path'].encode('utf8'))
break
elif errno == -20:
verify_params = self._handle_captcha(shareinfo.bdstoken)
post_form.update(verify_params)
response = self._request(
'POST', url, extra_params=extra_params, post_data=post_form)
_json = response.json()
errno = _json['errno']
continue
elif errno == 116:
raise DownloadError("The share file does not exist")
else:
raise UnknownError
return fi
def download_mp3_by_id(id, q=urllib2.urlopen):
"""download .mp3 of the given id"""
r = re.search( '<a href="/music/top100/url\?q=(http%3A%2F%2Ffile.*\.mp3)',
q('http://www.google.cn/music/top100/musicdownload?id=' + id).read() )
if r:
return url_unquote(r.groups()[0])
else:
return ''
def get_dlink(self, link, secret=None, fsid=None):
info = FileInfo()
js = self._get_js(link, secret)
# Fix #15
self.session.get('http://d.pcs.baidu.com/rest/2.0/pcs/file?method=plantcookie&type=ett')
self.pcsett = self.session.cookies.get('pcsett')
logger.debug(self.pcsett, extra={'type': 'cookies', 'method': 'SetCookies'})
if info.match(js):
# Fix #17
# Need to update fsid if we download it by providing fsid way
if fsid:
info.fid_list = fsid
extra_params = dict(bdstoken=info.bdstoken, sign=info.sign, timestamp=info.timestamp)
post_form = {
'encrypt': '0',
'product': 'share',
'uk': info.uk,
'primaryid': info.share_id,
'fid_list': '[{0}]'.format(info.fid_list),
}
if self.session.cookies.get('BDCLND'):
post_form['extra'] = '{"sekey":"%s"}' % (url_unquote(self.session.cookies['BDCLND'])),
logger.debug(post_form, extra={'type': 'form', 'method': 'POST'})
url = BAIDUPAN_SERVER + 'sharedownload'
while True:
response = self._request('POST', url, extra_params=extra_params, post_data=post_form)
if not response.ok:
raise UnknownError
_json = response.json()
errno = _json['errno']
logger.debug(_json, extra={'type': 'json', 'method': 'POST'})
if errno == 0:
# FIXME: only support single file for now
dlink = _json['list'][0]['dlink']
setattr(info, 'dlink', dlink)
# Fix #17
# Need to update filename if we download it by providing fsid way
if fsid:
info.filename = _json['list'][0]['server_filename']
break
elif errno == -20:
verify_params = self._handle_captcha(info.bdstoken)
post_form.update(verify_params)
response = self._request('POST', url, extra_params=extra_params, post_data=post_form)
_json = response.json()
errno = _json['errno']
continue
elif errno == 116:
raise DownloadError("The share file does not exist")
else:
raise UnknownError
return info
def unquote_url(request):
form = URLUnquoteForm(request.POST)
if form.is_valid():
cd = form.cleaned_data
jdata = json.dumps({'result': url_unquote(cd['value'])})
return JSONResponse(jdata)
else:
jdata = json.dumps({'error': form.errors})
return JSONResponseBadRequest(jdata)
def _parse_attributes_v3(self, s):
attributes = {}
for pair_string in s.split(";"):
try:
tag, value = pair_string.split("=")
attributes[url_unquote(tag)] = map(url_unquote,
value.split(","))
except ValueError:
raise FormatError("Invalid attributes string: " + s)
return attributes
def _parse_record_v3(self, line):
self._references_resolved = False
fields = line[:-1].split('\t')
if len(fields) != 9:
raise FormatError, "Invalid number of fields (should be 9):\n" + line
try:
return GFF(seqid=url_unquote(fields[0]),
source=url_unquote(fields[1]),
type=url_unquote(fields[2]),
start=int(fields[3]),
end=int(fields[4]),
score=parse_maybe_empty(fields[5], float),
strand=parse_maybe_empty(fields[6]),
phase=parse_maybe_empty(fields[7], int),
attributes=self._parse_attributes_v3(fields[8]))
except ValueError, e:
raise FormatError, "GFF field format error: " + e.message
def main(argv):
if len(argv) != 2:
print("Usage: {0} <URL>".format(sys.argv[0]))
sys.exit(1)
url = argv[1]
tracker_list = url_unquote(url).split('&tr=')[1:]
tracker_list = [tracker.strip() for tracker in tracker_list]
with open('tracker.txt', 'a') as fp:
fp.write('# Date: {0}\n'.format(time.strftime('%Y-%m-%d %H:%M:%S')) +
'\n'.join(tracker_list) + '\n')
def _parse_record_v3(self, line):
self._references_resolved = False
fields = line[:-1].split('\t')
if len(fields) != 9:
raise FormatError, "Invalid number of fields (should be 9):\n" + line
try:
return GFF(seqid=url_unquote(fields[0]),
source=url_unquote(fields[1]),
type=url_unquote(fields[2]),
start=int(fields[3]),
end=int(fields[4]),
score=parse_maybe_empty(fields[5], float),
strand=parse_maybe_empty(fields[6]),
phase=parse_maybe_empty(fields[7], int),
attributes=self._parse_attributes_v3(fields[8]))
except ValueError, e:
raise FormatError, "GFF field format error: " + e.message
def execute(macro, args):
formatter = macro.formatter
macro.request.page.formatter = formatter
request = macro.request
_ = request.getText
# Import the plugin action to print out the graph html form
dotviewer = wikiutil.importPlugin(request.cfg,
'action', 'ViewDot',
'execute')
arglist = [x.strip() for x in args.split(',') if x]
kw = {}
for arg in arglist:
data = arg.split('=')
key = data[0]
val = '='.join(data[1:])
if key in ['height', 'width']:
kw[str(key)] = str(val)
if not arglist:
return ""
uri, args = uri_params(arglist[0])
if not args:
return ""
pagename = url_unquote(uri)
old_page = request.page
old_values = request.values
old_url = getattr(request, 'url', '')
request.page = Page(request, pagename)
req_url = url_construct(request, args, pagename)
request.values = CombinedMultiDict([MultiDict(args)])
request.url = req_url
request.write(u'<div class="viewdot">')
dotviewer(pagename, request, **kw)
request.page = old_page
request.values = old_values
del request.url
if old_url:
request.url = old_url
return '<a href="%s&view=View" class="graph-link">[%s]</a>\n</div>' % \
(req_url, _('View'))
def _parse_attributes_v3(self, s):
attributes = {}
for pair_string in s.split(";"):
if len(pair_string) == 0:
continue
try:
tag, value = pair_string.split("=")
attributes[url_unquote(tag)] = map(url_unquote, value.split(","))
except ValueError:
print >> sys.stderr, "WARNING: Invalid attributes string: ", s
# raise FormatError("Invalid attributes string: " + s)
return attributes
def get(self):
url=self.request.url
r=re.search(r'^http://(.*)$', url)
is_dev = os.environ['SERVER_SOFTWARE'][:3]
if r and is_dev!='Dev':
self.redirect('https://'+r.groups()[0], permanent=True)
else:
r=re.search(r'http.*(http.?://.*$)', url_unquote(url))
if r:
self.redirect(r.groups()[0], permanent=True)
else:
self.response.set_status(400)
self.response.out.write(url)
def get_dlink(self, link, secret=None):
info = FileInfo()
js = self._get_js(link, secret)
if info.match(js):
extra_params = dict(bdstoken=info.bdstoken,
sign=info.sign,
timestamp=info.timestamp)
post_form = {
'encrypt': '0',
'product': 'share',
'uk': info.uk,
'primaryid': info.share_id,
'fid_list': '[{0}]'.format(info.fid_list),
}
if self.session.cookies.get('BDCLND'):
post_form['extra'] = '{"sekey":"%s"}' % (url_unquote(
self.session.cookies['BDCLND'])),
logger.debug(post_form, extra={'type': 'form', 'method': 'POST'})
url = BAIDUPAN_SERVER + 'sharedownload'
while True:
response = self._request('POST',
url,
extra_params=extra_params,
post_data=post_form)
if not response.ok:
raise UnknownError
_json = response.json()
errno = _json['errno']
logger.debug(_json, extra={'type': 'json', 'method': 'POST'})
if errno == 0:
# FIXME: only support single file for now
dlink = _json['list'][0]['dlink']
setattr(info, 'dlink', dlink)
break
elif errno == -20:
verify_params = self._handle_captcha(info.bdstoken)
post_form.update(verify_params)
response = self._request('POST',
url,
extra_params=extra_params,
post_data=post_form)
_json = response.json()
errno = _json['errno']
continue
elif errno == 116:
raise DownloadError("The share file does not exist")
else:
raise UnknownError
return info
def _parse_attributes_v3(self, s):
attributes = {}
for pair_string in s.split(";"):
if (len (pair_string) == 0):
continue
try:
tag, value = pair_string.split("=")
attributes[url_unquote(tag)] = map(url_unquote,
value.split(","))
except ValueError:
print >>sys.stderr, "WARNING: Invalid attributes string: ", s
# raise FormatError("Invalid attributes string: " + s)
return attributes
def get_links(text):
query = SEARCH_QUERY.format(url_quote(text))
#print('Query: ', query)
html = pg(requests.get(query).text)
links = [a.attrib['href'] for a in html('.r')('a')]
result = []
for link in links:
# q={url}
match_result = RE_LINK.match(link)
if not match_result:
continue
link = match_result.groups()[0]
result.append(url_unquote(link))
return result
def parse_ticket(cls, secret, ticket, ip, session):
'''Parse cookie and check its signature.
:var secret: shared secret used between multiple trusted peers to
verify signature of cookie
:var ticket: signed cookie content
:var ip: originating client IP address - extracted from X Forwarded
or Remote address iterms in HTTP header
:var session: AuthKit session object content
:return: tuple of parsed cookie content
'''
if session is not None:
if not session.has_key('authkit.cookie.user'):
raise BadTicket('No authkit.cookie.user key exists in the '
'session')
if not session.has_key('authkit.cookie.user_data'):
raise BadTicket('No authkit.cookie.user_data key exists in the '
'session')
decrypted_ticket = Encoder().decode_msg(ticket, secret)
try:
timestamp = int(decrypted_ticket[:8], 16)
except ValueError as e:
raise BadTicket('Timestamp is not a hex integer: %s' % e)
try:
userid, data = decrypted_ticket[8:].split('!', 1)
except ValueError:
raise BadTicket('userid is not followed by !')
userid = url_unquote(userid)
if '!' in data:
tokens, user_data = data.split('!', 1)
else:
# @@: Is this the right order?
tokens = ''
user_data = data
tokens = tokens.split(',')
return timestamp, userid, tokens, user_data
def parse_ticket(cls, secret, ticket, ip, session):
"""Parse cookie and check its signature.
:var secret: shared secret used between multiple trusted peers to
verify signature of cookie
:var ticket: signed cookie content
:var ip: originating client IP address - extracted from X Forwarded
or Remote address iterms in HTTP header
:var session: AuthKit session object content
:return: tuple of parsed cookie content
"""
if session is not None:
if not session.has_key("authkit.cookie.user"):
raise BadTicket("No authkit.cookie.user key exists in the " "session")
if not session.has_key("authkit.cookie.user_data"):
raise BadTicket("No authkit.cookie.user_data key exists in the " "session")
decrypted_ticket = Encoder().decode_msg(ticket, secret)
try:
timestamp = int(decrypted_ticket[:8], 16)
except ValueError as e:
raise BadTicket("Timestamp is not a hex integer: %s" % e)
try:
userid, data = decrypted_ticket[8:].split("!", 1)
except ValueError:
raise BadTicket("userid is not followed by !")
userid = url_unquote(userid)
if "!" in data:
tokens, user_data = data.split("!", 1)
else:
# @@: Is this the right order?
tokens = ""
user_data = data
tokens = tokens.split(",")
return timestamp, userid, tokens, user_data
def get_dlink(self, link, secret=None):
info = FileInfo()
js = self._get_js(link, secret)
if info.match(js):
extra_params = dict(bdstoken=info.bdstoken, sign=info.sign, timestamp=info.timestamp)
post_form = {
'encrypt': '0',
'product': 'share',
'uk': info.uk,
'primaryid': info.share_id,
'fid_list': '[{0}]'.format(info.fid_list),
}
if self.session.cookies.get('BDCLND'):
post_form['extra'] = '{"sekey":"%s"}' % (url_unquote(self.session.cookies['BDCLND'])),
logger.debug(post_form, extra={'type': 'form', 'method': 'POST'})
url = BAIDUPAN_SERVER + 'sharedownload'
while True:
response = self._request('POST', url, extra_params=extra_params, post_data=post_form)
if not response.ok:
raise UnknownError
_json = response.json()
errno = _json['errno']
logger.debug(_json, extra={'type': 'json', 'method': 'POST'})
if errno == 0:
# FIXME: only support single file for now
dlink = _json['list'][0]['dlink']
setattr(info, 'dlink', dlink)
break
elif errno == -20:
verify_params = self._handle_captcha(info.bdstoken)
post_form.update(verify_params)
response = self._request('POST', url, extra_params=extra_params, post_data=post_form)
_json = response.json()
errno = _json['errno']
continue
elif errno == 116:
raise DownloadError("The share file does not exist")
else:
raise UnknownError
return info
def parse_ticket(secret, ticket, ip, digest_algo=DEFAULT_DIGEST):
"""
Parse the ticket, returning (timestamp, userid, tokens, user_data).
If the ticket cannot be parsed, ``BadTicket`` will be raised with
an explanation.
"""
if isinstance(digest_algo, str):
# correct specification of digest from hashlib or fail
digest_algo = getattr(hashlib, digest_algo)
digest_hexa_size = digest_algo().digest_size * 2
ticket = ticket.strip('"')
digest = ticket[:digest_hexa_size]
try:
timestamp = int(ticket[digest_hexa_size:digest_hexa_size + 8], 16)
except ValueError as e:
raise BadTicket('Timestamp is not a hex integer: %s' % e)
try:
userid, data = ticket[digest_hexa_size + 8:].split('!', 1)
except ValueError:
raise BadTicket('userid is not followed by !')
userid = url_unquote(userid)
if '!' in data:
tokens, user_data = data.split('!', 1)
else:
# @@: Is this the right order?
tokens = ''
user_data = data
expected = calculate_digest(ip, timestamp, secret,
userid, tokens, user_data,
digest_algo)
if expected != digest:
raise BadTicket('Digest signature is not correct',
expected=(expected, digest))
tokens = tokens.split(',')
return (timestamp, userid, tokens, user_data)
def parse_ticket(secret, ticket, ip, digest_algo=DEFAULT_DIGEST):
"""
Parse the ticket, returning (timestamp, userid, tokens, user_data).
If the ticket cannot be parsed, ``BadTicket`` will be raised with
an explanation.
"""
if isinstance(digest_algo, six.binary_type):
# correct specification of digest from hashlib or fail
digest_algo = getattr(hashlib, digest_algo)
digest_hexa_size = digest_algo().digest_size * 2
ticket = ticket.strip('"')
digest = ticket[:digest_hexa_size]
try:
timestamp = int(ticket[digest_hexa_size:digest_hexa_size + 8], 16)
except ValueError as e:
raise BadTicket('Timestamp is not a hex integer: %s' % e)
try:
userid, data = ticket[digest_hexa_size + 8:].split('!', 1)
except ValueError:
raise BadTicket('userid is not followed by !')
userid = url_unquote(userid)
if '!' in data:
tokens, user_data = data.split('!', 1)
else:
# @@: Is this the right order?
tokens = ''
user_data = data
expected = calculate_digest(ip, timestamp, secret,
userid, tokens, user_data,
digest_algo)
if expected != digest:
raise BadTicket('Digest signature is not correct',
expected=(expected, digest))
tokens = tokens.split(',')
return (timestamp, userid, tokens, user_data)
def __call__(self, value):
try:
# if the URL does not misuse the '%' character
if not self.GENERIC_URL.search(value):
# if the URL is only composed of valid characters
if self.GENERIC_URL_VALID.match(value):
# Then split up the URL into its components and check on
# the scheme
scheme = url_split_regex.match(value).group(2)
# Clean up the scheme before we check it
if scheme is not None:
scheme = url_unquote(scheme).lower()
# If the scheme really exists
if scheme in self.allowed_schemes:
# Then the URL is valid
return value, None
else:
# else, for the possible case of abbreviated URLs with
# ports, check to see if adding a valid scheme fixes
# the problem (but only do this if it doesn't have
# one already!)
if value.find('://') < 0 and \
None in self.allowed_schemes:
schemeToUse = self.prepend_scheme or 'http'
prependTest = self.__call__(
schemeToUse + '://' + value)
# if the prepend test succeeded
if prependTest[1] is None:
# if prepending in the output is enabled
if self.prepend_scheme:
return prependTest
else:
# else return the original,
# non-prepended value
return value, None
except:
pass
# else the URL is not valid
return value, translate(self.message)
def __call__(self, value):
try:
# if the URL does not misuse the '%' character
if not self.GENERIC_URL.search(value):
# if the URL is only composed of valid characters
if self.GENERIC_URL_VALID.match(value):
# Then split up the URL into its components and check on
# the scheme
scheme = url_split_regex.match(value).group(2)
# Clean up the scheme before we check it
if scheme is not None:
scheme = url_unquote(scheme).lower()
# If the scheme really exists
if scheme in self.allowed_schemes:
# Then the URL is valid
return value, None
else:
# else, for the possible case of abbreviated URLs with
# ports, check to see if adding a valid scheme fixes
# the problem (but only do this if it doesn't have
# one already!)
if value.find('://') < 0 and \
None in self.allowed_schemes:
schemeToUse = self.prepend_scheme or 'http'
prependTest = self.__call__(schemeToUse + '://' +
value)
# if the prepend test succeeded
if prependTest[1] is None:
# if prepending in the output is enabled
if self.prepend_scheme:
return prependTest
else:
# else return the original,
# non-prepended value
return value, None
except:
pass
# else the URL is not valid
return value, translate(self.message)
"""
if isinstance(digest_algo, str):
# correct specification of digest from hashlib or fail
digest_algo = getattr(hashlib, digest_algo)
digest_hexa_size = digest_algo().digest_size * 2
ticket = ticket.strip('"')
digest = ticket[:digest_hexa_size]
try:
timestamp = int(ticket[digest_hexa_size:digest_hexa_size + 8], 16)
except ValueError, e:
raise BadTicket('Timestamp is not a hex integer: %s' % e)
try:
userid, data = ticket[digest_hexa_size + 8:].split('!', 1)
except ValueError:
raise BadTicket('userid is not followed by !')
userid = url_unquote(userid)
if '!' in data:
tokens, user_data = data.split('!', 1)
else:
# @@: Is this the right order?
tokens = ''
user_data = data
expected = calculate_digest(ip, timestamp, secret, userid, tokens,
user_data, digest_algo)
if expected != digest:
raise BadTicket('Digest signature is not correct',
expected=(expected, digest))
tokens = tokens.split(',')
def send(self,
request,
stream=False,
verify=None,
cert=None,
proxies=None,
timeout=None):
"""issue request"""
data = url_unquote(request.url[len('data:'):])
if ',' not in data:
raise InvalidURL('data URL missing comma')
mime, content = data.split(',', 1)
content = content.strip()
base64 = False
charset = None
while ';' in mime:
mime, encoding_spec = mime.rsplit(';', 1)
encoding_spec = encoding_spec.strip()
if encoding_spec == 'base64':
base64 = True
elif not encoding_spec.startswith('charset='):
raise InvalidURL('unrecognized encoding parameter: %r' %
encoding_spec)
else:
charset = encoding_spec[len('charset='):]
try:
if base64:
content = a2b_base64(content)
content_type = mime.strip()
if charset:
content_type += "; charset=" + charset
response = Response()
response.url = request.url
response.headers['Date'] = formatdate(timeval=None, localtime=True)
if request.method in ('GET', 'HEAD'):
response.status_code = 200
response.headers['Content-Length'] = len(content)
response.headers['Last-Modified'] = formatdate()
response.headers['Content-Type'] = content_type
if charset:
response.encoding = charset
response.raw = StringIO(str(content))
else:
response.status_code = 405
response.headers['Status'] = '405 Method Not Allowed'
except Exception:
response.status_code = 500
response.headers['Status'] = '500 Internal Server Error'
response.raw = StringIO(format_exc())
# context
response.request = request
response.connection = self
# hooks
response = dispatch_hook('response', request.hooks, response)
# streaming
if not stream:
response.content
return response
req2 = urllib2.Request(url2)
#first_opener.addheaders = headers
first_opener.open(req2)
cookie.save(ignore_discard=True, ignore_expires=True)
for item in cookie:
print 'Name = ' + item.name
print 'Value = ' + item.value
#get SHARE_ID,SEKEY,SIGN,SHARE_UK
share_id = info.share_id
share_uk = info.uk
sign = info.sign
print 'sign is', sign
sekey = info.sekey #暂时为None
sekey = '{"sekey":"%s"}' % (
url_unquote(u'f9ciJq7HsoXeVjfiJjgrYn%2FppjvK7p8uETVQzEZBfxQ%3D'))
fid_list = info.fid_list
timestamp = info.timestamp
bdstoken = info.bdstoken
url5 = 'http://pan.baidu.com/api/sharedownload?'
data_get_req = {
'sign': sign,
'timestamp': timestamp,
'bdstoken': bdstoken,
'type': 'batch'
}
data_get_req_encode = urllib.urlencode(data_get_req)
print 'data_get', data_get_req
url5 += data_get_req_encode
print 'url5 is', url5
def decleanId(id):
"""Reverse cleanId."""
if id:
id = id.replace('--', '\x00').replace('-', '%').replace('\x00', '-')
return url_unquote(id)
return ''
def send(
self, request, stream=False, verify=None, cert=None, proxies=None,
timeout=None
):
"""issue request"""
fname = url_unquote(request.url[len('file://'):])
if not fname:
raise InvalidURL('missing file name')
if '/' not in fname:
raise InvalidURL(
'hostname without filename (perhaps missing a /?)'
)
host, fname = fname.split('/', 1)
fname = self.resolve_host(host, fname)
response = Response()
response.url = request.url
response.headers['Date'] = formatdate(timeval=None, localtime=True)
try:
if request.method in ('GET', 'HEAD'):
statdata = stat(fname)
etag = '"%s/%s/%s' \
% (statdata.st_dev, statdata.st_ino, statdata.st_mtime)
if S_ISLNK(statdata.st_mode):
# handle relative symlinks!
target_file = abspath(readlink(fname))
response.status_code = 302
response.headers['Status'] = '302 Found'
response.headers['Location'] = \
url_quote('file://' + target_file)
elif S_ISDIR(statdata.st_mode):
response.status_code = 200
response.headers['Status'] = '200 Ok'
body = \
"""<html><head><title>%s</title></head><body><ul>""" \
% fname
for subfname in sorted(listdir(fname)):
body += '<li><a href="file://' + \
url_quote(subfname) + '">' + \
html_escape(fname) + '</a></li>'
body += '</body></html>'
response.headers['ETag'] = 'W/' + etag
response.raw = StringIO(body)
elif S_ISREG(statdata.st_mode):
response.status_code = 200
response.headers['Content-Length'] = statdata.st_size
response.headers['Last-Modified'] = formatdate(
timeval=statdata.st_mtime,
localtime=True
)
mt, enc = guess_mime_type(request.url, strict=False)
if mt is None:
mt = 'application/octet-stream'
if enc is not None:
response.headers['Content-Encoding'] = enc
response.headers['Content-Type'] = mt
response.headers['ETag'] = etag
if request.method == 'GET':
response.raw = open(fname, 'r')
else:
response.status_code = 500
response.headers['Status'] = '500 Internal Server Error'
elif request.method == 'PUT':
open(fname, 'w').write(request.body) # FIXME: Is this right?
response.status_code = 200
response.headers['Status'] = '200 Ok'
elif request.method == 'POST':
if exists(fname): # FIXME: Is this right?
response.status_code = 409
response.headers['Status'] = '409 Conflict'
else:
open(fname, 'w').write(request.body)
elif request.method == 'DELETE':
unlink(fname)
response.status_code = 200
response.headers['Status'] = '200 Ok'
else:
response.status_code = 405
response.headers['Status'] = '405 Method Not Allowed'
except OSError as e:
if e.errno == errno.ENOENT:
if request.method == 'DELETE':
response.status_code = 410
response.headers['Status'] = '410 Gone'
else:
response.status_code = 404
response.headers['Status'] = '404 Not Found'
elif e.errno == errno.EISDIR:
response.status_code = 405
response.headers['Status'] = '405 Method Not Allowed'
response.raw = StringIO('Cannot %r a directory...'
% request.method)
elif e.errno == errno.EACCES:
response.status_code = 403
response.headers['Status'] = '403 Forbidden'
else:
response.status_code = 500
response.headers['Status'] = '500 Internal Server Error'
response.raw = StringIO('OSError: ' + strerror(e.errno))
except Exception:
response.status_code = 500
response.headers['Status'] = '500 Internal Server Error'
response.raw = StringIO(format_exc())
# context
response.request = request
response.connection = self
# hooks
response = dispatch_hook('response', request.hooks, response)
# streaming
if not stream:
response.content
return response
def url_unquote_text(v, encoding='utf-8', errors='replace'): # pragma: no cover
v = url_unquote(v)
return v.decode(encoding, errors)
def construct_table(macro, pagelist, metakeys,
legend='', checkAccess=True, styles={}):
request = macro.request
request.page.formatter = request.formatter
days = dict()
times = dict()
orginalPage = request.page
for page in pagelist:
pageobj = Page(request, url_unquote(page))
request.page = pageobj
request.formatter.page = pageobj
metas = get_metas(request, page,
metakeys, checkAccess=checkAccess)
spans = days.setdefault(page, spanner.Spanner())
count = times.get(page, 0)
for key in metakeys:
values = metas[key]
key = url_unquote(key)
style = styles.get(key, {})
for value in values:
value = value.strip().split()
if len(value) != 2:
continue
start, end = value
start = stringToTime(start)
end = stringToTime(end)
if not (start and end):
continue
count += 1
spans.addSpan(start, end)
times[page] = count
request.write(macro.formatter.linebreak() +
u'<div class="metatable">' +
macro.formatter.table(1))
request.write(macro.formatter.table_row(1, {'rowclass':
'meta_header'}))
t_cell(macro, "User")
t_cell(macro, "Used (days)")
t_cell(macro, "Used (times)")
request.write(macro.formatter.table_row(0))
tmp = list()
for key in days:
value = days[key]
total = sum(map(lambda (start, end): end-start, value))
total /= float(60*60*24)
count = times.get(key, 0)
if count == 0:
continue
tmp.append((total, count, key))
tmp.sort(reverse=True)
row = 0
for (days, times, user) in tmp:
row = row + 1
if row % 2:
request.write(macro.formatter.table_row(1, {'rowclass':
'metatable-odd-row'}))
else:
request.write(macro.formatter.table_row(1, {'rowclass':
'metatable-even-row'}))
t_cell(macro, url_unquote(user), head=1)
t_cell(macro, u"%.02f" % days)
t_cell(macro, u"%d" % times)
request.write(macro.formatter.table_row(0))
request.page = orginalPage
request.formatter.page = orginalPage
request.write(macro.formatter.table(0))
request.write(u'</div>')
def url_unquote_text(v,
encoding='utf-8',
errors='replace'): # pragma: no cover
v = url_unquote(v)
return v.decode(encoding, errors)
#url5='http://pan.baidu.com/api/sharedownload?'
url5 = 'http://eyun.baidu.com/api/sharedownload?'
data_get_req5 = {
'sign': sign,
'timestamp': timestamp,
'bdstoken': bdstoken,
'cid': info.cid,
'web': 1,
'clienttype': 12
}
data_get_req5_encode = urllib.urlencode(data_get_req5)
print 'data_get', data_get_req5
url5 += data_get_req5_encode
print 'url5 is', url5
extra = '{"sekey":"%s","cid":"%s"}' % (url_unquote(sekey),
url_unquote(info.cid))
data_post5 = {
'encrypt': int('0'),
'extra': extra,
'fid_list': fid_list,
'product': "enterprise".strip('"'),
'primaryid': int(share_id),
'uk': share_uk,
}
data_post5_encode = urllib.urlencode(data_post5)
while True:
print 'data_post5', data_post5_encode
req5 = urllib2.Request(url5, data_post5_encode)
returned_info = first_opener.open(req5).read()
"""
if isinstance(digest_algo, str):
# correct specification of digest from hashlib or fail
digest_algo = getattr(hashlib, digest_algo)
digest_hexa_size = digest_algo().digest_size * 2
ticket = ticket.strip('"')
digest = ticket[:digest_hexa_size]
try:
timestamp = int(ticket[digest_hexa_size : digest_hexa_size + 8], 16)
except ValueError, e:
raise BadTicket("Timestamp is not a hex integer: %s" % e)
try:
userid, data = ticket[digest_hexa_size + 8 :].split("!", 1)
except ValueError:
raise BadTicket("userid is not followed by !")
userid = url_unquote(userid)
if "!" in data:
tokens, user_data = data.split("!", 1)
else:
# @@: Is this the right order?
tokens = ""
user_data = data
expected = calculate_digest(ip, timestamp, secret, userid, tokens, user_data, digest_algo)
if expected != digest:
raise BadTicket("Digest signature is not correct", expected=(expected, digest))
tokens = tokens.split(",")
return (timestamp, userid, tokens, user_data)
print 'fid_list is',fid_list
#url5='http://pan.baidu.com/api/sharedownload?'
url5='http://eyun.baidu.com/api/sharedownload?'
data_get_req5={
'sign':sign,
'timestamp':timestamp,
'bdstoken':bdstoken,
'cid':info.cid,
'web':1,
'clienttype':12
}
data_get_req5_encode=urllib.urlencode(data_get_req5)
print 'data_get',data_get_req5
url5+=data_get_req5_encode
print 'url5 is',url5
extra='{"sekey":"%s","cid":"%s"}' % (url_unquote(sekey),url_unquote(info.cid))
data_post5={
'encrypt':int('0'),
'extra':extra,
'fid_list':fid_list,
'product':"enterprise".strip('"'),
'primaryid':int(share_id),
'uk':share_uk,
}
data_post5_encode=urllib.urlencode(data_post5)
while True:
print 'data_post5',data_post5_encode
req5=urllib2.Request(url5,data_post5_encode)
returned_info=first_opener.open(req5).read()
print returned_info
class Pan(object):
headers = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/37.0.2062.120 Safari/537.36'
}
def __init__(self):
self.baiduid = ''
self.bduss = ''
self.bdstoken = ''
self.pcsett = ''
self.session = requests.Session()
self._load_cookies_from_file()
self.cookies = self.session.cookies
def _load_cookies_from_file(self):
"""Load cookies file if file exist."""
if os.access(global_config.cookies, os.F_OK):
with open(global_config.cookies) as f:
cookies = requests.utils.cookiejar_from_dict(pickle.load(f))
self.session.cookies = cookies
# NOT SURE stoken is bdstoken!
# self.token = self.session.cookies.get('STOKEN')
self.baiduid = self.session.cookies.get('BAIDUID')
self.bduss = self.session.cookies.get('BDUSS')
return True
return False
def _save_img(self, img_url):
"""Download vcode image and save it to path of source code."""
r = self.session.get(img_url)
data = r.content
img_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
VCODE)
with open(img_path, mode='wb') as fp:
fp.write(data)
print("Saved verification code to ",
os.path.dirname(os.path.abspath(__file__)))
def _try_open_img(self, vcode):
_platform = platform.system().lower()
if _platform == 'darwin':
os.system('open ' + vcode)
elif _platform == 'windows':
os.system('start ' + vcode)
def _handle_captcha(self, bdstoken=None):
url = BAIDUPAN_SERVER + 'getcaptcha'
d = {}
extra_params = {
'prod': 'share',
}
if bdstoken:
extra_params['bdstoken'] = bdstoken
res = self._request(base_url=url, extra_params=extra_params)
if res.ok:
t = res.json()
self._save_img(t['vcode_img'])
self._try_open_img(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
VCODE))
vcode_input = raw_input("Please input the captcha:\n")
d['vcode_str'] = t['vcode_str']
d['vcode_input'] = vcode_input
return d
@staticmethod
def _dict_to_utf8(dictionary):
"""Convert dictionary's value to utf-8"""
if not isinstance(dictionary, dict):
return
for k, v in dictionary.items():
if isinstance(v, unicode):
dictionary[k] = v.encode('utf-8')
def _get_js(self, link, secret=None):
"""Get javascript code in html which contains share files info
:param link: netdisk sharing link(publib or private).
:type link: str
:return str or None
"""
req = self.session.get(link, headers=self.headers)
if 'init' in req.url:
self.verify_passwd(req.url, secret)
req = self.session.get(link)
data = req.text
js_pattern = re.compile(
'<script\stype="text/javascript">!function\(\)([^<]+)</script>',
re.DOTALL)
js = re.findall(js_pattern, data)
return js[0] or None
def get_dlink(self, link, secret=None, fsid=None):
info = ShareInfo()
js = None
try:
js = self._get_js(link, secret)
except IndexError, e:
# Retry with new cookies
js = self._get_js(link, secret)
# Fix #15
self.session.get(
'http://d.pcs.baidu.com/rest/2.0/pcs/file?method=plantcookie&type=ett'
)
self.pcsett = self.session.cookies.get('pcsett')
logger.debug(self.pcsett,
extra={
'type': 'cookies',
'method': 'SetCookies'
})
if info.match(js):
# Fix #17
# Need to update fsid if we download it by providing fsid way
if fsid:
info.fid_list = fsid
extra_params = dict(bdstoken=info.bdstoken,
sign=info.sign,
timestamp=info.timestamp)
if info.sharepagetype == DLTYPE_MULTIPLE:
extra_params['type'] = 'batch'
post_form = {
'encrypt': '0',
'product': 'share',
'uk': info.uk,
'primaryid': info.share_id,
'fid_list': info.fid_list,
}
if self.session.cookies.get('BDCLND'):
post_form['extra'] = '{"sekey":"%s"}' % (url_unquote(
self.session.cookies['BDCLND'])),
logger.debug(post_form, extra={'type': 'form', 'method': 'POST'})
url = BAIDUPAN_SERVER + 'sharedownload'
while True:
response = self._request('POST',
url,
extra_params=extra_params,
post_data=post_form)
if not response.ok:
raise UnknownError
_json = response.json()
errno = _json['errno']
logger.debug(_json, extra={'type': 'json', 'method': 'POST'})
if errno == 0:
if info.sharepagetype == DLTYPE_MULTIPLE:
dlink = _json['dlink']
else:
dlink = _json['list'][0]['dlink']
setattr(info, 'dlink', dlink)
# Fix #17
# Need to update filename if we download it by providing
# fsid way
if fsid:
info.filename = _json['list'][0]['server_filename']
break
elif errno == -20:
verify_params = self._handle_captcha(info.bdstoken)
post_form.update(verify_params)
response = self._request('POST',
url,
extra_params=extra_params,
post_data=post_form)
_json = response.json()
errno = _json['errno']
continue
elif errno == 116:
raise DownloadError("The share file does not exist")
else:
raise UnknownError
return info
def __onDownloadRequested(self, url):
print "[Browser].__onDownloadRequested '%s'" %(url)
filename = url_unquote(url).split("/")[-1]
localfile = "%s/%s" %(config.plugins.WebBrowser.downloadpath.value, filename)
downloadManager.AddJob(DownloadJob(url, localfile, filename))
self.session.open(MessageBox, _("Download started..."), type = MessageBox.TYPE_INFO, timeout = 3)
def parse_editform(request, form):
r"""
>>> from graphingwiki.editing import _doctest_request
>>> request = _doctest_request()
>>> parse_editform(request, {"Test-gwikiseparator-" : ["1", "2"], ":: " : ["a"]})
{'Test': ({}, {'a': ['1', '2']})}
>>> request = _doctest_request({"Test" : {"meta" : {"a" : ["x"]}}})
>>> parse_editform(request, {"Test-gwikiseparator-a" : ["1", "2"], ":: a" : ["a"]})
{'Test': ({'a': ['x']}, {'a': ['1', '2']})}
>>> request = _doctest_request({"Test" : {"meta" : {"a" : ["x"]}}})
>>> parse_editform(request, {"Test-gwikiseparator-a" : ["x"], ":: a" : [""]})
{'Test': ({'a': ['x']}, {'a': ['']})}
>>> request = _doctest_request({"Test" : {"meta" : {"a" : ["1", "2"]}}})
>>> parse_editform(request, {"Test-gwikiseparator-a" : ["1"], ":: a" : ["a"]})
{'Test': ({'a': ['1', '2']}, {'a': ['1', '']})}
"""
_ = request.getText
msgs = list()
keys = dict()
pages = dict()
files = dict()
for key in request.files:
f = request.files[key]
if not f.filename:
continue
fileobj = f.stream
page, key = key.split(SEPARATOR, 1)
files.setdefault(page, dict())[key] = (f.filename, fileobj)
# Key changes
for oldKey, newKeys in form.iteritems():
if not newKeys or not oldKey.startswith(':: '):
continue
newKey = newKeys[0].strip()
oldKey = oldKey[3:].strip()
if oldKey == newKey:
continue
keys[oldKey] = newKey
# Value changes
for pageAndKey, newValues in form.iteritems():
if pageAndKey.endswith("__filename__"):
continue
# At least the key 'save' may be there and should be ignored
if not SEPARATOR in pageAndKey:
continue
# Form keys not autodecoded from utf-8
page, oldKey = pageAndKey.split(SEPARATOR, 1)
# Decoding pagename
page = url_unquote(page)
newKey = keys.get(oldKey, oldKey)
if not request.user.may.write(page):
err = '%s: ' % page + _('You are not allowed to edit this page.')
if not err in msgs:
msgs.append(err)
continue
oldMeta, newMeta = pages.setdefault(page, (dict(), dict()))
if oldKey:
oldMetas = get_metas(request,
page, [oldKey],
abs_attach=False,
includeGenerated=False)
oldValues = oldMetas[oldKey]
oldMeta.setdefault(oldKey, list()).extend(oldValues)
if newKey != oldKey:
# Remove the corresponding values
newMeta.setdefault(oldKey,
list()).extend([""] * len(oldValues))
if newKey:
missing = 0
if oldKey:
missing = len(oldValues) - len(newValues)
newMeta.setdefault(newKey, list()).extend(newValues)
newMeta.setdefault(newKey, list()).extend([""] * missing)
# Prune
for page, (oldMeta, newMeta) in list(pages.iteritems()):
for key in list(newMeta):
if key not in oldMeta:
continue
if oldMeta[key] != newMeta[key]:
continue
del oldMeta[key]
del newMeta[key]
if not (oldMeta or newMeta):
del pages[page]
for page in files:
metas = pages.get(page, (dict(), dict()))[1]
for key in files[page]:
filename, descriptor = files[page][key]
values = metas.get(key, list())
try:
index = values.index(descriptor)
values[index] = "[[attachment:%s]]" % filename
except ValueError:
values.append("[[attachment:%s]]" % filename)
return pages, msgs, files
def parse_editform(request, form):
r"""
>>> from MoinMoin.metadata.util import doctest_request
>>> request = doctest_request()
>>> parse_editform(request, {"Test-gwikiseparator-" : ["1", "2"], ":: " : ["a"]})
{'Test': ({}, {'a': ['1', '2']})}
>>> request = doctest_request({"Test" : {"meta" : {"a" : ["x"]}}})
>>> parse_editform(request, {"Test-gwikiseparator-a" : ["1", "2"], ":: a" : ["a"]})
{'Test': ({'a': ['x']}, {'a': ['1', '2']})}
>>> request = doctest_request({"Test" : {"meta" : {"a" : ["x"]}}})
>>> parse_editform(request, {"Test-gwikiseparator-a" : ["x"], ":: a" : [""]})
{'Test': ({'a': ['x']}, {'a': ['']})}
>>> request = doctest_request({"Test" : {"meta" : {"a" : ["1", "2"]}}})
>>> parse_editform(request, {"Test-gwikiseparator-a" : ["1"], ":: a" : ["a"]})
{'Test': ({'a': ['1', '2']}, {'a': ['1', '']})}
"""
_ = request.getText
msgs = list()
keys = dict()
pages = dict()
files = dict()
for key in request.files:
f = request.files[key]
if not f.filename:
continue
fileobj = f.stream
page, key = key.split(SEPARATOR, 1)
files.setdefault(page, dict())[key] = (f.filename, fileobj)
# Key changes
for oldKey, newKeys in form.iteritems():
if not newKeys or not oldKey.startswith(':: '):
continue
newKey = newKeys[0].strip()
oldKey = oldKey[3:].strip()
if oldKey == newKey:
continue
keys[oldKey] = newKey
# Value changes
for pageAndKey, newValues in form.iteritems():
if pageAndKey.endswith("__filename__"):
continue
# At least the key 'save' may be there and should be ignored
if not SEPARATOR in pageAndKey:
continue
# Form keys not autodecoded from utf-8
page, oldKey = pageAndKey.split(SEPARATOR, 1)
# Decoding pagename
page = url_unquote(page)
newKey = keys.get(oldKey, oldKey)
if not request.user.may.write(page):
err = '%s: ' % page + _('You are not allowed to edit this page.')
if not err in msgs:
msgs.append(err)
continue
oldMeta, newMeta = pages.setdefault(page, (dict(), dict()))
if oldKey:
oldMetas = get_metas(request, page, [oldKey],
abs_attach=False, includeGenerated=False)
oldValues = oldMetas[oldKey]
oldMeta.setdefault(oldKey, list()).extend(oldValues)
if newKey != oldKey:
# Remove the corresponding values
newMeta.setdefault(oldKey, list()).extend([""] * len(oldValues))
if newKey:
missing = 0
if oldKey:
missing = len(oldValues) - len(newValues)
newMeta.setdefault(newKey, list()).extend(newValues)
newMeta.setdefault(newKey, list()).extend([""] * missing)
# Prune
for page, (oldMeta, newMeta) in list(pages.iteritems()):
for key in list(newMeta):
if key not in oldMeta:
continue
if oldMeta[key] != newMeta[key]:
continue
del oldMeta[key]
del newMeta[key]
if not (oldMeta or newMeta):
del pages[page]
for page in files:
metas = pages.get(page, (dict(), dict()))[1]
for key in files[page]:
filename, descriptor = files[page][key]
values = metas.get(key, list())
try:
index = values.index(descriptor)
values[index] = "[[attachment:%s]]" % filename
except ValueError:
values.append("[[attachment:%s]]" % filename)
return pages, msgs, files
def send(self, request, stream=False, verify=None, cert=None, proxies=None,
timeout=None):
"""issue request"""
data = url_unquote(request.url[len('data:'):])
if ',' not in data:
raise InvalidURL('data URL missing comma')
mime, content = data.split(',', 1)
content = content.strip()
base64 = False
charset = None
while ';' in mime:
mime, encoding_spec = mime.rsplit(';', 1)
encoding_spec = encoding_spec.strip()
if encoding_spec == 'base64':
base64 = True
elif not encoding_spec.startswith('charset='):
raise InvalidURL(
'unrecognized encoding parameter: %r' % encoding_spec
)
else:
charset = encoding_spec[len('charset='):]
try:
if base64:
content = a2b_base64(content)
content_type = mime.strip()
if charset:
content_type += "; charset=" + charset
response = Response()
response.url = request.url
response.headers['Date'] = formatdate(timeval=None, localtime=True)
if request.method in ('GET', 'HEAD'):
response.status_code = 200
response.headers['Content-Length'] = len(content)
response.headers['Last-Modified'] = formatdate()
response.headers['Content-Type'] = content_type
if charset:
response.encoding = charset
response.raw = StringIO(str(content))
else:
response.status_code = 405
response.headers['Status'] = '405 Method Not Allowed'
except Exception:
response.status_code = 500
response.headers['Status'] = '500 Internal Server Error'
response.raw = StringIO(format_exc())
# context
response.request = request
response.connection = self
# hooks
response = dispatch_hook('response', request.hooks, response)
# streaming
if not stream:
response.content
return response
def decleanId(id):
"""Reverse cleanId."""
if id:
id = id.replace('--', '\x00').replace('-', '%').replace('\x00', '-')
return url_unquote(id)
return ''
def url_unquote_text(v, encoding="utf-8", errors="replace"): # pragma: no cover
v = url_unquote(v)
return v.decode(encoding, errors)
