def get_user_info(self, auth_token, auth_verifier=""):
"""Get User Info.
Exchanges the auth token for an access token and returns a dictionary
of information about the authenticated user.
"""
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
auth_secret = memcache.get(self._get_memcache_auth_key(auth_token))
if not auth_secret:
logging.error("The auth token %s is missing from memcache" % auth_token)
response = self.make_request(self.access_url,
token=auth_token,
secret=auth_secret,
additional_params={"oauth_verifier":
auth_verifier})
# Extract the access token/secret from the response.
result = self._extract_credentials(response)
# Try to collect some information about this user from the service.
user_info = self._lookup_user_info(result["token"], result["secret"])
user_info.update(result)
return user_info
def get_user_info(self, auth_token, auth_verifier=""):
"""Get User Info.
Exchanges the auth token for an access token and returns a dictionary
of information about the authenticated user.
"""
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
auth_secret = memcache.get(self._get_memcache_auth_key(auth_token))
if not auth_secret:
result = AuthToken.gql(""" WHERE token = :1 LIMIT 1 """, auth_token).get()
if not result:
logging.error("The auth token %s was not found in our db" % auth_token)
raise Exception, "Could not find Auth Token in database"
else:
auth_secret = result.secret
response = self.make_request(self.access_url,
token=auth_token,
secret=auth_secret,
additional_params={"oauth_verifier":
auth_verifier})
# Extract the access token/secret from the response.
result = self._extract_credentials(response)
# Try to collect some information about this user from the service.
user_info = self._lookup_user_info(result["token"], result["secret"])
user_info.update(result)
return user_info
def _url2path(self,url):
"""Convert a server-side URL into a client-side path."""
path = urlunquote(urlparse(url).path)
root = urlunquote(self._url_p.path)
path = path[len(root)-1:].decode("utf8")
while path.endswith("/"):
path = path[:-1]
return path
def _url2path(self, url):
"""Convert a server-side URL into a client-side path."""
path = urlunquote(urlparse(url).path)
root = urlunquote(self._url_p.path)
path = path[len(root) - 1:].decode("utf8")
while path.endswith("/"):
path = path[:-1]
return path
def get_access_token(self, auth_token, auth_verifier):
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
response = self.make_request( self.access_url,
token=auth_token,
additional_params={"oauth_verifier": auth_verifier} )
result = parse_qs(response.content)
return result["user_id"][0], result["oauth_token"][0], result["oauth_token_secret"][0]
def run(self, edit):
for s in self.view.sel():
if s.empty():
s = self.view.word(s)
selected = self.view.substr(s)
try:
txt = urlunquote(selected.encode('utf8')).decode('utf8')
except TypeError:
txt = urlunquote(selected)
self.view.replace(edit, s, txt)
def urldecode(qs):
r = []
for pair in qs.replace(';', '&').split('&'):
if not pair: continue
nv = pair.split('=', 1)
if len(nv) != 2: nv.append('')
key = urlunquote(nv[0].replace('+', ' '))
value = urlunquote(nv[1].replace('+', ' '))
r.append((key, value))
return r
def urldecode(qs):
r = []
for pair in qs.replace(';', '&').split('&'):
if not pair: continue
nv = pair.split('=', 1)
if len(nv) != 2: nv.append('')
key = urlunquote(nv[0].replace('+', ' '))
value = urlunquote(nv[1].replace('+', ' '))
r.append((key, value))
return r
def get_access_token(self, auth_token, auth_verifier):
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
response = self.make_request(
self.access_url, token=auth_token, additional_params={"oauth_verifier": auth_verifier}
)
# Extract the access token/secret from the response.
result = self._extract_credentials(response)
return result["token"], result["secret"], result["screen_name"]
def get_access_token(self, auth_token, auth_verifier):
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
response = self.make_request(
self.access_url,
token=auth_token,
additional_params={"oauth_verifier": auth_verifier})
# Extract the access token/secret from the response.
result = self._extract_credentials(response)
return result['token'], result['secret'], result['screen_name']
def update_status(self, status,auth_token,auth_verifier=""):
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
global auth_secret
response = self.make_request(self.access_url,
token=auth_token,
secret=auth_secret,
additional_params={"oauth_verifier":auth_verifier})
result = self._extract_credentials(response)
data = self._update_status(result["token"], result["secret"],status)
return data
def from_parsedurl(cls, parsedurl, **kwargs):
auth, host = urllib2.splituser(parsedurl.netloc)
password = parsedurl.password
if password is not None:
password = urlunquote(password)
username = parsedurl.username
if username is not None:
username = urlunquote(username)
# TODO(jelmer): This also strips the username
parsedurl = parsedurl._replace(netloc=host)
return cls(urlparse.urlunparse(parsedurl),
password=password, username=username, **kwargs)
def get_user_info(self, auth_token, auth_verifier=""):
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
global auth_secret
response = self.make_request(self.access_url,
token=auth_token,
secret=auth_secret,
additional_params={"oauth_verifier":auth_verifier})
result = self._extract_credentials(response)
user_info = self._lookup_user_info(result["token"], result["secret"])
user_info.update(result)
return user_info
def get_user_info(self, auth_token, auth_verifier=""):
"""Get User Info.
Exchanges the auth token for an access token and returns a dictionary
of information about the authenticated user.
"""
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
auth_secret = memcache.get(self._get_memcache_auth_key(auth_token))
if not auth_secret:
result = AuthToken.gql("""
WHERE
service = :1 AND
token = :2
LIMIT
1
""", self.service_name, auth_token).get()
if not result:
logging.error("The auth token %s was not found in our db" % auth_token)
raise Exception, "Could not find Auth Token in database"
else:
auth_secret = result.secret
response = self.make_request(self.access_url,
token=auth_token,
secret=auth_secret,
additional_params={"oauth_verifier":
auth_verifier})
key_name = ""
username = ""
if self.service_name == 'gdi-acc':
respdict = parse_qs(response.content)
username = respdict['user_name']
key_name = 'id-%s' % uuid4();
# Extract the access token/secret from the response.
result = self._extract_credentials(response)
# Try to collect some information about this user from the service.
#user_info = self._lookup_user_info(result["token"], result["secret"])
#user_info.update(result)
self.set_cookie(key_name)
self.set_cookie_username(username[0])
return username[0]
def get_access_token(self, request_token, verifier):
"""Exchanges a Request Token for an Access Token. Returns (access_token, access_secret).
Note that the access_token and access_secret can be stored, allowing future
requests to be made without going through the happy OAuth dance again.
"""
request_token, verifier = urlunquote(request_token), urlunquote(verifier)
request_secret = self._retrieve_request_secret(request_token)
access_request = self._make_request(self.access_url,
method = 'POST',
token = request_token,
secret = request_secret,
additional_oauth = {"oauth_verifier": verifier})
access_response = self._extract_credentials(access_request)
return (access_response["token"], access_response["secret"])
def parse_content(content):
content_dict = {}
content_pairs = content.split("&")
for content_pair in content_pairs:
k,v = content_pair.split("=")
content_dict[k] = urlunquote(v)
return content_dict
def _evict(self, path, top):
assert self._lock is not None
if self.state.numbytes <= self.config.limitbytes:
return
# eliminate in sort order
items = os.listdir(path)
items.sort()
for fn in items:
if self.state.numbytes <= self.config.limitbytes:
return
subpath = os.path.join(path, fn)
if os.path.isdir(subpath):
# descend to the next level
self._evict(subpath, False)
else:
# delete a file
name = urlunquote(fn[fn.index(self.config.delimiter) + 1:])
self._del(name)
self.state.numbytes -= os.path.getsize(subpath)
os.remove(subpath)
# clean up empty directories
if not top:
try:
os.rmdir(path)
except:
pass
def verify(self, request):
""" verify: Called by the external service to verify you authentication request
"""
auth_token = urlunquote(request.get("oauth_token"))
auth_verifier = urlunquote(request.get("oauth_verifier"))
# Extract the access token/secret from the response.
response = self._make_request(self.access_token_url, token=auth_token, secret=self.store['oauth_secret'], additional_params={"oauth_verifier": auth_verifier})
if response.status_code == 200:
data = self._extract_credentials(response)
self.user.credentials = OAuth1Credentials(user_token=data['token'], user_secret=data['secret'])
else:
raise OAuthClientError(response.status_code, response.content)
return super(OAuth1Client, self).verify(request)
def get_user_info(self, auth_token, auth_verifier=""):
"""Get User Info.
Exchanges the auth token for an access token and returns a dictionary
of information about the authenticated user.
"""
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
auth_secret = memcache.get(self._get_memcache_auth_key(auth_token))
if not auth_secret:
result = AuthToken.gql(
"""
WHERE
service = :1 AND
token = :2
LIMIT
1
""", self.service_name, auth_token).get()
if not result:
logging.error(
"!!!!!!!!!!!!!! The auth token %s was not found in our db"
% auth_token)
raise Exception, "Could not find Auth Token in database"
else:
auth_secret = result.secret
response = self.make_request(
self.access_url,
token=auth_token,
secret=auth_secret,
additional_params={"oauth_verifier": auth_verifier})
# Extract the access token/secret from the response.
result = self._extract_credentials(response)
auth_secret = result["secret"]
# Try to collect some information about this user from the service.
user_info = self._lookup_user_info(result["token"], result["secret"])
user_info.update(result)
return user_info
def unquote(src):
if not isinstance(src, str):
return src
try:
return urlunquote(src).decode('utf-8', 'ignore')
except:
pass
return src
def verify(self, request):
connected = False
if self.version == '1.0':
auth_token = urlunquote(request.get("oauth_token"))
auth_verifier = urlunquote(request.get("oauth_verifier"))
# Extract the access token/secret from the response.
response = self._make_request(self.access_url, token=auth_token, secret=self.tokens.oauth_secret, additional_params={"oauth_verifier": auth_verifier})
if response.status_code == 200:
data = self._extract_credentials(response)
self.modified = datetime.datetime.now()
self.tokens.oauth_user_token = data['token']
self.tokens.oauth_user_secret = data['secret']
self.tokens.put()
connected = True
elif self.version == '2.0':
# prepare a post to get the access code
params = {
"grant_type": "authorization_code",
"client_id": self.consumer_key,
"client_secret": self.consumer_secret,
"redirect_uri": self.callback_url
}
params_str = "code=" + request.get("code") + "&" + urlencode(params)
response = urlfetch.fetch(self.access_url, method=urlfetch.POST, payload=params_str, deadline=20, follow_redirects=True)
if response.status_code == 200:
data = json.loads(response.content)
self.tokens.modified = datetime.datetime.now()
self.tokens.oauth_access_token = data["access_token"]
self.tokens.oauth_refresh_token = data["refresh_token"]
self.tokens.oauth_expires_in = data["expires_in"]
self.tokens.put()
connected = True
else:
raise Error(response.status_code, response.content)
if connected: # Get the user information
self.setUserInfo()
def get_user_info(self, auth_token, auth_verifier=''):
'''Get User Info.
Exchanges the auth token for an access token and returns a dictionary
of information about the authenticated user.
'''
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
auth_secret = memcache.get(self._get_memcache_auth_key(auth_token))
if not auth_secret:
result = AuthToken.gql('''
WHERE
service = :1 AND
token = :2
LIMIT
1
''', self.service_name, auth_token).get()
if not result:
logging.error('The auth token %s was not found in our db' % auth_token)
raise Exception, 'Could not find Auth Token in database'
else:
auth_secret = result.secret
response = self.make_request(self.access_url,
token=auth_token,
secret=auth_secret,
additional_params={'oauth_verifier':
auth_verifier})
# Extract the access token/secret from the response.
result = self._extract_credentials(response)
# Try to collect some information about this user from the service.
user_info = self._lookup_user_info(result['token'], result['secret'])
user_info.update(result)
return user_info
def _cover_from_html(self, hcover):
from calibre.ebooks import render_html_svg_workaround
with TemporaryDirectory('_html_cover') as tdir:
writer = OEBWriter()
writer(self.oeb, tdir)
path = os.path.join(tdir, urlunquote(hcover.href))
data = render_html_svg_workaround(path, self.logger)
if not data:
data = ''
id, href = self.oeb.manifest.generate('cover', 'cover.jpg')
item = self.oeb.manifest.add(id, href, JPEG_MIME, data=data)
return item
def recurse(depth, num, path):
items = os.listdir(path)
items.sort()
if len(items) == 0:
raise ValueError(
"cannot join {0} because {1} is an empty directory".format(
repr(directory), repr(path)))
elif len(items) > out.config.maxperdir:
raise ValueError(
"cannot join {0} because {1} has more than maxperdir ({2}) items"
.format(repr(directory), repr(path), out.config.maxperdir))
elif all(
os.path.isdir(os.path.join(path, fn)) and digits.match(fn)
for fn in items):
for fn in items:
recurse(depth + 1, (num + int(fn)) * out.config.maxperdir,
os.path.join(path, fn))
elif all(not os.path.isdir(os.path.join(path, fn))
for fn in items):
if depth != out.state.depth:
raise ValueError(
"cannot join {0} because depth in {1} ({2}) disagrees with depth in directory tree ({3})"
.format(repr(directory), repr(DiskCache.CONFIG_FILE),
out.config.depth, depth))
for fn in items:
if out.config.delimiter not in fn or not digits.match(
fn[:fn.index(out.config.delimiter)]):
raise ValueError(
"cannot join {0} because {1} is not a proper file name with delimiter {2}"
.format(repr(directory), repr(fn),
repr(out.config.delimiter)))
i = fn.index(out.config.delimiter)
number = num + int(fn[:i])
name = urlunquote(fn[i + 1:])
path = out._get(name)
if not os.path.exists(path):
raise ValueError(
"cannot join {0} because {1} (for key {2}) does not exist"
.format(repr(directory), repr(path), repr(name)))
out._unchecked[out._index(name)] = False
else:
raise ValueError(
"cannot join {0} because directory {1} is neither an all-directory nor an all-file directory"
.format(repr(directory), repr(path)))
def href_to_name(self, href, base=None):
"""
Convert an href (relative to base) to a name (i.e. a path
relative to self.root with POSIX separators).
base must be an absolute path with OS separators or None, in which case
the href is interpreted relative to the dir containing the OPF.
"""
if base is None:
base = self.opf_dir
href = urlunquote(href.partition("#")[0])
fullpath = os.path.join(base, *href.split("/"))
return self.abspath_to_name(fullpath)
def get_user_info(self, auth_token, auth_verifier="", detailed_info=False):
"""Get User Info.
Exchanges the auth token for an access token and returns a dictionary
of information about the authenticated user.
"""
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
key = AuthToken.key_name(self.service_name, auth_token)
auth_model = memcache.get(key) # @UndefinedVariable
if not auth_model:
auth_model = AuthToken.get_by_key_name(key)
if not auth_model:
logging.error("The auth token %s was not found in our db" % auth_token)
raise Exception, "Could not find Auth Token in database"
if not auth_model.auth_token:
response = self.make_request(self.access_url, token=auth_token, secret=auth_model.secret,
additional_params={"oauth_verifier": auth_verifier})
# Extract the access token/secret from the response.
result = self._extract_credentials(response)
auth_model.auth_token = json.dumps(result)
auth_model.put()
memcache.set(key, auth_model, 20 * 60) # @UndefinedVariable
else:
result = json.loads(auth_model.auth_token)
# Try to collect some information about this user from the service.
if detailed_info:
user_info = self._lookup_user_info(result["token"], result["secret"])
user_info.update(result)
return user_info
else:
return result
def lambda_handler(event, context):
bucketname = 'dbarnetttestimageforlambda'
tilex, tiley = urlunquote(event['coords']).strip("()").replace(' ', '').split(',')
zoom = event['zoom']
keyname = Key="%s:%s:%s" % (zoom, tilex, tiley)
s3 = boto3.resource('s3')
try:
#s3.Bucket(bucketname).get_object(keyname)
s3.Object(bucketname, keyname).load()
except botocore.exceptions.ClientError as e:
Mandel(int(zoom), int(tilex), int(tiley))
s3.Bucket(bucketname).put_object(Key=keyname, Body=open('/tmp/tile.png', 'rb'))
return json.loads('{"location": "https://s3-us-west-2.amazonaws.com/%s/%s"}' % (bucketname, keyname))
def read_manifest(self):
if '/manifest' not in self.entries:
raise LitError('Lit file does not have a valid manifest')
raw = self.get_file('/manifest')
self.manifest = {}
self.paths = {self.opf_path: None}
while raw:
slen, raw = ord(raw[0]), raw[1:]
if slen == 0:
break
root, raw = raw[:slen].decode('utf8'), raw[slen:]
if not raw:
raise LitError('Truncated manifest')
for state in ['spine', 'not spine', 'css', 'images']:
num_files, raw = int32(raw), raw[4:]
if num_files == 0:
continue
for i in range(num_files):
if len(raw) < 5:
raise LitError('Truncated manifest')
offset, raw = u32(raw), raw[4:]
internal, raw = consume_sized_utf8_string(raw)
original, raw = consume_sized_utf8_string(raw)
# The path should be stored unquoted, but not always
original = urlunquote(original)
# Is this last one UTF-8 or ASCIIZ?
mime_type, raw = consume_sized_utf8_string(raw, zpad=True)
self.manifest[internal] = ManifestItem(
original, internal, mime_type, offset, root, state)
mlist = self.manifest.values()
# Remove any common path elements
if len(mlist) > 1:
shared = mlist[0].path
for item in mlist[1:]:
path = item.path
while shared and not path.startswith(shared):
try:
shared = shared[:shared.rindex("/", 0, -2) + 1]
except ValueError:
shared = None
if not shared:
break
if shared:
slen = len(shared)
for item in mlist:
item.path = item.path[slen:]
# Fix any straggling absolute paths
for item in mlist:
if item.path[0] == '/':
item.path = os.path.basename(item.path)
self.paths[item.path] = item
def read_manifest(self):
if '/manifest' not in self.entries:
raise LitError('Lit file does not have a valid manifest')
raw = self.get_file('/manifest')
self.manifest = {}
self.paths = {self.opf_path: None}
while raw:
slen, raw = ord(raw[0]), raw[1:]
if slen == 0:
break
root, raw = raw[:slen].decode('utf8'), raw[slen:]
if not raw:
raise LitError('Truncated manifest')
for state in ['spine', 'not spine', 'css', 'images']:
num_files, raw = int32(raw), raw[4:]
if num_files == 0:
continue
for i in xrange(num_files):
if len(raw) < 5:
raise LitError('Truncated manifest')
offset, raw = u32(raw), raw[4:]
internal, raw = consume_sized_utf8_string(raw)
original, raw = consume_sized_utf8_string(raw)
# The path should be stored unquoted, but not always
original = urlunquote(original)
# Is this last one UTF-8 or ASCIIZ?
mime_type, raw = consume_sized_utf8_string(raw, zpad=True)
self.manifest[internal] = ManifestItem(
original, internal, mime_type, offset, root, state)
mlist = self.manifest.values()
# Remove any common path elements
if len(mlist) > 1:
shared = mlist[0].path
for item in mlist[1:]:
path = item.path
while shared and not path.startswith(shared):
try:
shared = shared[:shared.rindex("/", 0, -2) + 1]
except ValueError:
shared = None
if not shared:
break
if shared:
slen = len(shared)
for item in mlist:
item.path = item.path[slen:]
# Fix any straggling absolute paths
for item in mlist:
if item.path[0] == '/':
item.path = os.path.basename(item.path)
self.paths[item.path] = item
def get_toc(self):
self.stream.seek(24)
toc_offset = self.read_i32()
self.stream.seek(toc_offset)
pages = self.read_i32()
toc = RBToc()
for i in range(pages):
name = urlunquote(self.stream.read(32).strip('\x00'))
size, offset, flags = self.read_i32(), self.read_i32(), self.read_i32()
toc.append(RBToc.Item(name=name, size=size, offset=offset, flags=flags))
return toc
def get_user_info(self, auth_token, auth_verifier=""):
"""Get User Info.
Exchanges the auth token for an access token and returns a dictionary
of information about the authenticated user.
"""
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
auth_secret = self._load_auth_secret(auth_token)
access_token = self._get_access_token(
auth_token, auth_verifier, auth_secret)
# Try to collect some information about this user from the service.
user_info = self._lookup_user_info(
access_token.access_token,
access_token.access_secret
)
user_info['service'] = self.service_name
user_info['auth_token'] = auth_token
return user_info
def get_credentials(self, auth_token, auth_verifier=""):
"""Gets credentials
Exchanges the auth token for an access token and returns it for storage elsewhere.
"""
auth_token = urlunquote(auth_token)
auth_verifier = urlunquote(auth_verifier)
auth_secret = memcache.get(self._get_memcache_auth_key(auth_token))
if not auth_secret:
result = AuthToken.gql("""
WHERE
service = :1 AND
token = :2
LIMIT
1
""", self.service_name, auth_token).get()
if not result:
logging.error("The auth token %s was not found in our db" % auth_token)
raise Exception, "Could not find AuthToken in database"
else:
auth_secret = result.secret
response = self.make_request(self.access_url,
token=auth_token,
secret=auth_secret,
additional_params={"oauth_verifier":
auth_verifier})
# Extract the access token/secret from the response.
result = self._extract_credentials(response)
return result
def href_to_name(self, href, base=None):
'''
Convert an href (relative to base) to a name. base must be a name or
None, in which case self.root is used.
'''
if base is None:
base = self.root
else:
base = os.path.dirname(self.name_to_abspath(base))
purl = urlparse(href)
if purl.scheme or not purl.path or purl.path.startswith('/'):
return None
href = urlunquote(purl.path)
fullpath = os.path.join(base, *href.split('/'))
return self.abspath_to_name(fullpath)
def href_to_name(self, href, base=None):
'''
Convert an href (relative to base) to a name. base must be a name or
None, in which case self.root is used.
'''
if base is None:
base = self.root
else:
base = os.path.dirname(self.name_to_abspath(base))
purl = urlparse(href)
if purl.scheme or not purl.path or purl.path.startswith('/'):
return None
href = urlunquote(purl.path)
fullpath = os.path.join(base, *href.split('/'))
return self.abspath_to_name(fullpath)
def _walkorder(self, path, sort=True, reverse=False):
assert self._lock is not None
items = os.listdir(path)
if sort:
items.sort(reverse=reverse)
for fn in items:
subpath = os.path.join(path, fn)
if os.path.isdir(subpath):
for x in self._walkorder(subpath, sort=sort, reverse=reverse):
yield x
else:
i = fn.index(self.config.delimiter)
num = int(fn[:i])
name = urlunquote(fn[i + 1:])
yield num, name.encode("utf-8")
def read(self, name):
entry = self._litfile.paths[urlunquote(name)] if name else None
if entry is None:
content = OPF_DECL + self._read_meta()
elif 'spine' in entry.state:
internal = '/'.join(('/data', entry.internal, 'content'))
raw = self._litfile.get_file(internal)
manifest = self._litfile.manifest
atoms = self._litfile.get_atoms(entry)
unbin = UnBinary(raw, name, manifest, HTML_MAP, atoms)
content = HTML_DECL + str(unbin)
tags = ('personname', 'place', 'city', 'country-region')
pat = r'(?i)</{0,1}st1:(%s)>'%('|'.join(tags))
content = re.sub(pat, '', content)
content = re.sub(r'<(/{0,1})form>', r'<\1div>', content)
else:
internal = '/'.join(('/data', entry.internal))
content = self._litfile.get_file(internal)
return content
def _percentDecode(text):
"""
Replace percent-encoded characters with their UTF-8 equivalents.
@param text: The text with percent-encoded UTF-8 in it.
@type text: L{unicode}
@return: the encoded version of C{text}
@rtype: L{unicode}
"""
try:
quotedBytes = text.encode("ascii")
except UnicodeEncodeError:
return text
unquotedBytes = urlunquote(quotedBytes)
try:
return unquotedBytes.decode("utf-8")
except UnicodeDecodeError:
return text
def _percentDecode(text):
"""
Replace percent-encoded characters with their UTF-8 equivalents.
@param text: The text with percent-encoded UTF-8 in it.
@type text: L{unicode}
@return: the encoded version of C{text}
@rtype: L{unicode}
"""
try:
quotedBytes = text.encode("ascii")
except UnicodeEncodeError:
return text
unquotedBytes = urlunquote(quotedBytes)
try:
return unquotedBytes.decode("utf-8")
except UnicodeDecodeError:
return text
def _percent_decode(text):
"""
Replace percent-encoded characters with their UTF-8 equivalents.
Args:
text (unicode): The text with percent-encoded UTF-8 in it.
Returns:
unicode: The encoded version of *text*.
"""
try:
quotedBytes = text.encode("ascii")
except UnicodeEncodeError:
return text
unquotedBytes = urlunquote(quotedBytes)
try:
return unquotedBytes.decode("utf-8")
except UnicodeDecodeError:
return text
def read(self, name):
entry = self._litfile.paths[urlunquote(name)] if name else None
if entry is None:
content = OPF_DECL + self._read_meta()
elif 'spine' in entry.state:
internal = '/'.join(('/data', entry.internal, 'content'))
raw = self._litfile.get_file(internal)
manifest = self._litfile.manifest
atoms = self._litfile.get_atoms(entry)
unbin = UnBinary(raw, name, manifest, HTML_MAP, atoms)
content = HTML_DECL + str(unbin)
tags = ('personname', 'place', 'city', 'country-region')
pat = r'(?i)</{0,1}st1:(%s)>' % ('|'.join(tags))
content = re.sub(pat, '', content)
content = re.sub(r'<(/{0,1})form>', r'<\1div>', content)
else:
internal = '/'.join(('/data', entry.internal))
content = self._litfile.get_file(internal)
return content
def _build_manifest(self):
states = ['linear', 'nonlinear', 'css', 'images']
manifest = dict((state, []) for state in states)
for item in self._oeb.manifest.values():
if item.spine_position is not None:
key = 'linear' if item.linear else 'nonlinear'
manifest[key].append(item)
elif item.media_type in OEB_STYLES:
manifest['css'].append(item)
elif item.media_type in LIT_IMAGES:
manifest['images'].append(item)
data = StringIO()
data.write(pack('<Bc', 1, '\\'))
offset = 0
for state in states:
items = manifest[state]
items.sort()
data.write(pack('<I', len(items)))
for item in items:
id, media_type = item.id, item.media_type
if media_type in OEB_DOCS:
# Needs to have 'html' in media-type
media_type = XHTML_MIME
elif media_type in OEB_STYLES:
media_type = CSS_MIME
href = urlunquote(item.href)
item.offset = offset \
if state in ('linear', 'nonlinear') else 0
data.write(pack('<I', item.offset))
entry = [
unichr(len(id)),
unicode(id),
unichr(len(href)),
unicode(href),
unichr(len(media_type)),
unicode(media_type)
]
for value in entry:
data.write(value.encode('utf-8'))
data.write('\0')
offset += item.size
self._add_file('/manifest', data.getvalue())
def _build_manifest(self):
states = ['linear', 'nonlinear', 'css', 'images']
manifest = dict((state, []) for state in states)
for item in self._oeb.manifest.values():
if item.spine_position is not None:
key = 'linear' if item.linear else 'nonlinear'
manifest[key].append(item)
elif item.media_type in OEB_STYLES:
manifest['css'].append(item)
elif item.media_type in LIT_IMAGES:
manifest['images'].append(item)
data = StringIO()
data.write(pack('<Bc', 1, '\\'))
offset = 0
for state in states:
items = manifest[state]
items.sort()
data.write(pack('<I', len(items)))
for item in items:
id, media_type = item.id, item.media_type
if media_type in OEB_DOCS:
# Needs to have 'html' in media-type
media_type = XHTML_MIME
elif media_type in OEB_STYLES:
media_type = CSS_MIME
href = urlunquote(item.href)
item.offset = offset \
if state in ('linear', 'nonlinear') else 0
data.write(pack('<I', item.offset))
entry = [unichr(len(id)), unicode(id),
unichr(len(href)), unicode(href),
unichr(len(media_type)), unicode(media_type)]
for value in entry:
data.write(value.encode('utf-8'))
data.write('\0')
offset += item.size
self._add_file('/manifest', data.getvalue())
def refresh(self, access_token, access_secret, session_handle):
"""Get User Info.
Exchanges the auth token for an access token and returns a dictionary
of information about the authenticated user.
"""
access_token = urlunquote(access_token)
response = self.make_request(self.access_url,
token=access_token,
secret=access_secret,
additional_params={"oauth_session_handle": session_handle})
if response.status_code != 200:
raise Exception("The refresh failed | %s" % (response.content))
# Extract the access token/secret from the response.
result = self._extract_credentials(response)
# Try to collect some information about this user from the service.
user_info = self._lookup_user_info(result["token"], result["secret"])
user_info.update(result)
return user_info
def handle_http_request(
env, start_response, dict=dict, isinstance=isinstance, urlunquote=urlunquote,
unicode=unicode, get_response_headers=lambda: None
):
reqlocal.template_error_traceback = None
try:
http_method = env['REQUEST_METHOD']
ssl_mode = env['wsgi.url_scheme'] == 'https'
if http_method == 'OPTIONS':
start_response(*RESPONSE_OPTIONS)
return []
if http_method not in SUPPORTED_HTTP_METHODS:
start_response(*RESPONSE_NOT_IMPLEMENTED)
return []
_path_info = env['PATH_INFO']
if isinstance(_path_info, unicode):
_args = [arg for arg in _path_info.split(u'/') if arg]
else:
_args = [
unicode(arg, 'utf-8', 'strict')
for arg in _path_info.split('/') if arg
]
kwargs = {}
for part in [
sub_part
for part in env['QUERY_STRING'].lstrip('?').split('&')
for sub_part in part.split(';')
]:
if not part:
continue
part = part.split('=', 1)
if len(part) == 1:
value = None
else:
value = part[1]
key = urlunquote(part[0].replace('+', ' '))
if value:
value = unicode(
urlunquote(value.replace('+', ' ')), 'utf-8', 'strict'
)
else:
value = None
if key in kwargs:
_val = kwargs[key]
if isinstance(_val, list):
_val.append(value)
else:
kwargs[key] = [_val, value]
continue
kwargs[key] = value
ctx = Context(env, ssl_mode)
router = handle_http_request.router
if router:
_info = router(ctx, _args, kwargs)
if not _info:
logging.error("No handler found for: %s" % _path_info)
raise NotFound
name, args = _info
else:
if _args:
name = _args[0]
args = _args[1:]
else:
name = '/'
args = ()
if name not in HANDLERS:
logging.error("Handler not found: %s" % name)
raise NotFound
handler, renderers, config = HANDLERS[name]
json = config['json']
# Parse the POST body if it exists and is of a known content type.
if http_method == 'POST':
content_type = env.get('CONTENT-TYPE', '')
if not content_type:
content_type = env.get('CONTENT_TYPE', '')
if ';' in content_type:
content_type = content_type.split(';', 1)[0]
if json or content_type == 'application/json':
payload = json_decode(env['wsgi.input'].read())
if json and not (json is True):
kwargs[json] = payload
else:
kwargs.update(payload)
elif content_type in VALID_REQUEST_CONTENT_TYPES:
post_environ = env.copy()
post_environ['QUERY_STRING'] = ''
if config['post_encoding']:
ctx.request_body = env['wsgi.input'].read()
env['wsgi.input'] = StringIO(ctx.request_body)
post_encoding = config['post_encoding']
else:
post_encoding = 'utf-8'
post_data = FieldStorage(
environ=post_environ, fp=env['wsgi.input'],
keep_blank_values=True
).list or []
for field in post_data:
key = field.name
if field.filename:
if config['blob']:
value = parse_blob_info(field)
else:
value = field
else:
value = unicode(field.value, post_encoding, 'strict')
if key in kwargs:
_val = kwargs[key]
if isinstance(_val, list):
_val.append(value)
else:
kwargs[key] = [_val, value]
continue
kwargs[key] = value
def get_response_headers():
# Figure out the HTTP headers for the response ``cookies``.
cookie_output = SimpleCookie()
for name, values in ctx._response_cookies.iteritems():
name = str(name)
cookie_output[name] = values.pop('value')
cur = cookie_output[name]
for key, value in values.items():
if key == 'max_age':
key = 'max-age'
if key not in COOKIE_KEY_NAMES:
continue
cur[key] = value
if cookie_output:
raw_headers = ctx._raw_headers + [
('Set-Cookie', ck.split(' ', 1)[-1])
for ck in str(cookie_output).split('\r\n')
]
else:
raw_headers = ctx._raw_headers
str_headers = []; new_header = str_headers.append
for k, v in raw_headers:
if isinstance(k, unicode):
k = k.encode('utf-8')
if isinstance(v, unicode):
v = v.encode('utf-8')
new_header((k, v))
return str_headers
if 'submit' in kwargs:
del kwargs['submit']
if 'callback' in kwargs:
ctx.json_callback = kwargs.pop('callback')
if env.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest':
ctx.ajax_request = 1
if '__ajax__' in kwargs:
ctx.ajax_request = 1
del kwargs['__ajax__']
if config['ssl'] and RUNNING_ON_GOOGLE_SERVERS and not ssl_mode:
raise NotFound
if config['xsrf']:
if 'xsrf' not in kwargs:
raise AuthError("XSRF token not present.")
provided_xsrf = kwargs.pop('xsrf')
if not secure_string_comparison(provided_xsrf, ctx.xsrf_token):
raise AuthError("XSRF tokens do not match.")
if config['admin'] and not ctx.is_admin:
raise NotFound
if (not config['anon']) and (not ctx.user_id):
if ctx.ajax_request:
ctx.response_headers['Content-Type'] = 'application/json'
raise HTTPContent(encode_json({
"error": {
"type": "AuthError",
"redirect": ctx.get_login_url()
}
}))
raise Redirect(ctx.get_login_url())
# Try and respond with the result of calling the handler.
content = handler(ctx, *args, **kwargs)
for renderer in renderers:
if ctx.end_pipeline:
break
if content is None:
content = {
'content': ''
}
elif not isinstance(content, dict):
content = {
'content': content
}
if isinstance(renderer, str):
content = ctx.render_mako_template(renderer, **content)
else:
content = renderer(ctx, **content)
if content is None:
content = ''
elif isinstance(content, unicode):
content = content.encode('utf-8')
raise HTTPContent(content)
# Return the content.
except HTTPContent, payload:
content = payload.content
if 'Content-Type' not in ctx.response_headers:
ctx.response_headers['Content-Type'] = 'text/html; charset=utf-8'
ctx.response_headers['Content-Length'] = str(len(content))
start_response(('%d %s\r\n' % ctx._status), get_response_headers())
if http_method == 'HEAD':
return []
return [content]
def _url2path(self, url):
"""Convert a server-side URL into a client-side path."""
path = urlunquote(urlparse(url).path)
root = self._url_p.path
return path[len(root) - 1:].decode("utf8")
def fileurl_to_path(url):
assert url.startswith('file:'), ('Illegal scheme:%s' % url)
url = '/' + url[len('file:'):].lstrip('/')
return urlunquote(url)
def unquote(self, s):
# Don't unquote slashes if they're already quoted.
return "%2F".join(urlunquote(part) for part in s.split("%2F"))
def exists(self, name):
return urlunquote(name) in self._litfile.paths
def handle_http_request(env,
start_response,
dict=dict,
isinstance=isinstance,
urlunquote=urlunquote,
unicode=unicode,
get_response_headers=lambda: None):
reqlocal.template_error_traceback = None
try:
http_method = env['REQUEST_METHOD']
ssl_mode = env['wsgi.url_scheme'] == 'https'
if http_method == 'OPTIONS':
start_response(*RESPONSE_OPTIONS)
return []
if http_method not in SUPPORTED_HTTP_METHODS:
start_response(*RESPONSE_NOT_IMPLEMENTED)
return []
_path_info = env['PATH_INFO']
if isinstance(_path_info, unicode):
_args = [arg for arg in _path_info.split(u'/') if arg]
else:
_args = [
unicode(arg, 'utf-8', 'strict')
for arg in _path_info.split('/') if arg
]
kwargs = {}
for part in [
sub_part for part in env['QUERY_STRING'].lstrip('?').split('&')
for sub_part in part.split(';')
]:
if not part:
continue
part = part.split('=', 1)
if len(part) == 1:
value = None
else:
value = part[1]
key = urlunquote(part[0].replace('+', ' '))
if value:
value = unicode(urlunquote(value.replace('+', ' ')), 'utf-8',
'strict')
else:
value = None
if key in kwargs:
_val = kwargs[key]
if isinstance(_val, list):
_val.append(value)
else:
kwargs[key] = [_val, value]
continue
kwargs[key] = value
ctx = Context(env, ssl_mode)
router = handle_http_request.router
if router:
_info = router(ctx, _args, kwargs)
if not _info:
logging.error("No handler found for: %s" % _path_info)
raise NotFound
name, args = _info
else:
if _args:
name = _args[0]
args = _args[1:]
else:
name = '/'
args = ()
if name not in HANDLERS:
logging.error("Handler not found: %s" % name)
raise NotFound
handler, renderers, config = HANDLERS[name]
json = config['json']
# Parse the POST body if it exists and is of a known content type.
if http_method == 'POST':
content_type = env.get('CONTENT-TYPE', '')
if not content_type:
content_type = env.get('CONTENT_TYPE', '')
if ';' in content_type:
content_type = content_type.split(';', 1)[0]
if json or content_type == 'application/json':
payload = json_decode(env['wsgi.input'].read())
if json and not (json is True):
kwargs[json] = payload
else:
kwargs.update(payload)
elif content_type in VALID_REQUEST_CONTENT_TYPES:
post_environ = env.copy()
post_environ['QUERY_STRING'] = ''
if config['post_encoding']:
ctx.request_body = env['wsgi.input'].read()
env['wsgi.input'] = StringIO(ctx.request_body)
post_encoding = config['post_encoding']
else:
post_encoding = 'utf-8'
post_data = FieldStorage(environ=post_environ,
fp=env['wsgi.input'],
keep_blank_values=True).list or []
for field in post_data:
key = field.name
if field.filename:
if config['blob']:
value = parse_blob_info(field)
else:
value = field
else:
value = unicode(field.value, post_encoding, 'strict')
if key in kwargs:
_val = kwargs[key]
if isinstance(_val, list):
_val.append(value)
else:
kwargs[key] = [_val, value]
continue
kwargs[key] = value
def get_response_headers():
# Figure out the HTTP headers for the response ``cookies``.
cookie_output = SimpleCookie()
for name, values in ctx._response_cookies.iteritems():
name = str(name)
cookie_output[name] = values.pop('value')
cur = cookie_output[name]
for key, value in values.items():
if key == 'max_age':
key = 'max-age'
if key not in COOKIE_KEY_NAMES:
continue
cur[key] = value
if cookie_output:
raw_headers = ctx._raw_headers + [
('Set-Cookie', ck.split(' ', 1)[-1])
for ck in str(cookie_output).split('\r\n')
]
else:
raw_headers = ctx._raw_headers
str_headers = []
new_header = str_headers.append
for k, v in raw_headers:
if isinstance(k, unicode):
k = k.encode('utf-8')
if isinstance(v, unicode):
v = v.encode('utf-8')
new_header((k, v))
return str_headers
if 'submit' in kwargs:
del kwargs['submit']
if 'callback' in kwargs:
ctx.json_callback = kwargs.pop('callback')
if env.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest':
ctx.ajax_request = 1
if '__ajax__' in kwargs:
ctx.ajax_request = 1
del kwargs['__ajax__']
if config['ssl'] and RUNNING_ON_GOOGLE_SERVERS and not ssl_mode:
raise NotFound
if config['xsrf']:
if 'xsrf' not in kwargs:
raise AuthError("XSRF token not present.")
provided_xsrf = kwargs.pop('xsrf')
if not secure_string_comparison(provided_xsrf, ctx.xsrf_token):
raise AuthError("XSRF tokens do not match.")
if config['admin'] and not ctx.is_admin:
raise NotFound
if (not config['anon']) and (not ctx.user_id):
if ctx.ajax_request:
ctx.response_headers['Content-Type'] = 'application/json'
raise HTTPContent(
encode_json({
"error": {
"type": "AuthError",
"redirect": ctx.get_login_url()
}
}))
raise Redirect(ctx.get_login_url())
# Try and respond with the result of calling the handler.
content = handler(ctx, *args, **kwargs)
for renderer in renderers:
if ctx.end_pipeline:
break
if content is None:
content = {'content': ''}
elif not isinstance(content, dict):
content = {'content': content}
if isinstance(renderer, str):
content = ctx.render_mako_template(renderer, **content)
else:
content = renderer(ctx, **content)
if content is None:
content = ''
elif isinstance(content, unicode):
content = content.encode('utf-8')
raise HTTPContent(content)
# Return the content.
except HTTPContent, payload:
content = payload.content
if 'Content-Type' not in ctx.response_headers:
ctx.response_headers['Content-Type'] = 'text/html; charset=utf-8'
ctx.response_headers['Content-Length'] = str(len(content))
start_response(('%d %s\r\n' % ctx._status), get_response_headers())
if http_method == 'HEAD':
return []
return [content]
def handle_http_request(env,
start_response,
dict=dict,
isinstance=isinstance,
urlunquote=urlunquote,
unicode=unicode,
get_response_headers=lambda: None):
reqlocal.template_error_traceback = None
try:
http_method = env['REQUEST_METHOD']
ssl_mode = env['wsgi.url_scheme'] == 'https'
if RUNNING_ON_GOOGLE_SERVERS and env['HTTP_HOST'] in ALTERNATIVE_HOSTS:
if SSL_ONLY or ssl_mode:
dest = 'https://' + CANONICAL_HOST
else:
dest = 'http://' + CANONICAL_HOST
if http_method == 'GET':
dest += env['PATH_INFO']
query_string = env['QUERY_STRING']
if query_string:
dest += '?' + query_string
else:
dest += '/'
start_response(STATUS_301, [('Location', dest)])
return []
if http_method == 'OPTIONS':
start_response(*RESPONSE_OPTIONS)
return []
if http_method not in SUPPORTED_HTTP_METHODS:
start_response(*RESPONSE_NOT_IMPLEMENTED)
return []
_path_info = env['PATH_INFO']
if isinstance(_path_info, unicode):
_args = [arg for arg in _path_info.split(u'/') if arg]
else:
_args = [
unicode(arg, 'utf-8', 'strict')
for arg in _path_info.split('/') if arg
]
if _args:
name = _args[0]
args = _args[1:]
else:
name = '/'
args = ()
routed = 0
if name not in HANDLERS:
router = handle_http_request.router
if router:
_info = router(env, _args)
if not _info:
logging.error("No handler found for: %s" % _path_info)
raise NotFound
name, args = _info
routed = 1
else:
logging.error("Handler not found: %s" % name)
raise NotFound
handler, renderers, config = HANDLERS[name]
kwargs = {}
ctx = Context(name, env, ssl_mode)
ctx.was_routed = routed
for part in [
sub_part for part in env['QUERY_STRING'].lstrip('?').split('&')
for sub_part in part.split(';')
]:
if not part:
continue
part = part.split('=', 1)
if len(part) == 1:
value = None
else:
value = part[1]
key = urlunquote(part[0].replace('+', ' '))
if value:
value = unicode(urlunquote(value.replace('+', ' ')), 'utf-8',
'strict')
else:
value = None
kwargs[key] = value
# Parse the POST body if it exists and is of a known content type.
if http_method == 'POST':
content_type = env.get('CONTENT-TYPE', '')
if not content_type:
content_type = env.get('CONTENT_TYPE', '')
if ';' in content_type:
content_type = content_type.split(';', 1)[0]
if content_type in VALID_REQUEST_CONTENT_TYPES:
post_environ = env.copy()
post_environ['QUERY_STRING'] = ''
if config['post_encoding']:
ctx.request_body = env['wsgi.input'].read()
env['wsgi.input'] = StringIO(ctx.request_body)
post_encoding = config['post_encoding']
else:
post_encoding = 'utf-8'
post_data = FieldStorage(environ=post_environ,
fp=env['wsgi.input'],
keep_blank_values=True).list or []
for field in post_data:
key = field.name
if field.filename:
if config['blob']:
value = parse_file_info(field)
else:
value = field
else:
value = unicode(field.value, post_encoding, 'strict')
kwargs[key] = value
elif content_type == 'application/json':
kwargs.update(json_decode(env['wsgi.input'].read()))
def get_response_headers():
# Figure out the HTTP headers for the response ``cookies``.
cookie_output = SimpleCookie()
for name, values in ctx._response_cookies.iteritems():
name = str(name)
cookie_output[name] = values.pop('value')
cur = cookie_output[name]
for key, value in values.items():
if key == 'max_age':
key = 'max-age'
if key not in COOKIE_KEY_NAMES:
continue
cur[key] = value
if cookie_output:
raw_headers = ctx._raw_headers + [
('Set-Cookie', ck.split(' ', 1)[-1])
for ck in str(cookie_output).split('\r\n')
]
else:
raw_headers = ctx._raw_headers
str_headers = []
new_header = str_headers.append
for k, v in raw_headers:
if isinstance(k, unicode):
k = k.encode('utf-8')
if isinstance(v, unicode):
v = v.encode('utf-8')
new_header((k, v))
return str_headers
if 'submit' in kwargs:
del kwargs['submit']
if RUNNING_ON_GOOGLE_SERVERS:
if config['cron']:
if not env.get('HTTP_X_APPENGINE_CRON'):
ctx.check_task_auth(kwargs)
elif config['task']:
if not env.get('HTTP_X_APPENGINE_TASKNAME'):
ctx.check_task_auth(kwargs)
elif config['ssl'] and not ssl_mode:
raise NotFound
if config['xsrf']:
if 'xsrf' not in kwargs:
raise AuthorisationError("XSRF token not present.")
provided_xsrf = kwargs.pop('xsrf')
if not secure_string_comparison(provided_xsrf, ctx.xsrf_token):
raise AuthorisationError("XSRF token does not match.")
if config['admin'] and not ctx.is_admin:
if ctx.user_id:
raise NotFound
raise Redirect(ctx.get_login_url())
if (not config['anon']) and (not ctx.user_id):
raise Redirect(ctx.get_login_url())
# Try and respond with the result of calling the handler.
content = handler(ctx, *args, **kwargs)
for renderer in renderers:
if ctx.end_pipeline:
break
if content is None:
content = {'content': ''}
elif not isinstance(content, dict):
content = {'content': content}
if isinstance(renderer, str):
content = ctx.render_mako_template(renderer, **content)
else:
content = renderer(ctx, **content)
if content is None:
content = ''
elif isinstance(content, unicode):
content = content.encode('utf-8')
elif not isinstance(content, str):
content = str(content)
if ctx.after_runners:
for func in ctx.after_runners:
func(ctx)
cache = config['cache']
if cache and http_method != 'POST':
cache, duration = cache
etag = md5(content).hexdigest()
if cache == 'public':
ctx.cache_response(etag, duration)
elif cache == 'private':
ctx.cache_private_response(etag, duration)
else:
ctx.do_not_cache_response()
raise HTTPContent(content)
# Return the content.
except HTTPContent, payload:
content = payload.content
if 'Content-Type' not in ctx.response_headers:
ctx.response_headers['Content-Type'] = 'text/html; charset=utf-8'
ctx.response_headers['Content-Length'] = str(len(content))
start_response(('%d %s\r\n' % ctx._status), get_response_headers())
if http_method == 'HEAD':
return []
return [content]
