def get_lyrics(self, title, artist):
lyrics = ''
artist = artist.replace(' ', '_').lower()
artist = normalize('NFD', artist).encode('ascii', 'ignore')
title = title.replace(' ', '_').lower()
title = normalize('NFD', title).encode('ascii', 'ignore')
url = ('http://www.lyricsmode.com/lyrics/%s/%s/%s.html' %
(urllib_quote(artist.decode('utf-8'))[0],
urllib_quote(artist.decode('utf-8')),
urllib_quote(title.decode('utf-8'))))
try:
page = self.get_html(url)
except HTTPError:
page = ''
clean_reg = re_compile('<.*?>')
for txt in re_findall(
'(?s)<p id="lyrics_text" ' +
'class="ui-annotatable">(.*?)</p>', str(page)):
txt = re_sub(clean_reg, '', txt)
txt = txt.replace('\\\'', "'")
txt = txt.replace('\\n', '\n')
lyrics = txt
if lyrics != '':
return lyrics
else:
return None
def get_lyrics(self, title, artist):
lyrics = ''
artist = artist.replace(' ', '_')
artist = normalize('NFD', artist).encode('ascii', 'ignore')
title = title.replace(' ', '_')
title = normalize('NFD', title).encode('ascii', 'ignore')
url = ('http://lyrics.wikia.com/wiki/%s:%s' % (urllib_quote(
artist.decode('utf-8')), urllib_quote(title.decode('utf-8'))))
try:
page = self.get_html(url)
except HTTPError:
page = ''
soup = BeautifulSoup(page, 'html.parser')
rew = soup.find('div', {'class': 'lyricbox'})
if rew is None:
return None
else:
for txt in re_findall('(?s)</script>(.*?)<!--', str(rew)):
txt = txt.replace('<br/>', '\n')
lyrics = txt
if lyrics != '':
return lyrics
else:
return None
def apiRequest(self, movie='', useYear=True):
if type(movie) != dict: return False
if sys.version_info[0] == 3: title, year, gotError = urllib_quote(movie['title'].encode('utf8')), urllib_quote(movie['year'].encode('utf8')) if useYear else '', False # py3
else: title, year, gotError = urllib_quote(movie['title']).encode('utf8'), urllib_quote(movie['year']).encode('utf8') if useYear else '', False #py2
for api_name, api in _iteritems(self.api_list):
try:
if sys.version_info[0] == 3: data = json_loads( urllib2_urlopen(api['url'] % (title, year)).read().decode('utf-8') ) # py3
else: data = json_loads( urllib2_urlopen(api['url'] % (title, year)).read() ) # py2
if api['response_status_key'] in data and data[api['response_status_key']] == api['status_success_value']:
dataRated = data[api['response_rated_key']].upper() if api['response_rated_key'] in data else 'UNKNOWN'
if dataRated not in self.rated_aliases:
self.log('Unrecognized Rated: "%s" - API: "%s" - Movie: "%s" - Year: "%s"' % (dataRated, api_name, title, year))
self.rated_aliases[dataRated] = 'UNKNOWN'
movie['rated'] = dataRated
except (urllib2_HTTPError, urllib2_URLError) as e:
gotError = True
#print('\r%s' % self.clear_line),
self.rePrint()
#print('\r%s' % self.clear_line, end='')
self.log('\rError occurred while checking the rated for "%s"' % movie['basename'])
if self.rated_aliases[movie['rated']] != 'UNKNOWN':
break # Found the Rated, so let's exit the loop
if not self.args.quiet and not gotError:
self.rePrint('Live :%s%s ---> %s' % ((10-len(movie['rated']))*' ', movie['rated'], movie['basename']), False)
self.rated_results[self.rated_aliases[movie['rated']]].append(movie)
return True
def escape(self):
'''Make sure that the path is correctly escaped'''
upath = urllib_unquote(self.path).encode('utf-8')
self.path = urllib_quote(upath, safe=URL.PATH)
if py2:
self.path = self.path.decode('utf-8')
# Safe characters taken from:
# http://tools.ietf.org/html/rfc3986#page-50
uquery = urllib_unquote(self.query).encode('utf-8')
self.query = urllib_quote(uquery, safe=URL.QUERY)
if py2:
self.query = self.query.decode('utf-8')
# The safe characters for URL parameters seemed a little more vague.
# They are interpreted here as *pchar despite this page, since the
# updated RFC seems to offer no replacement
# http://tools.ietf.org/html/rfc3986#page-54
uparams = urllib_unquote(self.params).encode('utf-8')
self.params = urllib_quote(uparams, safe=URL.QUERY)
if py2:
self.params = self.params.decode('utf-8')
if self.userinfo:
uuserinfo = urllib_unquote(self.userinfo).encode('utf-8')
self.userinfo = urllib_quote(uuserinfo, safe=URL.USERINFO)
if py2:
self.userinfo = self.userinfo.decode('utf-8')
return self
def export_data_as_hdfs_file(self, snippet, target_file, overwrite):
db = self._get_db(snippet, interpreter=self.interpreter)
handle = self._get_handle(snippet)
max_rows = DOWNLOAD_ROW_LIMIT.get()
max_bytes = DOWNLOAD_BYTES_LIMIT.get()
upload(target_file, handle, self.request.user, db, self.request.fs, max_rows=max_rows, max_bytes=max_bytes)
return '/filebrowser/view=%s' % urllib_quote(urllib_quote(target_file.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)) # Quote twice, because of issue in the routing on client
def user_playlist(self, user, playlist_id=None, fields=None):
""" Gets playlist of a user
Parameters:
- user - the id of the user
- playlist_id - the id of the playlist
- fields - which fields to return
"""
if playlist_id is None:
url = 'users/%s/starred' % urllib_quote(user)
return self._get(url, fields=fields)
plid = self._get_id('playlist', playlist_id)
url = 'users/%s/playlists/%s' % (urllib_quote(user), plid)
return self._get(url, fields=fields)
def forward(self,
query: str,
bounds=None,
region=None) -> Optional[GeoCoderResult]:
query = query.lower().strip()
if not query:
return None
url = ["http://maps.googleapis.com/maps/api/geocode/json?"]
if region:
url += ["®ion=%s" % region]
if bounds:
# &bounds=34.172684,-118.604794|34.236144,-118.500938
# bounds = ("34.172684,-118.604794", "34.236144,-118.500938")
url += ["&bounds=%s|%s" % bounds]
url += ["&address=%s" % urllib_quote(query)]
if self.key:
url += ["&key=%s" % urllib_quote(self.key)]
if self.language:
url += ["&language=%s" % urllib_quote(self.language)]
code, response = self.get("".join(url))
if code != 200:
raise GeoCoderError("%s: %s" % (code, response))
try:
r = orjson.loads(response)
except ValueError:
raise GeoCoderError("Cannot decode result")
if r["status"] != "OK":
return None
for rr in r["results"]:
lon = self.get_path(rr, "geometry.location.lng")
lat = self.get_path(rr, "geometry.location.lat")
if not rr.get("address_components"):
return None
path = [x["short_name"] for x in rr["address_components"]]
if "postal_code" in rr["address_components"][-1]["types"]:
path = path[:-1]
path.reverse()
is_exact = (self.get_path(
rr, "GeoObject.metaDataProperty.GeocoderMetaData.precision") ==
"exact")
return GeoCoderResult(
exact=is_exact,
query=query,
path=path,
lon=lon,
lat=lat,
id=None,
scope="google",
address=None,
)
def _doi_validate_helper(self, db_object):
doi = db_object.doi
url = 'http://dx.doi.org/' + urllib_quote(doi)
resp = requests.get(url)
print('Validating DOIs in library')
# Should probably get rid of these print statements
# in favor of a progress bar?
# print(resp.status_code)
# print('Entered URL: ' + url)
# print('Response URL: ' + resp.url)
# print()
if resp.ok:
db_object.valid_doi = 1
# ScienceDirect often does not allow requests to go directly to an article page.
# It will return status code 404, but will still end up going to a ScienceDirect
# URL if the DOI is valid and corresponds with a ScienceDirect-hosted paper
elif resp.status_code == 404:
if 'sciencedirect' in resp.url or 'ScienceDirect' in resp.content:
db_object.valid_doi = 1
else:
db_object.valid_doi = 0
else:
db_object.valid_doi = 0
def _quote_query(query):
"""Turn a dictionary into a query string in a URL, with keys
in alphabetical order."""
return "&".join("%s=%s" % (
k, urllib_quote(
str(query[k]).encode('utf-8'), safe='~'))
for k in sorted(query))
def _dissect_and_escape_url(cls, parsed_url):
""" Splits a full URL into parts that can be used directly by the client to
establish a connection and fetch the page. IDNs are converted to punycode
and non-ASCII characters in the path and query are url-encoded.
"""
assert parsed_url.scheme in cls.DEFAULT_PORTS.keys()
if not ':' in parsed_url.netloc:
host = parsed_url.netloc
port = ''
else:
host, port = parsed_url.netloc.split(':')
port = int(port) if port != '' else cls.DEFAULT_PORTS[parsed_url.scheme]
# The fragment part (after #) can be discarded. It's only relevant to a client.
path = parsed_url.path if parsed_url.path != '' else '/'
path_and_query = path + ('?' + parsed_url.query if parsed_url.query != '' else '')
# TODO: Omitting '=&?/' seems enough for most cases but it's probably not a complete set of special.
# characters. This can be made more robust.
escaped_host = host.encode('idna').decode('ascii')
escaped_path_and_query = urllib_quote(path_and_query, '=&?/')
return (escaped_host, port, escaped_path_and_query)
def _get_char_id_ps(self, font, ccode):
"""
Return a unique id for the given font and character-code set (for tex).
"""
ps_name = font.get_ps_font_info()[2]
char_id = urllib_quote('%s-%d' % (ps_name, ccode))
return char_id
def export_large_data_to_hdfs(self, notebook, snippet, destination):
response = self._get_current_statement(notebook, snippet)
session = self._get_session(notebook, snippet['type'])
query = self._prepare_hql_query(snippet, response.pop('statement'),
session)
if 'select' not in query.hql_query.strip().lower():
raise PopupException(
_('Only SELECT statements can be saved. Provided statement: %(query)s'
) % {'query': query.hql_query})
hql = '''
DROP TABLE IF EXISTS `%(table)s`;
CREATE TABLE `%(table)s` ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\\t'
ESCAPED BY '\\\\'
LINES TERMINATED BY '\\n'
STORED AS TEXTFILE LOCATION '%(location)s'
AS
%(hql)s;
ALTER TABLE `%(table)s` SET TBLPROPERTIES('EXTERNAL'='TRUE');
DROP TABLE IF EXISTS `%(table)s`;
''' % {
'table': _get_snippet_name(
notebook, unique=True, table_format=True),
'location': self.request.fs.netnormpath(destination),
'hql': query.hql_query
}
success_url = '/filebrowser/view=%s' % urllib_quote(
destination.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
return hql, success_url
def _escape(path):
"""Handle any characters in the path that should be escaped."""
# FIXME: web browsers don't appear to blindly quote every character
# when converting filenames to files. Instead of using urllib's default
# rules, we allow a small list of other characters through un-escaped.
# It's unclear if this is the best possible solution.
return urllib_quote(path, safe='/+:')
def update_flume_config(self, cluster_name, config_name, config_value):
service = 'FLUME-1'
cluster = self._get_cluster(cluster_name)
roleConfigGroup = [
role['roleConfigGroupRef']['roleConfigGroupName']
for role in self._get_roles(cluster['name'], service, 'AGENT')
]
data = {
u'items': [{
u'url':
u'/api/v8/clusters/%(cluster_name)s/services/%(service)s/roleConfigGroups/%(roleConfigGroups)s/config?message=Updated%20service%20and%20role%20type%20configurations.'
.replace('%(cluster_name)s',
urllib_quote(cluster['name'])).replace(
'%(service)s',
service).replace('%(roleConfigGroups)s',
roleConfigGroup[0]),
u'body': {
u'items': [{
u'name': config_name,
u'value': config_value
}]
},
u'contentType':
u'application/json',
u'method':
u'PUT'
}]
}
return self.batch(items=data)
def user_playlist_change_details(self,
user,
playlist_id,
name=None,
public=None,
collaborative=None,
description=None):
""" Changes a playlist's name and/or public/private state
Parameters:
- user - the id of the user
- playlist_id - the id of the playlist
- name - optional name of the playlist
- public - optional is the playlist public
- collaborative - optional is the playlist collaborative
- description - optional description of the playlist
"""
data = {}
if isinstance(name, six.string_types):
data['name'] = name
if isinstance(public, bool):
data['public'] = public
if isinstance(collaborative, bool):
data['collaborative'] = collaborative
if isinstance(description, six.string_types):
data['description'] = description
url = 'users/%s/playlists/%s' % (urllib_quote(user), playlist_id)
return self._put(url, payload=data)
def execute(self, sql=None, args=None, nodb=False, post=None, extra=None):
def q(v):
# @todo: quote dates
if isinstance(v, str):
return "'%s'" % (v.replace("\\", "\\\\").replace("'", "\\'"))
else:
return str(v)
qs = []
if not nodb:
qs += ["database=%s" % config.clickhouse.db]
if extra:
qs += ["%s=%s" % (k, v) for k, v in extra]
if sql:
if args:
sql = sql % tuple(q(v) for v in args)
if post:
qs += ["query=%s" % urllib_quote(sql.encode("utf8"))]
else:
post = sql.encode("utf8")
url = "http://%s/?%s" % (random.choice(self.addresses), "&".join(qs))
code, headers, body = fetch_sync(
url,
method="POST",
body=post,
user=self.user,
password=self.password,
connect_timeout=config.clickhouse.connect_timeout,
request_timeout=config.clickhouse.request_timeout,
)
if code != 200:
raise ClickhouseError("%s: %s" % (code, body))
return [smart_text(row).split("\t") for row in body.splitlines()]
def cat_file(self, path, rev='tip', base_commit_id=None):
# If the base commit id is provided it should override anything
# that was parsed from the diffs.
if rev != PRE_CREATION and base_commit_id is not None:
rev = base_commit_id
if rev == HEAD or rev == UNKNOWN:
rev = "tip"
elif rev == PRE_CREATION:
rev = ""
for rawpath in ["raw-file", "raw", "hg-history"]:
try:
url = self.FULL_FILE_URL % {
'url': self.path_stripped,
'rawpath': rawpath,
'revision': rev,
'quoted_path': urllib_quote(path.lstrip('/')),
}
return self.get_file_http(url, path, rev)
except Exception:
# It failed. Error was logged and we may try again.
pass
raise FileNotFoundError(path, rev)
def test_python_3_7_quoting():
"""
This test verifies that the userinfo encoding is identical with the defaul urllib encoding
"""
from urllib.parse import quote as urllib_quote
quot = quoting.PYTHON_3_7_QUOTING
# Control characters
ascii_bytes = bytes(range(0, 32))
ascii_str = ascii_bytes.decode("latin1")
utf8_bytes = ascii_str.encode("utf-8")
expected = "%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F"
actual = quote(utf8_bytes, quot).decode("utf-8")
assert expected == actual
# Printable
ascii_bytes = bytes(range(32, 127))
ascii_str = ascii_bytes.decode("latin1")
utf8_bytes = ascii_str.encode("utf-8")
# expected = "%20!%22%23$%&'()*+,-.%2F0123456789%3A%3B%3C%3D%3E%3F%40ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~"
expected = urllib_quote(utf8_bytes)
actual = quote(utf8_bytes, quot).decode("utf-8")
assert expected == actual
def sync(cls):
"""
Synchronize the local RAT device registry with upstream data
@returns: success True|False
"""
settings = current.deployment_settings
vseturl = settings.get_custom("dcc_rules_url", VSETURL)
vsetid = settings.get_custom("dcc_rules_id", VSETID)
# Retrieve the value set list
valuesets = cls.get(vseturl)
if valuesets is None:
return False
# Get the current hash of the target value set
vsethash = None
for item in valuesets:
if item.get("id") == vsetid:
vsethash = item.get("hash")
break
if not vsethash:
return False
# Construct the source URI
source = "/".join((vseturl, urllib_quote(vsethash)))
# Retrieve the value set and update the device registry
valueset = cls.get(source)
if valueset:
return cls.update_device_registry(source, valueset)
return True
def _get_char_id_ps(self, font, ccode):
"""
Return a unique id for the given font and character-code set (for tex).
"""
ps_name = font.get_ps_font_info()[2]
char_id = urllib_quote('%s-%d' % (ps_name, ccode))
return char_id
def read_url(self, path, offset=0, length=None, bufsize=None):
"""
read(path, offset, length[, bufsize]) -> data
Read data from a file.
"""
path = self.strip_normpath(path)
params = self._getparams()
params['op'] = 'OPEN'
params['offset'] = long(offset)
if length is not None:
params['length'] = long(length)
if bufsize is not None:
params['bufsize'] = bufsize
if self._security_enabled:
token = self.get_delegation_token(self.user)
if token:
params['delegation'] = token
# doas should not be present with delegation token as the token includes the username
# https://hadoop.apache.org/docs/r1.0.4/webhdfs.html
if 'doas' in params:
del params['doas']
if 'user.name' in params:
del params['user.name']
quoted_path = urllib_quote(smart_str(path))
return self._client._make_url(quoted_path, params)
def execute(self, http_method, path, params=None, data=None, headers=None, allow_redirects=False, urlencode=True,
files=None, stream=False, clear_cookies=False, timeout=conf.REST_CONN_TIMEOUT.get()):
"""
Submit an HTTP request.
@param http_method: GET, POST, PUT, DELETE
@param path: The path of the resource. Unsafe characters will be quoted.
@param params: Key-value parameter data.
@param data: The data to attach to the body of the request.
@param headers: The headers to set for this request.
@param allow_redirects: requests should automatically resolve redirects.
@param urlencode: percent encode paths.
@param files: for posting Multipart-Encoded files
@param stream: Bool to stream the response
@param clear_cookies: flag to force clear any cookies set in the current session
@return: The result of urllib2.urlopen()
"""
if urlencode:
path = urllib_quote(smart_str(path))
url = self._make_url(path, params)
if http_method in ("GET", "DELETE"):
if data is not None:
self.logger.warn("GET and DELETE methods do not pass any data. Path '%s'" % path)
data = None
if headers and 'timeout' in headers:
timeout = int(headers['timeout'])
LOG.debug("Overriding timeout xxx via header value %d" % timeout)
request_kwargs = {'allow_redirects': allow_redirects, 'timeout': timeout}
if headers:
request_kwargs['headers'] = headers
if data:
request_kwargs['data'] = data
if files:
request_kwargs['files'] = files
if stream:
request_kwargs['stream'] = True
if self._cookies and not clear_cookies:
request_kwargs['cookies'] = self._cookies
if clear_cookies:
self._session.cookies.clear()
try:
resp = getattr(self._session, http_method.lower())(url, **request_kwargs)
if resp.status_code >= 300:
resp.raise_for_status()
raise exceptions.HTTPError(response=resp)
self._cookies = resp.cookies # Cache request cookie for the next http_client call.
return resp
except (exceptions.ConnectionError,
exceptions.HTTPError,
exceptions.RequestException,
exceptions.URLRequired,
exceptions.TooManyRedirects) as ex:
raise self._exc_class(ex)
def user_playlist_reorder_tracks(self,
user,
playlist_id,
range_start,
insert_before,
range_length=1,
snapshot_id=None):
""" Reorder tracks in a playlist
Parameters:
- user - the id of the user
- playlist_id - the id of the playlist
- range_start - the position of the first track to be reordered
- range_length - optional the number of tracks to be reordered (default: 1)
- insert_before - the position where the tracks should be inserted
- snapshot_id - optional playlist's snapshot ID
"""
plid = self._get_id('playlist', playlist_id)
payload = {
"range_start": range_start,
"range_length": range_length,
"insert_before": insert_before
}
if snapshot_id:
payload["snapshot_id"] = snapshot_id
url = 'users/%s/playlists/%s/tracks' % (urllib_quote(user), plid)
return self._put(url, payload=payload)
def user_playlist_remove_specific_occurrences_of_tracks(
self, user, playlist_id, tracks, snapshot_id=None):
""" Removes all occurrences of the given tracks from the given playlist
Parameters:
- user - the id of the user
- playlist_id - the id of the playlist
- tracks - an array of objects containing Spotify URIs of the tracks to remove with their current positions in the playlist. For example:
[ { "uri":"4iV5W9uYEdYUVa79Axb7Rh", "positions":[2] },
{ "uri":"1301WleyT98MSxVHPZCA6M", "positions":[7] } ]
- snapshot_id - optional id of the playlist snapshot
"""
plid = self._get_id('playlist', playlist_id)
ftracks = []
for tr in tracks:
ftracks.append({
"uri": self._get_uri("track", tr["uri"]),
"positions": tr["positions"],
})
payload = {"tracks": ftracks}
if snapshot_id:
payload["snapshot_id"] = snapshot_id
url = 'users/%s/playlists/%s/tracks' % (urllib_quote(user), plid)
return self._delete(url, payload=payload)
def root_page():
log('Root page is requested')
files = []
items = storage.enumerate_files()
now = time_time()
for item in items:
full_disk_filename = item['full_disk_filename']
url_filename = item['url_filename']
display_filename = item['display_filename']
modified_unixtime = get_file_modified_unixtime(full_disk_filename)
files.append(
{
'display_filename': display_filename,
'url': URLPREFIX + urllib_quote(url_filename),
'url_filename': url_filename,
'size': format_size(os_path.getsize(full_disk_filename)),
'age': format_age(now - modified_unixtime),
'sortBy': now - modified_unixtime,
})
files = sorted(files, key=lambda item: item['sortBy'])
return {
'title': 'Limbo: the file sharing lightweight service',
'h1': 'Limbo. The file sharing lightweight service',
'files': files,
}
def script_link1(script,linktext):
''' if script exists return a plain text name as link; else return "no" icon, plain text name
used in translate (all scripts should exist, missing script is an error).
'''
if os.path.exists(script):
return '<a href="/srcfiler/?src=%s" target="_blank">%s</a>'%(urllib_quote(script.encode("utf-8")),linktext)
else:
return '<img src="/media/admin/img/icon-no.gif"></img> %s'%linktext
def user(self, user):
""" Gets basic profile information about a Spotify User
Parameters:
- user - the id of the usr
"""
url = 'users/%s' % urllib_quote(user)
return self._get(url)
def mongo_connection_args(self):
"""
Mongo connection arguments. Suitable to pass to
pymongo.connect and mongoengine.connect
"""
if not hasattr(self, "_mongo_connection_args"):
self._mongo_connection_args = {
"db": self.mongo.db,
"username": self.mongo.user,
"password": self.mongo.password,
}
if self.mongo.app_name:
self._mongo_connection_args["appname"] = self.mongo.app_name
if self.mongo.retry_writes:
self._mongo_connection_args["retryWrites"] = True
has_credentials = self.mongo.user or self.mongo.password
if has_credentials:
self._mongo_connection_args[
"authentication_source"] = self.mongo.db
hosts = self.mongo.addresses
if self.mongo.rs:
self._mongo_connection_args["replicaSet"] = self.mongo.rs
self._mongo_connection_args[
"readPreference"] = "secondaryPreferred"
elif len(hosts) > 1:
raise ValueError("Replica set name must be set")
if self.mongo.max_idle_time:
self._mongo_connection_args[
"maxIdleTimeMS"] = self.mongo.max_idle_time * 1000
url = ["mongodb://"]
if has_credentials:
url += [
"%s:%s@" % (urllib_quote(
self.mongo.user), urllib_quote(self.mongo.password))
]
url += [",".join(str(h) for h in hosts)]
url += ["/%s" % self.mongo.db]
self._mongo_connection_args["host"] = "".join(url)
if self.metrics.enable_mongo_hist:
from noc.core.mongo.monitor import MongoCommandSpan
self._mongo_connection_args["event_listeners"] = [
MongoCommandSpan()
]
return self._mongo_connection_args
def evaluate(self, context=None):
uri = self.get_argument(context, cls=str)
if uri is None:
return ''
try:
return urllib_quote(uri, safe=''.join(chr(cp) for cp in range(32, 127)))
except TypeError:
self.wrong_type("the argument must be a string: %r" % uri)
def _create_link_dict(self, link):
# type: (Union[Tuple[str, Dict[str, str]], str]) -> Dict[str, str]
if isinstance(link, (list, tuple)):
href, extra_attributes = link
else:
href, extra_attributes = link, {}
link_dict = {'href': urllib_quote(href) if self._quote else href}
link_dict.update(extra_attributes)
return link_dict
def script_link1(script, linktext):
''' if script exists return a plain text name as link; else return "no" icon, plain text name
used in translate (all scripts should exist, missing script is an error).
'''
if os.path.exists(script):
return '<a href="/srcfiler/?src=%s" target="_blank">%s</a>' % (
urllib_quote(script.encode("utf-8")), linktext)
else:
return '<img src="/media/admin/img/icon-no.gif"></img> %s' % linktext
def encode_header_value(value: Union[str, datetime, Number],
params: Dict[str, Union[str, datetime, Number]], *,
encoding: str = 'ascii', always_quote: bool = False) -> bytes:
"""
Encode a structured header value for transmission over the network.
If a parameter value cannot be encoded to the given encoding, the :rfc:`5987` method is used to
encode the value in an alternate field using :rfc:`2231` encoding where the field name ends
with ``*``. The original field will have an urlencoded version of the value.
Any datetimes in the ``value`` argument or any of the parameter values will be formatted
as defined by :rfc:`822`. Any numeric values will be converted to strings. If a parameter value
is ``False`` or ``None``, the parameter is omitted entirely from the output. If the value is
``True``, only the key will be included in the output (without any ``=``). All other value
types are disallowed.
:param value: the main value of the header
:param params: a dictionary of parameter names and values
:param encoding: the character encoding to use (either ``ascii`` or ``iso-8859-1``)
:param always_quote: always enclose the parameter values in quotes, even if it's unnecessary
:return: the encoded bytestring
"""
def transform_value(val):
if isinstance(val, str):
return val
elif isinstance(val, datetime):
return format_datetime(val.astimezone(timezone.utc), usegmt=True)
else:
return str(val)
assert check_argument_types()
buffer = transform_value(value).encode(encoding)
for key, value in params.items():
key = key.encode(encoding)
buffer += b'; ' + key
value = transform_value(value)
quoted_value = quote(value)
add_quotes = always_quote or quoted_value != value
try:
quoted_value = quoted_value.encode(encoding)
except UnicodeEncodeError:
ascii_value = urllib_quote(quoted_value).encode('ascii')
rfc2231_value = encode_rfc2231(quoted_value, 'utf-8').encode('utf-8')
if add_quotes:
ascii_value = b'"' + ascii_value + b'"'
rfc2231_value = b'"' + rfc2231_value + b'"'
buffer += b'=' + ascii_value + b'; ' + key + b'*=' + rfc2231_value
else:
if add_quotes:
quoted_value = b'"' + quoted_value + b'"'
buffer += b'=' + quoted_value
return buffer
def user_playlists(self, user, limit=50, offset=0):
""" Gets playlists of a user
Parameters:
- user - the id of the usr
- limit - the number of items to return
- offset - the index of the first item to return
"""
url = 'users/%s/playlists' % urllib_quote(user)
return self._get(url, limit=limit, offset=offset)
def script_link2(script):
''' if script exists return "yes" icon + view link; else return "no" icon
used in routes, channels (scripts are optional)
'''
if os.path.exists(script):
return mark_safe(
'<a class="nowrap" href="/srcfiler/?src=%s" target="_blank"><img src="/media/admin/img/icon-yes.gif"></img> view</a>'
% urllib_quote(script.encode("utf-8")))
else:
return mark_safe('<img src="/media/admin/img/icon-no.gif"></img>')
def user_playlist_unfollow(self, user, playlist_id):
""" Unfollows (deletes) a playlist for a user
Parameters:
- user - the id of the user
- name - the name of the playlist
"""
url = 'users/%s/playlists/%s/followers' % (urllib_quote(user),
playlist_id)
return self._delete(url)
def _get_char_id(self, font, ccode):
"""
Return a unique id for the given font and character-code set.
"""
sfnt = font.get_sfnt()
try:
ps_name = sfnt[(1, 0, 0, 6)].decode('macroman')
except KeyError:
ps_name = sfnt[(3, 1, 0x0409, 6)].decode('utf-16be')
char_id = urllib_quote('%s-%x' % (ps_name, ccode))
return char_id
def search(query):
"""Return an array of jpeople results for the given search. """
try:
query = urllib_quote(query) # We have to quote it.
# Make a connection
conn = httplib.HTTPConnection(jpeople_server_name)
conn.request("GET", jpeople_server_path+"?action=fullAutoComplete&str="+query, "", jpeople_search_headers)
# Get the response
conn_respone = conn.getresponse()
if conn_respone.status == 200:
# Get the result into a tree
people_tree = json.loads(unicode(conn_respone.read(), errors="ignore"))["records"]
# Set an empty result stack
people_list = []
# Lets make it a hash-style-object
for person in people_tree:
# A Directory for the person
person_dict = {}
for tag in person:
# We want to rename some properties
if tag in jpeople_attr_map.keys():
tag = jpeople_attr_map[tag]
if person[tag]==None:
person_dict[tag] = ""
else:
person_dict[tag] = person[tag].replace("\n", "")
person_dict["photo"] = "http://"+jpeople_server_name+jpeople_server_image_prefix+person_dict["eid"]+jpeople_server_image_suffix
people_list.append(person_dict)
# Return the list of all the lovely people
return people_list
else:
return False
except Exception:
return False
def cgi_enumerate():
log('Enumerate files')
bottle.response.content_type = 'application/json'
files = []
items = storage.enumerate_files()
for item in items:
full_disk_filename = item['full_disk_filename']
url_filename = item['url_filename']
display_filename = item['display_filename']
modified_unixtime = get_file_modified_unixtime(full_disk_filename)
files.append(
{
'display_filename': display_filename,
'url': URLPREFIX + urllib_quote(url_filename),
'url_filename': url_filename,
'size': os_path.getsize(full_disk_filename),
'modified': modified_unixtime,
})
files = sorted(files, key=lambda item: item['modified'])
return json_dumps(files, indent=4)
def server_storage(url_filename):
log('File download: ' + url_filename)
filedir, disk_filename, display_filename = \
storage.get_file_info_to_read(url_filename)
# show preview for images and text files
# force text files to be shown as text/plain
# (and not text/html for example)
# Empty extenstion is treated as .txt extension
mime_filename = display_filename \
if '.' in display_filename else display_filename + '.txt'
mimetype, encoding = mimetypes.guess_type(mime_filename)
mimetype = str(mimetype)
if mimetype.startswith('text/'):
mimetype = 'text/plain'
elif not mimetype.startswith('image/'):
mimetype = ''
showpreview = mimetype != ''
quoted_display_filename = urllib_quote(display_filename)
if showpreview:
response = bottle.static_file(disk_filename,
root=filedir,
mimetype=mimetype)
content_disposition = 'inline; filename="%s"' % \
quoted_display_filename
response.set_header('Content-Disposition', content_disposition)
else:
response = bottle.static_file(disk_filename,
root=filedir,
download=quoted_display_filename)
response.set_header('Cache-Control', 'no-cache, no-store, must-revalidate')
response.set_header('Pragma', 'no-cache')
response.set_header('Expires', '0')
return response
def GetPathFromLink(resource_link, resource_type=''):
"""Gets path from resource link with optional resource type
:Parameters:
- `resource_link`: str
- `resource_type`: str
:Returns:
str, path from resource link with resource type appended(if provided).
"""
resource_link = TrimBeginningAndEndingSlashes(resource_link)
if IsNameBased(resource_link):
# Replace special characters in string using the %xx escape. For example, space(' ') would be replaced by %20
# This function is intended for quoting the path section of the URL and excludes '/' to be quoted as that's the default safe char
resource_link = urllib_quote(resource_link)
# Padding leading and trailing slashes to the path returned both for name based and resource id based links
if resource_type:
return '/' + resource_link + '/' + resource_type + '/'
else:
return '/' + resource_link + '/'
def _make_request(self, extra_data, method="GET", **kwargs):
"""Make request to Amazon MWS API with these parameters
"""
# Remove all keys with an empty value because
# Amazon's MWS does not allow such a thing.
extra_data = remove_empty(extra_data)
params = {
'AWSAccessKeyId': self.access_key,
self.ACCOUNT_TYPE: self.account_id,
'SignatureVersion': '2',
'Timestamp': self.get_timestamp(),
'Version': self.version,
'SignatureMethod': 'HmacSHA256',
}
if self.auth_token:
params['MWSAuthToken'] = self.auth_token
params.update(extra_data)
request_description = '&'.join(
['%s=%s' % (k, urllib_quote(params[k], safe='-_.~'))
for k in sorted(params)])
signature = self.calc_signature(method, request_description)
url = '%s%s?%s&Signature=%s' % (self.domain, self.uri,
request_description,
urllib_quote(signature))
headers = {'User-Agent': 'amazon-mws/0.%s (Language=Python)' %
self.version}
headers.update(kwargs.get('extra_headers', {}))
try:
# Some might wonder as to why i don't pass the params dict as the
# params argument to request. My answer is, here i have to get the
# url parsed string of params in order to sign it, so if i pass the
# params dict as params to request, request will repeat that step
# because it will need to convert the dict to a url parsed string,
# so why do it twice if i can just pass the full url :).
response = request(method, url, data=kwargs.get('body', ''),
headers=headers)
response.raise_for_status()
# When retrieving data from the response object,
# be aware that response.content returns the content in bytes while
# response.text calls response.content and converts it to unicode.
data = response.content
# I do not check the headers to decide which content structure to
# server simply because sometimes Amazon's MWS API returns XML
# error responses with "text/plain" as the Content-Type.
try:
parsed_response = DictWrapper(data, extra_data.get("Action") +
"Result")
except XMLError:
parsed_response = DataWrapper(data, response.headers)
except HTTPError as e:
error = MWSError(str(e))
error.response = e.response
raise error
# Store the response object in the parsed_response for quick access
parsed_response.response = response
return parsed_response
def script_link2(script):
''' if script exists return "yes" icon + view link; else return "no" icon
used in routes, channels (scripts are optional)
'''
if os.path.exists(script):
return '<a class="nowrap" href="/srcfiler/?src=%s" target="_blank"><img src="/media/admin/img/icon-yes.gif"></img> view</a>'%urllib_quote(script.encode("utf-8"))
else:
return '<img src="/media/admin/img/icon-no.gif"></img>'
def svn_uri_quote(url): # svn defines a different set of "safe" characters than Python does, so # we need to avoid escaping them. see subr/path.c:uri_char_validity[] return urllib_quote(url, "!$&'()*+,-./:=@_~")
def GetHeaders(document_client,
default_headers,
verb,
path,
resource_id,
resource_type,
options):
"""Gets HTTP request headers.
:Parameters:
- `document_client`: document_client.DocumentClient
- `default_headers`: dict
- `verb`: str
- `path`: str
- `resource_id`: str
- `resource_type`: str
- `options`: dict
:Returns:
dict, the HTTP request headers.
"""
headers = dict(default_headers)
options = options or {}
if options.get('continuation'):
headers[http_constants.HttpHeaders.Continuation] = (
options['continuation'])
pre_trigger_include = options.get('preTriggerInclude')
if pre_trigger_include:
headers[http_constants.HttpHeaders.PreTriggerInclude] = (
pre_trigger_include
if isinstance(pre_trigger_include, str)
else (',').join(pre_trigger_include))
post_trigger_include = options.get('postTriggerInclude')
if post_trigger_include:
headers[http_constants.HttpHeaders.PostTriggerInclude] = (
post_trigger_include
if isinstance(post_trigger_include, str)
else (',').join(post_trigger_include))
if options.get('maxItemCount'):
headers[http_constants.HttpHeaders.PageSize] = options['maxItemCount']
access_condition = options.get('accessCondition')
if access_condition:
if access_condition['type'] == 'IfMatch':
headers[http_constants.HttpHeaders.IfMatch] = access_condition['condition']
else:
headers[http_constants.HttpHeaders.IfNoneMatch] = access_condition['condition']
if options.get('indexingDirective'):
headers[http_constants.HttpHeaders.IndexingDirective] = (
options['indexingDirective'])
# TODO: add consistency level validation.
if options.get('consistencyLevel'):
headers[http_constants.HttpHeaders.ConsistencyLevel] = (
options['consistencyLevel'])
# TODO: add session token automatic handling in case of session consistency.
if options.get('sessionToken'):
headers[http_constants.HttpHeaders.SessionToken] = (
options['sessionToken'])
if options.get('enableScanInQuery'):
headers[http_constants.HttpHeaders.EnableScanInQuery] = (
options['enableScanInQuery'])
if options.get('resourceTokenExpirySeconds'):
headers[http_constants.HttpHeaders.ResourceTokenExpiry] = (
options['resourceTokenExpirySeconds'])
if options.get('offerType'):
headers[http_constants.HttpHeaders.OfferType] = options['offerType']
if options.get('offerThroughput'):
headers[http_constants.HttpHeaders.OfferThroughput] = options['offerThroughput']
if 'partitionKey' in options:
# if partitionKey value is Undefined, serialize it as {} to be consistent with other SDKs
if options.get('partitionKey') is documents.Undefined:
import sys
if sys.version_info > (3,):
headers[http_constants.HttpHeaders.PartitionKey] = '[{}]'.encode('utf-8')
else:
headers[http_constants.HttpHeaders.PartitionKey] = [{}]
# else serialize using json dumps method which apart from regular values will serialize None into null
else:
headers[http_constants.HttpHeaders.PartitionKey] = json.dumps([options['partitionKey']])
if options.get('enableCrossPartitionQuery'):
headers[http_constants.HttpHeaders.EnableCrossPartitionQuery] = options['enableCrossPartitionQuery']
if document_client.master_key:
headers[http_constants.HttpHeaders.XDate] = (
datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT'))
if document_client.master_key or document_client.resource_tokens:
# -_.!~*'() are valid characters in url, and shouldn't be quoted.
headers[http_constants.HttpHeaders.Authorization] = urllib_quote(
auth.GetAuthorizationHeader(document_client,
verb,
path,
resource_id,
resource_type,
headers),
'-_.!~*\'()')
if verb == 'post' or verb == 'put':
if not headers.get(http_constants.HttpHeaders.ContentType):
headers[http_constants.HttpHeaders.ContentType] = runtime_constants.MediaTypes.Json
if not headers.get(http_constants.HttpHeaders.Accept):
headers[http_constants.HttpHeaders.Accept] = runtime_constants.MediaTypes.Json
return headers
def download_url_async(url, dest='tmp', min_size=0,
complete_cb=None, progress_cb=None,
urlencode=True, *args, **kargs):
"""Download the given url in async way.
TODO:
If dest is set to None than the data will be passed as the dest param
in the complete_cb.
Args:
url: must be a valid url to download.
dest: If set to a local file name then the download data will be written
to that file (created and overwritten if necessary, also the
necessary parent directories are created).
If dest is omitted (or is 'tmp') than the data will be written
to a random new temp file.
min_size: if min_size is set (and > 0) than downloaded files smaller that
min_size will be discarted.
complete_cb: if given, will be called when the download is done.
signature: complete_cb(file, status, *args, **kargs)
progress_cb: will be called while the download is in progress.
signature: progress_cb(file, dltotal, dlnow, *args, **kargs)
urlencode: encode the given url (default to True).
*args: any other arguments will be passed back in the callbacks.
**kargs: any other keyword arguments will be passed back in the callbacks.
Returns:
The Ecore FileDownload instance
"""
def _cb_download_complete(dest, status, dwl_data, *args, **kargs):
(complete_cb, progress_cb, min_size) = dwl_data
# if file size < min_size: report as error
if status == 200 and min_size > 0 and os.path.getsize(dest) < min_size:
DBG('MIN_SIZE not reached, discard download')
status = 404 # HTTP NotFound code
# on errors delete the downloaded file
if status != 200 and os.path.exists(dest):
DBG('download error, HTTP code: ' + str(status))
os.remove(dest)
# call the user complete_cb if available
if complete_cb and callable(complete_cb):
complete_cb(dest, status, *args, **kargs)
def _cb_download_progress(dest, dltotal, dlnow, uptotal, upnow, dwl_data, *args, **kargs):
(complete_cb, progress_cb, min_size) = dwl_data
#TODO filter out some call (maybe report only when dlnow change)
if progress_cb and callable(progress_cb):
progress_cb(dest, dltotal, dlnow, *args, **kargs)
return 0 # always continue the download
# urlencode the url (but not the http:// part, or ':' will be converted)
if urlencode:
(_prot, _url) = url.split('://', 1)
encoded = '://'.join((_prot, urllib_quote(_url)))
else:
encoded = url
# use a random temp file
if dest == 'tmp':
dest = tempfile.mktemp()
elif dest:
# create dest path if necessary,
dirname = os.path.dirname(dest)
if not os.path.exists(dirname):
os.makedirs(dirname)
# remove destination file if exists (overwrite)
if os.path.exists(dest):
os.remove(dest)
# store download data for later use
dwl_data = (complete_cb, progress_cb, min_size)
# start the download
return ecore.FileDownload(encoded, dest,
_cb_download_complete, _cb_download_progress,
dwl_data=dwl_data, *args, **kargs)
def quote(text):
''' Perform URL escaping, including forward slash characters. '''
return urllib_quote(text, safe='')
