def base_handle_copy_request(self, src_container, src_obj, dest_container, dest_object):
"""
Unified path for:
PUT verb with X-Copy-From and
COPY verb with Destination
"""
# Get an iterator over the source object
source_path = "/%s/%s/%s/%s" % (self.api_version, self.account, src_container, src_obj)
source_req = self.request.copy_get()
source_req.headers.pop("X-Backend-Storage-Policy-Index", None)
source_req.headers.pop("X-Run-Storlet", None)
source_req.path_info = source_path
source_req.headers["X-Newest"] = "true"
src_resp = source_req.get_response(self.app)
sreq = self._build_storlet_request(self.request, src_resp.headers, src_resp.app_iter)
self.gather_extra_sources()
sresp = self.gateway.invocation_flow(sreq, self.extra_sources)
resp = self.handle_put_copy_response(sresp.user_metadata, sresp.data_iter)
acct, path = src_resp.environ["PATH_INFO"].split("/", 3)[2:4]
resp.headers["X-Storlet-Generated-From-Account"] = quote(acct)
resp.headers["X-Storlet-Generated-From"] = quote(path)
if "last-modified" in src_resp.headers:
resp.headers["X-Storlet-Generated-From-Last-Modified"] = src_resp.headers["last-modified"]
return resp
def test_image_already_generated_by_thumbor_2_times(self):
with open(
normalize_unicode_path(u'./tests/fixtures/images/alabama1_ap620é.jpg'), 'r'
) as f:
self.context.modules.storage.put(
quote("http://test.com/smart/alabama1_ap620é"),
f.read()
)
crypto = CryptoURL('ACME-SEC')
image_url = self.get_url(
crypto.generate(
image_url=quote(self.get_url(
crypto.generate(
image_url=quote("http://test.com/smart/alabama1_ap620é")
)
))
)
)
url = crypto.generate(
image_url=quote(image_url)
)
response = self.fetch(url)
expect(response.code).to_equal(200)
def sites_linking_in(self, urls, count=MAX_SITES_LINKING_IN_COUNT, start=0):
if count > self.MAX_SITES_LINKING_IN_COUNT:
raise RuntimeError("Maximum SitesLinkingIn result count is %s." % self.MAX_SITES_LINKING_IN_COUNT)
params = { "Action": "SitesLinkingIn" }
if not isinstance(urls, (list, tuple)):
params.update({
"Url": urlparse.quote(urls),
"ResponseGroup": "SitesLinkingIn",
"Count": count,
"Start": start,
})
else:
if len(urls) > self.MAX_BATCH_REQUESTS:
raise RuntimeError("Maximum number of batch URLs is %s." % self.MAX_BATCH_REQUESTS)
params.update({
"SitesLinkingIn.Shared.ResponseGroup": "SitesLinkingIn",
"SitesLinkingIn.Shared.Count": count,
"SitesLinkingIn.Shared.Start": start,
})
for i, url in enumerate(urls):
params.update({"SitesLinkingIn.%d.Url" % (i + 1): urlparse.quote(url)})
return self.request(params)
def join_params(params, encode=True):
return "&".join(
[
"%s=%s" % (urlparse.quote(key, safe="") if encode else key, urlparse.quote(val, safe="") if encode else val)
for key, val in six.iteritems(params)
]
)
def _dump_common_links(self, obj):
"""Dump common links for deposits and records."""
links = {}
m = obj.get('metadata', {})
doi = m.get('doi')
if doi:
links['badge'] = ui_link_for('badge', doi=quote(doi))
links['doi'] = idutils.to_url(doi, 'doi', 'https')
conceptdoi = m.get('conceptdoi')
if conceptdoi:
links['conceptbadge'] = ui_link_for('badge', doi=quote(conceptdoi))
links['conceptdoi'] = idutils.to_url(conceptdoi, 'doi', 'https')
files = m.get('_files', [])
for f in files:
if f.get('type') in thumbnail_exts:
try:
links['thumb250'] = self._thumbnail_url(f, 250)
# First previewable image is used for preview.
except RuntimeError:
pass
break
return links
def list(self, stack_id, resource_name=None, **kwargs):
"""Get a list of events.
:param stack_id: ID of stack the events belong to
:param resource_name: Optional name of resources to filter events by
:rtype: list of :class:`Event`
"""
params = {}
if 'filters' in kwargs:
filters = kwargs.pop('filters')
params.update(filters)
for key, value in kwargs.items():
if value:
params[key] = value
if resource_name is None:
url = '/stacks/%s/events' % stack_id
else:
stack_id = self._resolve_stack_id(stack_id)
url = '/stacks/%s/resources/%s/events' % (
parse.quote(stack_id),
parse.quote(encodeutils.safe_encode(resource_name)))
if params:
# convert to a sorted dict for python3 predictible order
params = collections.OrderedDict(sorted(params.items()))
url += '?%s' % parse.urlencode(params, True)
return self._list(url, 'events')
def quote_user_host(user, host):
quoted = ''
if host:
quoted = parse.quote("%s@%s" % (user, host))
else:
quoted = parse.quote("%s" % user)
return quoted.replace('.', '%2e')
def create_container(self, req, container_path):
"""
Checks if the container exists and if not try to create it.
:params container_path: an unquoted path to a container to be created
:returns: True if created container, False if container exists
:raises CreateContainerError: when unable to create container
"""
head_cont_req = make_subrequest(
req.environ, method='HEAD', path=quote(container_path),
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
swift_source='EA')
resp = head_cont_req.get_response(self.app)
if resp.is_success:
return False
if resp.status_int == HTTP_NOT_FOUND:
create_cont_req = make_subrequest(
req.environ, method='PUT', path=quote(container_path),
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
swift_source='EA')
resp = create_cont_req.get_response(self.app)
if resp.is_success:
return True
raise CreateContainerError(
"Create Container Failed: " + container_path,
resp.status_int, resp.status)
def base_handle_copy_request(self, src_container, src_obj,
dest_container, dest_object):
"""
Unified path for:
PUT verb with X-Copy-From and
COPY verb with Destination
"""
# Get an iterator over the source object
source_path = '/%s/%s/%s/%s' % (self.api_version, self.account,
src_container, src_obj)
source_req = self.request.copy_get()
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.headers.pop('X-Run-Storlet', None)
source_req.path_info = source_path
source_req.headers['X-Newest'] = 'true'
source_resp = source_req.get_response(self.app)
# Do proxy copy flow
(out_md, app_iter) = self.gateway.gatewayProxyCopyFlow(self.request,
source_resp)
resp = self.handle_put_copy_response(out_md, app_iter)
acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4]
resp.headers['X-Storlet-Generated-From-Account'] = quote(acct)
resp.headers['X-Storlet-Generated-From'] = quote(path)
if 'last-modified' in source_resp.headers:
resp.headers['X-Storlet-Generated-From-Last-Modified'] = \
source_resp.headers['last-modified']
return resp
def test_check_symlink_header_invalid_format(self):
def do_test(headers, status, err_msg):
req = Request.blank('/v1/a/c/o', method='PUT',
headers=headers)
with self.assertRaises(swob.HTTPException) as cm:
symlink._check_symlink_header(req)
self.assertEqual(cm.exception.status, status)
self.assertEqual(cm.exception.body, err_msg)
do_test({'X-Symlink-Target': '/c1/o1'},
'412 Precondition Failed',
'X-Symlink-Target header must be of the '
'form <container name>/<object name>')
do_test({'X-Symlink-Target': 'c1o1'},
'412 Precondition Failed',
'X-Symlink-Target header must be of the '
'form <container name>/<object name>')
do_test({'X-Symlink-Target': 'c1/o1',
'X-Symlink-Target-Account': '/another'},
'412 Precondition Failed',
'Account name cannot contain slashes')
do_test({'X-Symlink-Target': 'c1/o1',
'X-Symlink-Target-Account': 'an/other'},
'412 Precondition Failed',
'Account name cannot contain slashes')
# url encoded case
do_test({'X-Symlink-Target': '%2Fc1%2Fo1'},
'412 Precondition Failed',
'X-Symlink-Target header must be of the '
'form <container name>/<object name>')
do_test({'X-Symlink-Target': 'c1/o1',
'X-Symlink-Target-Account': '%2Fanother'},
'412 Precondition Failed',
'Account name cannot contain slashes')
do_test({'X-Symlink-Target': 'c1/o1',
'X-Symlink-Target-Account': 'an%2Fother'},
'412 Precondition Failed',
'Account name cannot contain slashes')
# with multi-bytes
do_test(
{'X-Symlink-Target':
u'/\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3'},
'412 Precondition Failed',
'X-Symlink-Target header must be of the '
'form <container name>/<object name>')
target = u'/\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3'
encoded_target = quote(target.encode('utf-8'), '')
do_test(
{'X-Symlink-Target': encoded_target},
'412 Precondition Failed',
'X-Symlink-Target header must be of the '
'form <container name>/<object name>')
account = u'\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3'
encoded_account = quote(account.encode('utf-8'), '')
do_test(
{'X-Symlink-Target': 'c/o',
'X-Symlink-Target-Account': encoded_account},
'412 Precondition Failed',
'Account name cannot contain slashes')
def handle_COPY(self, req, start_response, account, container, obj):
if not req.headers.get('Destination'):
return HTTPPreconditionFailed(request=req,
body='Destination header required'
)(req.environ, start_response)
dest_account = account
if 'Destination-Account' in req.headers:
dest_account = unquote(req.headers.get('Destination-Account'))
dest_account = check_account_format(req, dest_account)
req.headers['X-Copy-From-Account'] = quote(account)
account = dest_account
del req.headers['Destination-Account']
dest_container, dest_object = _check_destination_header(req)
source = '/%s/%s' % (container, obj)
container = dest_container
obj = dest_object
# re-write the existing request as a PUT instead of creating a new one
req.method = 'PUT'
# As this the path info is updated with destination container,
# the proxy server app will use the right object controller
# implementation corresponding to the container's policy type.
ver, _junk = req.split_path(1, 2, rest_with_last=True)
req.path_info = '/%s/%s/%s/%s' % (
ver, dest_account, dest_container, dest_object)
req.headers['Content-Length'] = 0
req.headers['X-Copy-From'] = quote(source)
del req.headers['Destination']
return self.handle_PUT(req, start_response)
def test_list_alarms_sort_by_asc_desc(self):
alarm_definition_ids, expected_metric = self._create_alarms_for_test_alarms(num=3)
resp, response_body = self.monasca_client.list_alarms('?metric_name=' + expected_metric['name'])
self._verify_list_alarms_elements(resp, response_body,
expect_num_elements=3)
resp, response_body = self.monasca_client.list_alarms('?metric_name=' + expected_metric['name'] +
'&sort_by=' +
urlparse.quote('created_timestamp asc'))
self._verify_list_alarms_elements(resp, response_body,
expect_num_elements=3)
elements = response_body['elements']
last_timestamp = elements[0]['created_timestamp']
for element in elements:
assert element['created_timestamp'] >= last_timestamp,\
"Created_timestamps are not in ascending order {} came before {}".format(last_timestamp,
element['created_timestamp'])
last_timestamp = element['created_timestamp']
resp, response_body = self.monasca_client.list_alarms('?metric_name=' + expected_metric['name'] +
'&sort_by=' +
urlparse.quote('created_timestamp desc'))
self._verify_list_alarms_elements(resp, response_body,
expect_num_elements=3)
elements = response_body['elements']
last_timestamp = elements[0]['created_timestamp']
for element in elements:
assert element['created_timestamp'] <= last_timestamp,\
"Created_timestamps are not in descending order {} came before {}".format(last_timestamp,
element['created_timestamp'])
last_timestamp = element['created_timestamp']
def stack_path(self):
"""Return a URL-encoded path segment of a URL without a tenant.
Returned in the form:
<stack_name>/<stack_id>
"""
return "%s/%s" % (urlparse.quote(self.stack_name, ""), urlparse.quote(self.stack_id, ""))
def _listing_pages_iter(self, account_name, lcontainer, lprefix, env):
marker = ''
while True:
lreq = make_pre_authed_request(
env, method='GET', swift_source='VW',
path='/v1/%s/%s' % (account_name, lcontainer))
lreq.environ['QUERY_STRING'] = \
'format=json&prefix=%s&marker=%s' % (quote(lprefix),
quote(marker))
lresp = lreq.get_response(self.app)
if not is_success(lresp.status_int):
if lresp.status_int == HTTP_NOT_FOUND:
raise ListingIterNotFound()
elif is_client_error(lresp.status_int):
raise HTTPPreconditionFailed()
else:
raise ListingIterError()
if not lresp.body:
break
sublisting = json.loads(lresp.body)
if not sublisting:
break
marker = sublisting[-1]['name'].encode('utf-8')
yield sublisting
def id(self):
"""Return the id of this sequence."""
if self.sub_children:
id_ = ",".join(quote(child.id) for child in self.template.children())
else:
id_ = quote(self.template.id)
return id_
def make_query(*args, **kwargs):
'''Construct a URL query string, with marshalling markup.
If there are positional arguments, they must be dictionaries.
They are combined with the dictionary of keyword arguments to form
a dictionary of query names and values.
Query names (the keys) must be strings. Values may be strings,
integers, floats, or DateTimes, and they may also be lists or
namespaces containing these types. Names and string values
should not be URL-quoted. All arguments are marshalled with
complex_marshal().
'''
d = {}
for arg in args:
d.update(arg)
d.update(kwargs)
qlist = complex_marshal(list(d.items()))
for i in range(len(qlist)):
k, m, v = qlist[i]
if PY2 and isinstance(v, text_type):
v = v.encode(_default_encoding())
qlist[i] = '%s%s=%s' % (quote(k), m, quote(str(v)))
return '&'.join(qlist)
def getEditTileURL(tile, request):
"""Get the edit URL for the given tile.
If the tile is transient, the URL will contain a `_tiledata`
parameter with the tile data encoded in JSON. This avoids possible
collisions between raw data coming from the edit form and tile
data retrieved by the transient tile data manager.
"""
id = tile.id
name = tile.__name__
context = tile.__parent__
if name is None or context is None:
raise TypeError("Insufficient context to determine URL")
url = str(getMultiAdapter((context, request), IAbsoluteURL))
tileFragment = "@@edit-tile/" + urlparse.quote(name.encode('utf-8'), _safe)
if id:
tileFragment += '/' + urlparse.quote(id.encode('utf-8'), _safe)
url = '%s/%s' % (url, tileFragment,)
if not IPersistentTile.providedBy(tile):
data = ITileDataManager(tile).get()
if data:
tileType = queryUtility(ITileType, name=name)
if tileType is not None and tileType.schema is not None:
if '?' in url:
url += '&' + '_tiledata=' + json.dumps(data)
else:
url += '?' + '_tiledata=' + json.dumps(data)
return url
def object_request(self, req, api_version, account, container, obj,
allow_versioned_writes):
container_name = unquote(container)
object_name = unquote(obj)
orig_container = get_unversioned_container(container_name)
if orig_container != container_name:
orig_object, version = \
swift3_split_object_name_version(object_name)
req.environ['oio.query'] = {'version': version}
req.environ['PATH_INFO'] = '/%s/%s/%s/%s' % (api_version,
account,
quote(orig_container),
quote(orig_object))
elif req.method == 'DELETE':
ver_mode = req.headers.get('X-Backend-Versioning-Mode-Override',
'history')
if ver_mode == 'stack':
# Do not create a delete marker, delete the latest version
obj_inf = get_object_info(req.environ, self.app,
swift_source='VW')
req.environ['oio.query'] = {
'version': obj_inf.get('sysmeta', {}).get('version-id')
}
resp = req.get_response(self.app)
if req.method == 'HEAD':
close_if_possible(resp.app_iter)
return resp
def notify(self, check):
url = self.channel.value_down
if check.status == "up":
url = self.channel.value_up
if not url:
# If the URL is empty then we do nothing
return "no-op"
# Replace variables with actual values.
# There should be no bad translations if users use $ symbol in
# check's name or tags, because $ gets urlencoded to %24
if "$CODE" in url:
url = url.replace("$CODE", str(check.code))
if "$STATUS" in url:
url = url.replace("$STATUS", check.status)
if "$NAME" in url:
url = url.replace("$NAME", quote(check.name))
if "$TAG" in url:
for i, tag in enumerate(check.tags_list()):
placeholder = "$TAG%d" % (i + 1)
url = url.replace(placeholder, quote(tag))
return self.get(url)
def _db_create_special(
client,
source_db,
dest_db,
kwargs):
# Determine server location
kwargs['location'] = get_server_location(
server_name=dest_db.server_name,
resource_group_name=dest_db.resource_group_name)
# Set create mode properties
subscription_id = get_subscription_id()
kwargs['source_database_id'] = (
'/subscriptions/{}/resourceGroups/{}/providers/Microsoft.Sql/servers/{}/databases/{}'
.format(quote(subscription_id),
quote(source_db.resource_group_name),
quote(source_db.server_name),
quote(source_db.database_name)))
# Create
return client.create_or_update(
server_name=dest_db.server_name,
resource_group_name=dest_db.resource_group_name,
database_name=dest_db.database_name,
parameters=kwargs)
def list(self, stack_id, resource_name=None, **kwargs):
"""Get a list of events.
:param stack_id: ID of stack the events belong to
:param resource_name: Optional name of resources to filter events by
:rtype: list of :class:`Event`
"""
params = {}
if 'filters' in kwargs:
filters = kwargs.pop('filters')
params.update(filters)
for key, value in six.iteritems(kwargs):
if value:
params[key] = value
if resource_name is None:
url = '/stacks/%s/events' % stack_id
else:
stack_id = self._resolve_stack_id(stack_id)
url = '/stacks/%s/resources/%s/events' % (
parse.quote(stack_id, ''),
parse.quote(strutils.safe_encode(resource_name), ''))
if params:
url += '?%s' % parse.urlencode(params, True)
return self._list(url, 'events')
def __call__(self, env, start_response):
request = Request(env)
if not request.path.startswith(self.endpoints_path):
return self.app(env, start_response)
if request.method != 'GET':
return HTTPMethodNotAllowed(
req=request, headers={"Allow": "GET"})(env, start_response)
try:
version, account, container, obj = self._parse_path(request)
except ValueError as err:
return HTTPBadRequest(str(err))(env, start_response)
if account is not None:
account = unquote(account)
if container is not None:
container = unquote(container)
if obj is not None:
obj = unquote(obj)
storage_policy_index = None
if obj is not None:
container_info = get_container_info(
{'PATH_INFO': '/v1/%s/%s' % (account, container)},
self.app, swift_source='LE')
storage_policy_index = container_info['storage_policy']
obj_ring = self.get_object_ring(storage_policy_index)
partition, nodes = obj_ring.get_nodes(
account, container, obj)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}/{container}/{obj}'
elif container is not None:
partition, nodes = self.container_ring.get_nodes(
account, container)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}/{container}'
else:
partition, nodes = self.account_ring.get_nodes(
account)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}'
endpoints = []
for node in nodes:
endpoint = endpoint_template.format(
ip=node['ip'],
port=node['port'],
device=node['device'],
partition=partition,
account=quote(account),
container=quote(container or ''),
obj=quote(obj or ''))
endpoints.append(endpoint)
resp = self.response_map[version](
request, endpoints=endpoints,
storage_policy_index=storage_policy_index)
return resp(env, start_response)
def percent_encode(values):
rv = {}
for k, v in values.items():
if isinstance(v, six.string_types):
rv[k] = quote(v)
else:
rv[k] = [quote(s) for s in v]
return rv
def ZScriptHTML_tryAction(self, REQUEST, argvars):
"""Apply the test parameters.
"""
vv = []
for argvar in argvars:
if argvar.value:
vv.append("%s=%s" % (quote(argvar.name), quote(argvar.value)))
raise Redirect("%s?%s" % (REQUEST['URL1'], '&'.join(vv)))
def stack_path(self):
'''
Return a URL-encoded path segment of a URL,
in the form:
<stack_name>/<stack_id>
'''
return '%s/%s' % (urlparse.quote(self.stack_name, ''),
urlparse.quote(self.stack_id, ''))
def url_quote(v, name='(Unknown name)', md={}):
if six.PY2 and isinstance(v, unicode):
# quote does not handle unicode. Encoding to a "safe"
# intermediate encoding before quoting, then unencoding the result.
return quote(v.encode('utf-8')).decode('utf-8')
elif six.PY3 and isinstance(v, bytes):
return quote(v.decode('utf-8')).encode('utf-8')
return quote(str(v))
def _get_canonical_querystring(self, params):
querystring = []
for param in sorted(params):
querystring.append('{}={}'.format(
quote(param, safe='-_.~'),
quote(str(params[param]), safe='-_.~'),
))
return '&'.join(querystring)
def _listing_pages_iter(self, account_name, lcontainer, lprefix,
env, marker='', end_marker='', reverse=True):
'''Get "pages" worth of objects that start with a prefix.
The optional keyword arguments ``marker``, ``end_marker``, and
``reverse`` are used similar to how they are for containers. We're
either coming:
- directly from ``_listing_iter``, in which case none of the
optional args are specified, or
- from ``_in_proxy_reverse_listing``, in which case ``reverse``
is ``False`` and both ``marker`` and ``end_marker`` are specified
(although they may still be blank).
'''
while True:
lreq = make_pre_authed_request(
env, method='GET', swift_source='VW',
path='/v1/%s/%s' % (account_name, lcontainer))
lreq.environ['QUERY_STRING'] = \
'format=json&prefix=%s&marker=%s' % (
quote(lprefix), quote(marker))
if end_marker:
lreq.environ['QUERY_STRING'] += '&end_marker=%s' % (
quote(end_marker))
if reverse:
lreq.environ['QUERY_STRING'] += '&reverse=on'
lresp = lreq.get_response(self.app)
if not is_success(lresp.status_int):
close_if_possible(lresp.app_iter)
if lresp.status_int == HTTP_NOT_FOUND:
raise ListingIterNotFound()
elif is_client_error(lresp.status_int):
raise HTTPPreconditionFailed()
else:
raise ListingIterError()
if not lresp.body:
break
sublisting = json.loads(lresp.body)
if not sublisting:
break
# When using the ``reverse`` param, check that the listing is
# actually reversed
first_item = sublisting[0]['name'].encode('utf-8')
last_item = sublisting[-1]['name'].encode('utf-8')
page_is_after_marker = marker and first_item > marker
if reverse and (first_item < last_item or page_is_after_marker):
# Apparently there's at least one pre-2.6.0 container server
yield self._in_proxy_reverse_listing(
account_name, lcontainer, lprefix,
env, marker, sublisting)
return
marker = last_item
yield sublisting
def download_spk(category, kernel, download_dir=None):
"""Download generic kernel SPK files from
http://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/
Parameters:
category: asteroids, comets, lagrange_point, planets, satellites, or stations
kernel: Kernel name, e.g. `de430` will download `de430.bsp`
download_dir: Directory to download file to. By default, this is to a
platform-dependent astrodynamics directory: :py:const:`SPK_DIR`
"""
valid_categories = ['asteroids', 'comets', 'lagrange_point', 'planets',
'satellites', 'stations']
if category not in valid_categories:
s = 'Invalid category. Valid categories: {}'
raise InvalidCategoryError(s.format(', '.join(valid_categories)))
# We only wanted the name, strip the extension.
if kernel.endswith('.bsp'):
kernel = kernel[:-4]
urls = [
SPK_URL.format(category=category, kernel=quote(kernel)),
SPK_OLD_URL.format(category=category, kernel=quote(kernel))
]
# Get last path component for filename.
# e.g. 'de423_for_mercury_and_venus/de423' => 'de423.bsp'
# 'de421' => 'de421.bsp'
filename = kernel.split('/')[-1] + '.bsp'
if download_dir:
download_dir = Path(download_dir)
filepath = download_dir / filename
else:
with suppress_file_exists_error():
SPK_DIR.mkdir(parents=True)
filepath = SPK_DIR / filename
downloaded = False
for url in urls:
try:
download_file_with_progress(url, str(filepath))
except HTTPError as exc:
if exc.response.status_code != 404:
raise
else:
print('Kernel downloaded to', filepath)
downloaded = True
break
if not downloaded:
s = 'Kernel not found in the following locations:\n{}'
raise KernelNotFoundError(s.format('\n'.join(prefix(' ', urls))))
def setUp(self):
super(ErrorPageEmbedTest, self).setUp()
self.project = self.create_project()
self.project.update_option('sentry:origins', ['example.com'])
self.key = self.create_project_key(self.project)
self.event_id = uuid4().hex
self.path = '%s?eventId=%s&dsn=%s' % (
reverse('sentry-error-page-embed'), quote(self.event_id), quote(self.key.dsn_public),
)
def _upload_data_audit_artifacts(self, name):
# type: (str) -> ()
logger = self._task.get_logger()
pd_artifact = self._artifacts_container.get(name)
pd_metadata = self._artifacts_container.get_metadata(name)
# remove from artifacts watch list
if name in self._unregister_request:
try:
self._unregister_request.remove(name)
except KeyError:
pass
self._artifacts_container.unregister_artifact(name)
if pd_artifact is None:
return
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_csv = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
os.close(fd)
local_csv = Path(local_csv)
pd_artifact.to_csv(local_csv.as_posix(), index=False, compression=self._compression)
current_sha2, file_sha2 = sha256sum(
local_csv.as_posix(), skip_header=32, block_size=Artifacts._hash_block_size)
if name in self._last_artifacts_upload:
previous_sha2 = self._last_artifacts_upload[name]
if previous_sha2 == current_sha2:
# nothing to do, we can skip the upload
# noinspection PyBroadException
try:
local_csv.unlink()
except Exception:
pass
return
self._last_artifacts_upload[name] = current_sha2
# If old clearml-server, upload as debug image
if not Session.check_min_api_version('2.3'):
logger.report_image(title='artifacts', series=name, local_path=local_csv.as_posix(),
delete_after_upload=True, iteration=self._task.get_last_iteration(),
max_image_history=2)
return
# Find our artifact
artifact = None
for an_artifact in self._task_artifact_list:
if an_artifact.key == name:
artifact = an_artifact
break
file_size = local_csv.stat().st_size
# upload file
uri = self._upload_local_file(local_csv, name, delete_after_upload=True,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri)
# update task artifacts
with self._task_edit_lock:
if not artifact:
artifact = tasks.Artifact(key=name, type=self._pd_artifact_type)
self._task_artifact_list.append(artifact)
artifact_type_data = tasks.ArtifactTypeData()
artifact_type_data.data_hash = current_sha2
artifact_type_data.content_type = "text/csv"
artifact_type_data.preview = str(pd_artifact.__repr__(
)) + '\n\n' + self._get_statistics({name: pd_artifact})
artifact.type_data = artifact_type_data
artifact.uri = uri
artifact.content_size = file_size
artifact.hash = file_sha2
artifact.timestamp = int(time())
artifact.display_data = [(str(k), str(v)) for k, v in pd_metadata.items()] if pd_metadata else None
self._task.set_artifacts(self._task_artifact_list)
def get_tool_descriptor(self, tool_id, version="latest", tool_type="CWL"):
url = "%s/ga4gh/v1/tools/%s/versions/%s/%s/descriptor" % (
self.base_url, quote(tool_id, safe=''), version, tool_type)
return self._requests.get(url)
def percent_encode(val):
# see https://en.wikipedia.org/wiki/Percent-encoding
return quote(val.encode('utf8', errors='replace')).replace('%7E',
'~').replace(
'/', '%2F')
def publish_sample(self, env, bytes_received, bytes_sent):
path = urlparse.quote(env['PATH_INFO'])
method = env['REQUEST_METHOD']
headers = {}
for header in env:
if header.startswith('HTTP_') and env[header]:
key = header.strip('HTTP_')
if isinstance(env[header], six.text_type):
headers[key] = env[header].encode('utf-8')
else:
headers[key] = str(env[header])
try:
container = obj = None
version, account, remainder = path.replace(
'/', '', 1).split('/', 2)
if not version or not account:
raise ValueError('Invalid path: %s' % path)
if remainder:
if '/' in remainder:
container, obj = remainder.split('/', 1)
else:
container = remainder
except ValueError:
return
now = timeutils.utcnow().isoformat()
resource_metadata = {
"path": path,
"version": version,
"container": container,
"object": obj,
}
for header in self.metadata_headers:
if header.upper() in headers:
resource_metadata['http_header_%s' % header] = headers.get(
header.upper())
with self.pipeline_manager.publisher(
context.get_admin_context()) as publisher:
if bytes_received:
publisher([sample.Sample(
name='storage.objects.incoming.bytes',
type=sample.TYPE_DELTA,
unit='B',
volume=bytes_received,
user_id=env.get('HTTP_X_USER_ID'),
project_id=env.get('HTTP_X_TENANT_ID'),
resource_id=account.partition(self.reseller_prefix)[2],
timestamp=now,
resource_metadata=resource_metadata)])
if bytes_sent:
publisher([sample.Sample(
name='storage.objects.outgoing.bytes',
type=sample.TYPE_DELTA,
unit='B',
volume=bytes_sent,
user_id=env.get('HTTP_X_USER_ID'),
project_id=env.get('HTTP_X_TENANT_ID'),
resource_id=account.partition(self.reseller_prefix)[2],
timestamp=now,
resource_metadata=resource_metadata)])
# publish the event for each request
# request method will be recorded in the metadata
resource_metadata['method'] = method.lower()
publisher([sample.Sample(
name='storage.api.request',
type=sample.TYPE_DELTA,
unit='request',
volume=1,
user_id=env.get('HTTP_X_USER_ID'),
project_id=env.get('HTTP_X_TENANT_ID'),
resource_id=account.partition(self.reseller_prefix)[2],
timestamp=now,
resource_metadata=resource_metadata)])
def create_http_request(self, api_client):
# verify the required parameter 'name' is set
if self.name is None:
raise ValueError("Missing the required parameter `name` when calling `get_office_math_object`") # noqa: E501
# verify the required parameter 'index' is set
if self.index is None:
raise ValueError("Missing the required parameter `index` when calling `get_office_math_object`") # noqa: E501
path = '/v4.0/words/{name}/{nodePath}/OfficeMathObjects/{index}'
path_params = {}
if self.name is not None:
path_params['name'] = self.name # noqa: E501
else:
path_params['name'] = '' # noqa: E501
if self.index is not None:
path_params['index'] = self.index # noqa: E501
else:
path_params['index'] = '' # noqa: E501
if self.node_path is not None:
path_params['nodePath'] = self.node_path # noqa: E501
else:
path_params['nodePath'] = '' # noqa: E501
# path parameters
collection_formats = {}
if path_params:
path_params = api_client.sanitize_for_serialization(path_params)
path_params = api_client.parameters_to_tuples(path_params, collection_formats)
for k, v in path_params:
# specified safe chars, encode everything
path = path.replace(
'{%s}' % k,
quote(str(v), safe=api_client.configuration.safe_chars_for_path_param)
)
# remove optional path parameters
path = path.replace('//', '/')
query_params = []
if self.folder is not None:
query_params.append(('folder', self.folder)) # noqa: E501
if self.storage is not None:
query_params.append(('storage', self.storage)) # noqa: E501
if self.load_encoding is not None:
query_params.append(('loadEncoding', self.load_encoding)) # noqa: E501
if self.password is not None:
query_params.append(('password', self.password)) # noqa: E501
header_params = {}
form_params = []
body_params = None
return {
"method": "GET",
"path": path,
"query_params": query_params,
"header_params": header_params,
"form_params": form_params,
"body": body_params,
"collection_formats": collection_formats,
"response_type": 'OfficeMathObjectResponse' # noqa: E501
}
def _get_output_destination_suffix(self, extra_path=None):
return '/'.join(
quote(x, safe="'[]{}()$^,.; -_+-=")
for x in (self.get_project_name(),
'%s.%s' % (self.name, self.data.id), extra_path) if x)
def generate_signed_url(service_account_file,
bucket_name,
object_name,
expiration,
http_method='GET',
query_parameters=None,
headers=None):
if expiration > 604800:
print('Expiration Time can\'t be longer than 604800 seconds (7 days).')
sys.exit(1)
# [START storage_signed_url_canonical_uri]
escaped_object_name = quote(six.ensure_binary(object_name), safe=b'/~')
canonical_uri = '/{}/{}'.format(bucket_name, escaped_object_name)
# [END storage_signed_url_canonical_uri]
# [START storage_signed_url_canonical_datetime]
datetime_now = datetime.datetime.utcnow()
request_timestamp = datetime_now.strftime('%Y%m%dT%H%M%SZ')
datestamp = datetime_now.strftime('%Y%m%d')
# [END storage_signed_url_canonical_datetime]
# [START storage_signed_url_credentials]
# [START storage_signed_url_signer]
google_credentials = service_account.Credentials.from_service_account_file(
service_account_file)
# [END storage_signed_url_signer]
client_email = google_credentials.service_account_email
credential_scope = '{}/auto/storage/goog4_request'.format(datestamp)
credential = '{}/{}'.format(client_email, credential_scope)
# [END storage_signed_url_credentials]
if headers is None:
headers = dict()
# [START storage_signed_url_canonical_headers]
headers['host'] = 'storage.googleapis.com'
canonical_headers = ''
ordered_headers = collections.OrderedDict(sorted(headers.items()))
for k, v in ordered_headers.items():
lower_k = str(k).lower()
strip_v = str(v).lower()
canonical_headers += '{}:{}\n'.format(lower_k, strip_v)
# [END storage_signed_url_canonical_headers]
# [START storage_signed_url_signed_headers]
signed_headers = ''
for k, _ in ordered_headers.items():
lower_k = str(k).lower()
signed_headers += '{};'.format(lower_k)
signed_headers = signed_headers[:-1] # remove trailing ';'
# [END storage_signed_url_signed_headers]
if query_parameters is None:
query_parameters = dict()
# [START storage_signed_url_canonical_query_parameters]
query_parameters['X-Goog-Algorithm'] = 'GOOG4-RSA-SHA256'
query_parameters['X-Goog-Credential'] = credential
query_parameters['X-Goog-Date'] = request_timestamp
query_parameters['X-Goog-Expires'] = expiration
query_parameters['X-Goog-SignedHeaders'] = signed_headers
canonical_query_string = ''
ordered_query_parameters = collections.OrderedDict(
sorted(query_parameters.items()))
for k, v in ordered_query_parameters.items():
encoded_k = quote(str(k), safe='')
encoded_v = quote(str(v), safe='')
canonical_query_string += '{}={}&'.format(encoded_k, encoded_v)
canonical_query_string = canonical_query_string[:-1] # remove trailing ';'
# [END storage_signed_url_canonical_query_parameters]
# [START storage_signed_url_canonical_request]
canonical_request = '\n'.join([
http_method, canonical_uri, canonical_query_string, canonical_headers,
signed_headers, 'UNSIGNED-PAYLOAD'
])
# [END storage_signed_url_canonical_request]
# [START storage_signed_url_hash]
canonical_request_hash = hashlib.sha256(
canonical_request.encode()).hexdigest()
# [END storage_signed_url_hash]
# [START storage_signed_url_string_to_sign]
string_to_sign = '\n'.join([
'GOOG4-RSA-SHA256', request_timestamp, credential_scope,
canonical_request_hash
])
# [END storage_signed_url_string_to_sign]
# [START storage_signed_url_signer]
signature = binascii.hexlify(
google_credentials.signer.sign(string_to_sign)).decode()
# [END storage_signed_url_signer]
# [START storage_signed_url_construction]
host_name = 'https://storage.googleapis.com'
signed_url = '{}{}?{}&X-Goog-Signature={}'.format(host_name, canonical_uri,
canonical_query_string,
signature)
# [END storage_signed_url_construction]
return signed_url
def login_post():
self.redirect_home_if_logged_in()
"""Authenticate users"""
username = self.post_get('username')
password = self.post_get('password')
try:
move_info = self.get_move_temp_info()
except ValidationException as ve:
self.flash_message('Login Failed: ' + str(ve))
self.redirect('/')
return
# if a collection is being moved, auth user
# and then check for available space
# if not enough space, don't continue with login
if move_info and (self.manager.cork.is_authenticate(
username, password)):
if not self.manager.has_space_for_new_coll(
username, move_info['from_user'], 'temp'):
self.flash_message(
'Sorry, not enough space to import this Temporary Collection into your account.'
)
self.redirect('/')
return
if not self.manager.cork.login(username, password):
self.flash_message('Invalid Login. Please Try Again')
redir_to = LOGIN_PATH
self.redirect(redir_to)
sesh = self.get_session()
sesh.curr_user = username
if move_info:
try:
new_title = self.manager.move_temp_coll(
username, move_info)
if new_title:
self.flash_message(
'Collection <b>{0}</b> created!'.format(new_title),
'success')
except:
import traceback
traceback.print_exc()
remember_me = (self.post_get('remember_me') == '1')
sesh.logged_in(remember_me)
temp_prefix = self.manager.temp_prefix
redir_to = request.headers.get('Referer')
host = self.get_host()
if redir_to and redir_to.startswith(host):
redir_to = redir_to[len(host):]
if not redir_to or redir_to.startswith(('/' + temp_prefix, '/_')):
redir_to = self.get_path(username)
if self.content_host:
path = '/_clear_session?path=' + quote(redir_to)
self.redir_host(self.content_host, path)
else:
self.redirect(redir_to)
def download_df(data_df):
df = pd.DataFrame.from_dict(data_df, 'columns')
csv_string = df.to_csv(index=False, encoding='utf-8')
csv_string = "data:text/csv;charset=utf-8," + quote(csv_string)
return csv_string
def test_unsuppress(self):
email = "*****@*****.**"
self.cl.stub_request("clients/%s/unsuppress.json?email=%s" % (self.cl.client_id, quote(email)), None)
res = self.cl.unsuppress(email)
def test_set_primary_contact(self):
email = '*****@*****.**'
self.cl.stub_request("clients/%s/primarycontact.json?email=%s" % (self.cl.client_id, quote(email, '')), 'client_set_primary_contact.json')
result = self.cl.set_primary_contact(email)
self.assertEquals(email, result.EmailAddress)
def handle_extract_iter(self,
req,
compress_type,
out_content_type='text/plain'):
"""
A generator that can be assigned to a swob Response's app_iter which,
when iterated over, will extract and PUT the objects pulled from the
request body. Will occasionally yield whitespace while request is being
processed. When the request is completed will yield a response body
that can be parsed to determine success. See above documentation for
details.
:params req: a swob Request
:params compress_type: specifying the compression type of the tar.
Accepts '', 'gz', or 'bz2'
"""
resp_dict = {
'Response Status': HTTPCreated().status,
'Response Body': '',
'Number Files Created': 0
}
failed_files = []
last_yield = time()
separator = ''
containers_accessed = set()
req.environ['eventlet.minimum_write_chunk_size'] = 0
try:
if not out_content_type:
raise HTTPNotAcceptable(request=req)
if out_content_type.endswith('/xml'):
yield '<?xml version="1.0" encoding="UTF-8"?>\n'
if req.content_length is None and \
req.headers.get('transfer-encoding',
'').lower() != 'chunked':
raise HTTPLengthRequired(request=req)
try:
vrs, account, extract_base = req.split_path(2, 3, True)
except ValueError:
raise HTTPNotFound(request=req)
extract_base = extract_base or ''
extract_base = extract_base.rstrip('/')
tar = tarfile.open(mode='r|' + compress_type,
fileobj=req.body_file)
failed_response_type = HTTPBadRequest
containers_created = 0
while True:
if last_yield + self.yield_frequency < time():
separator = '\r\n\r\n'
last_yield = time()
yield ' '
tar_info = next(tar)
if tar_info is None or \
len(failed_files) >= self.max_failed_extractions:
break
if tar_info.isfile():
obj_path = tar_info.name
if obj_path.startswith('./'):
obj_path = obj_path[2:]
obj_path = obj_path.lstrip('/')
if extract_base:
obj_path = extract_base + '/' + obj_path
if '/' not in obj_path:
continue # ignore base level file
destination = '/'.join(['', vrs, account, obj_path])
container = obj_path.split('/', 1)[0]
if not constraints.check_utf8(destination):
failed_files.append([
quote(obj_path[:self.max_path_length]),
HTTPPreconditionFailed().status
])
continue
if tar_info.size > constraints.MAX_FILE_SIZE:
failed_files.append([
quote(obj_path[:self.max_path_length]),
HTTPRequestEntityTooLarge().status
])
continue
container_failure = None
if container not in containers_accessed:
cont_path = '/'.join(['', vrs, account, container])
try:
if self.create_container(req, cont_path):
containers_created += 1
if containers_created > self.max_containers:
raise HTTPBadRequest(
'More than %d containers to create '
'from tar.' % self.max_containers)
except CreateContainerError as err:
# the object PUT to this container still may
# succeed if acls are set
container_failure = [
quote(cont_path[:self.max_path_length]),
err.status
]
if err.status_int == HTTP_UNAUTHORIZED:
raise HTTPUnauthorized(request=req)
except ValueError:
failed_files.append([
quote(obj_path[:self.max_path_length]),
HTTPBadRequest().status
])
continue
tar_file = tar.extractfile(tar_info)
create_headers = {
'Content-Length': tar_info.size,
'X-Auth-Token': req.headers.get('X-Auth-Token'),
}
create_obj_req = make_subrequest(
req.environ,
method='PUT',
path=quote(destination),
headers=create_headers,
agent='%(orig)s BulkExpand',
swift_source='EA')
create_obj_req.environ['wsgi.input'] = tar_file
for pax_key, pax_value in tar_info.pax_headers.items():
header_name = pax_key_to_swift_header(pax_key)
if header_name:
# Both pax_key and pax_value are unicode
# strings; the key is already UTF-8 encoded, but
# we still have to encode the value.
create_obj_req.headers[header_name] = \
pax_value.encode("utf-8")
resp = create_obj_req.get_response(self.app)
containers_accessed.add(container)
if resp.is_success:
resp_dict['Number Files Created'] += 1
else:
if container_failure:
failed_files.append(container_failure)
if resp.status_int == HTTP_UNAUTHORIZED:
failed_files.append([
quote(obj_path[:self.max_path_length]),
HTTPUnauthorized().status
])
raise HTTPUnauthorized(request=req)
if resp.status_int // 100 == 5:
failed_response_type = HTTPBadGateway
failed_files.append([
quote(obj_path[:self.max_path_length]), resp.status
])
if failed_files:
resp_dict['Response Status'] = failed_response_type().status
elif not resp_dict['Number Files Created']:
resp_dict['Response Status'] = HTTPBadRequest().status
resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files'
except HTTPException as err:
resp_dict['Response Status'] = err.status
resp_dict['Response Body'] = err.body
except (tarfile.TarError, zlib.error) as tar_error:
resp_dict['Response Status'] = HTTPBadRequest().status
resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error
except Exception:
self.logger.exception('Error in extract archive.')
resp_dict['Response Status'] = HTTPServerError().status
yield separator + get_response_body(out_content_type, resp_dict,
failed_files, 'extract')
def create_http_request(self, api_client):
# verify the required parameter 'document' is set
if self.document is None:
raise ValueError(
"Missing the required parameter `document` when calling `delete_border_online`"
) # noqa: E501
# verify the required parameter 'border_type' is set
if self.border_type is None:
raise ValueError(
"Missing the required parameter `border_type` when calling `delete_border_online`"
) # noqa: E501
path = '/v4.0/words/online/delete/{nodePath}/borders/{borderType}'
path_params = {}
if self.border_type is not None:
path_params['borderType'] = self.border_type # noqa: E501
else:
path_params['borderType'] = '' # noqa: E501
if self.node_path is not None:
path_params['nodePath'] = self.node_path # noqa: E501
else:
path_params['nodePath'] = '' # noqa: E501
# path parameters
collection_formats = {}
if path_params:
path_params = api_client.sanitize_for_serialization(path_params)
path_params = api_client.parameters_to_tuples(
path_params, collection_formats)
for k, v in path_params:
# specified safe chars, encode everything
path = path.replace(
'{%s}' % k,
quote(str(v),
safe=api_client.configuration.
safe_chars_for_path_param))
# remove optional path parameters
path = path.replace('//', '/')
query_params = []
if self.load_encoding is not None:
query_params.append(
('loadEncoding', self.load_encoding)) # noqa: E501
if self.password is not None:
query_params.append(('password', self.password)) # noqa: E501
if self.dest_file_name is not None:
query_params.append(
('destFileName', self.dest_file_name)) # noqa: E501
if self.revision_author is not None:
query_params.append(
('revisionAuthor', self.revision_author)) # noqa: E501
if self.revision_date_time is not None:
query_params.append(
('revisionDateTime', self.revision_date_time)) # noqa: E501
header_params = {}
# HTTP header `Content-Type`
header_params[
'Content-Type'] = api_client.select_header_content_type( # noqa: E501
['multipart/form-data']) # noqa: E501
form_params = []
if self.document is not None:
form_params.append(['document', self.document,
'file']) # noqa: E501
body_params = None
return {
"method": "PUT",
"path": path,
"query_params": query_params,
"header_params": header_params,
"form_params": form_params,
"body": body_params,
"collection_formats": collection_formats,
"response_type": 'DeleteBorderOnlineResponse' # noqa: E501
}
def upload_artifact(self, name, artifact_object=None, metadata=None, preview=None,
delete_after_upload=False, auto_pickle=True, wait_on_upload=False):
# type: (str, Optional[object], Optional[dict], Optional[str], bool, bool, bool) -> bool
if not Session.check_min_api_version('2.3'):
LoggerRoot.get_base_logger().warning('Artifacts not supported by your ClearML-server version, '
'please upgrade to the latest server version')
return False
if name in self._artifacts_container:
raise ValueError("Artifact by the name of {} is already registered, use register_artifact".format(name))
# cast preview to string
if preview:
preview = str(preview)
# try to convert string Path object (it might reference a file/folder)
# dont not try to serialize long texts.
if isinstance(artifact_object, six.string_types) and len(artifact_object) < 2048:
# noinspection PyBroadException
try:
artifact_path = Path(artifact_object)
if artifact_path.exists():
artifact_object = artifact_path
elif '*' in artifact_object or '?' in artifact_object:
# hackish, detect wildcard in tr files
folder = Path('').joinpath(*artifact_path.parts[:-1])
if folder.is_dir() and folder.parts:
wildcard = artifact_path.parts[-1]
if list(Path(folder).rglob(wildcard)):
artifact_object = artifact_path
except Exception:
pass
artifact_type_data = tasks.ArtifactTypeData()
artifact_type_data.preview = ''
override_filename_in_uri = None
override_filename_ext_in_uri = None
uri = None
if np and isinstance(artifact_object, np.ndarray):
artifact_type = 'numpy'
artifact_type_data.content_type = 'application/numpy'
artifact_type_data.preview = preview or str(artifact_object.__repr__())
override_filename_ext_in_uri = '.npz'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
os.close(fd)
np.savez_compressed(local_filename, **{name: artifact_object})
delete_after_upload = True
elif pd and isinstance(artifact_object, pd.DataFrame):
artifact_type = 'pandas'
artifact_type_data.content_type = 'text/csv'
artifact_type_data.preview = preview or str(artifact_object.__repr__())
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.to_csv(local_filename, compression=self._compression)
delete_after_upload = True
elif isinstance(artifact_object, Image.Image):
artifact_type = 'image'
artifact_type_data.content_type = 'image/png'
desc = str(artifact_object.__repr__())
artifact_type_data.preview = preview or desc[1:desc.find(' at ')]
override_filename_ext_in_uri = '.png'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.save(local_filename)
delete_after_upload = True
elif isinstance(artifact_object, dict):
artifact_type = 'JSON'
artifact_type_data.content_type = 'application/json'
json_text = json.dumps(artifact_object, sort_keys=True, indent=4)
override_filename_ext_in_uri = '.json'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
os.write(fd, bytes(json_text.encode()))
os.close(fd)
preview = preview or json_text
if len(preview) < self.max_preview_size_bytes:
artifact_type_data.preview = preview
else:
artifact_type_data.preview = '# full json too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes//1024, preview[:self.max_preview_size_bytes]
)
delete_after_upload = True
elif isinstance(artifact_object, (Path, pathlib_Path,) if pathlib_Path is not None else (Path,)):
# check if single file
artifact_object = Path(artifact_object)
artifact_object.expanduser().absolute()
# noinspection PyBroadException
try:
create_zip_file = not artifact_object.is_file()
except Exception: # Hack for windows pathlib2 bug, is_file isn't valid.
create_zip_file = True
else: # We assume that this is not Windows os
if artifact_object.is_dir():
# change to wildcard
artifact_object /= '*'
if create_zip_file:
folder = Path('').joinpath(*artifact_object.parts[:-1])
if not folder.is_dir() or not folder.parts:
raise ValueError("Artifact file/folder '{}' could not be found".format(
artifact_object.as_posix()))
wildcard = artifact_object.parts[-1]
files = list(Path(folder).rglob(wildcard))
override_filename_ext_in_uri = '.zip'
override_filename_in_uri = folder.parts[-1] + override_filename_ext_in_uri
fd, zip_file = mkstemp(
prefix=quote(folder.parts[-1], safe="") + '.', suffix=override_filename_ext_in_uri
)
try:
artifact_type_data.content_type = 'application/zip'
archive_preview = 'Archive content {}:\n'.format(artifact_object.as_posix())
with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf:
for filename in sorted(files):
if filename.is_file():
relative_file_name = filename.relative_to(folder).as_posix()
archive_preview += '{} - {}\n'.format(
relative_file_name, format_size(filename.stat().st_size))
zf.write(filename.as_posix(), arcname=relative_file_name)
except Exception as e:
# failed uploading folder:
LoggerRoot.get_base_logger().warning('Exception {}\nFailed zipping artifact folder {}'.format(
folder, e))
return False
finally:
os.close(fd)
artifact_type_data.preview = preview or archive_preview
artifact_object = zip_file
artifact_type = 'archive'
artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0]
local_filename = artifact_object
delete_after_upload = True
else:
if not artifact_object.is_file():
raise ValueError("Artifact file '{}' could not be found".format(artifact_object.as_posix()))
override_filename_in_uri = artifact_object.parts[-1]
artifact_type_data.preview = preview or '{} - {}\n'.format(
artifact_object, format_size(artifact_object.stat().st_size))
artifact_object = artifact_object.as_posix()
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0]
local_filename = artifact_object
elif (
isinstance(artifact_object, six.string_types) and len(artifact_object) < 4096
and urlparse(artifact_object).scheme in remote_driver_schemes
):
# we should not upload this, just register
local_filename = None
uri = artifact_object
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0]
if preview:
artifact_type_data.preview = preview
elif isinstance(artifact_object, six.string_types):
# if we got here, we should store it as text file.
artifact_type = 'string'
artifact_type_data.content_type = 'text/plain'
if preview:
artifact_type_data.preview = preview
elif len(artifact_object) < self.max_preview_size_bytes:
artifact_type_data.preview = artifact_object
else:
artifact_type_data.preview = '# full text too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes//1024, artifact_object[:self.max_preview_size_bytes]
)
delete_after_upload = True
override_filename_ext_in_uri = '.txt'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
os.close(fd)
# noinspection PyBroadException
try:
with open(local_filename, 'wt') as f:
f.write(artifact_object)
except Exception:
# cleanup and raise exception
os.unlink(local_filename)
raise
elif auto_pickle:
# if we are here it means we do not know what to do with the object, so we serialize it with pickle.
artifact_type = 'pickle'
artifact_type_data.content_type = 'application/pickle'
# noinspection PyBroadException
try:
artifact_type_data.preview = preview or str(artifact_object.__repr__())[:self.max_preview_size_bytes]
except Exception:
artifact_type_data.preview = preview or ''
delete_after_upload = True
override_filename_ext_in_uri = '.pkl'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
os.close(fd)
# noinspection PyBroadException
try:
with open(local_filename, 'wb') as f:
pickle.dump(artifact_object, f)
except Exception:
# cleanup and raise exception
os.unlink(local_filename)
raise
else:
raise ValueError("Artifact type {} not supported".format(type(artifact_object)))
# remove from existing list, if exists
for artifact in self._task_artifact_list:
if artifact.key == name:
if artifact.type == self._pd_artifact_type:
raise ValueError("Artifact of name {} already registered, "
"use register_artifact instead".format(name))
self._task_artifact_list.remove(artifact)
break
if not local_filename:
file_size = None
file_hash = None
else:
# check that the file to upload exists
local_filename = Path(local_filename).absolute()
if not local_filename.exists() or not local_filename.is_file():
LoggerRoot.get_base_logger().warning('Artifact upload failed, cannot find file {}'.format(
local_filename.as_posix()))
return False
file_hash, _ = sha256sum(local_filename.as_posix(), block_size=Artifacts._hash_block_size)
file_size = local_filename.stat().st_size
uri = self._upload_local_file(local_filename, name,
delete_after_upload=delete_after_upload,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri,
wait_on_upload=wait_on_upload)
timestamp = int(time())
artifact = tasks.Artifact(key=name, type=artifact_type,
uri=uri,
content_size=file_size,
hash=file_hash,
timestamp=timestamp,
type_data=artifact_type_data,
display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None)
# update task artifacts
with self._task_edit_lock:
self._task_artifact_list.append(artifact)
self._task.set_artifacts(self._task_artifact_list)
return True
def __call_api(self,
resource_path,
method,
path_params=None,
query_params=None,
header_params=None,
body=None,
post_params=None,
files=None,
response_type=None,
auth_settings=None,
_return_http_data_only=None,
collection_formats=None,
_preload_content=True,
_request_timeout=None,
_host=None):
config = self.configuration
# header parameters
header_params = header_params or {}
header_params.update(self.default_headers)
if self.cookie:
header_params['Cookie'] = self.cookie
if header_params:
header_params = self.sanitize_for_serialization(header_params)
header_params = dict(
self.parameters_to_tuples(header_params, collection_formats))
# path parameters
if path_params:
path_params = self.sanitize_for_serialization(path_params)
path_params = self.parameters_to_tuples(path_params,
collection_formats)
for k, v in path_params:
# specified safe chars, encode everything
resource_path = resource_path.replace(
'{%s}' % k,
quote(str(v), safe=config.safe_chars_for_path_param))
# query parameters
if query_params:
query_params = self.sanitize_for_serialization(query_params)
query_params = self.parameters_to_tuples(query_params,
collection_formats)
# post parameters
if post_params or files:
post_params = post_params if post_params else []
post_params = self.sanitize_for_serialization(post_params)
post_params = self.parameters_to_tuples(post_params,
collection_formats)
post_params.extend(self.files_parameters(files))
# auth setting
self.update_params_for_auth(header_params, query_params, auth_settings)
# body
if body:
body = self.sanitize_for_serialization(body)
# request url
if _host is None:
url = self.configuration.host + resource_path
else:
# use server/host defined in path or operation instead
url = _host + resource_path
try:
# perform request and return response
response_data = self.request(method,
url,
query_params=query_params,
headers=header_params,
post_params=post_params,
body=body,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
except ApiException as e:
e.body = e.body.decode('utf-8') if six.PY3 else e.body
raise e
content_type = response_data.getheader('content-type')
self.last_response = response_data
return_data = response_data
if not _preload_content:
return return_data
if six.PY3 and response_type not in ["file", "bytes"]:
match = None
if content_type is not None:
match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?",
content_type)
encoding = match.group(1) if match else "utf-8"
response_data.data = response_data.data.decode(encoding)
# deserialize response data
if response_type:
return_data = self.deserialize(response_data, response_type)
else:
return_data = None
if _return_http_data_only:
return (return_data)
else:
return (return_data, response_data.status,
response_data.getheaders())
def percent_encode(val):
# see https://en.wikipedia.org/wiki/Percent-encoding
if isinstance(val, six.text_type):
val = val.encode("utf8", errors="replace")
return quote(val).replace("%7E", "~").replace("/", "%2F")
def get_tool_version(self, tool_id, version="latest"):
url = "%s/ga4gh/v1/tools/%s/versions/%s" % (
self.base_url, quote(tool_id, safe=''), version)
return self._requests.get(url)
def safe(s):
return quote(s) if urlencode else s
def GET(self, req):
"""
Handle GET Bucket (List Objects) request
"""
max_keys = req.get_validated_param('max-keys',
self.conf.max_bucket_listing)
# TODO: Separate max_bucket_listing and default_bucket_listing
tag_max_keys = max_keys
max_keys = min(max_keys, self.conf.max_bucket_listing)
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
query = {
'format': 'json',
'limit': max_keys + 1,
}
if 'prefix' in req.params:
query.update({'prefix': req.params['prefix']})
if 'delimiter' in req.params:
query.update({'delimiter': req.params['delimiter']})
fetch_owner = False
if 'versions' in req.params:
listing_type = 'object-versions'
if 'key-marker' in req.params:
query.update({'marker': req.params['key-marker']})
elif 'version-id-marker' in req.params:
err_msg = ('A version-id marker cannot be specified without '
'a key marker.')
raise InvalidArgument('version-id-marker',
req.params['version-id-marker'], err_msg)
elif int(req.params.get('list-type', '1')) == 2:
listing_type = 'version-2'
if 'start-after' in req.params:
query.update({'marker': req.params['start-after']})
# continuation-token overrides start-after
if 'continuation-token' in req.params:
decoded = b64decode(req.params['continuation-token'])
query.update({'marker': decoded})
if 'fetch-owner' in req.params:
fetch_owner = config_true_value(req.params['fetch-owner'])
else:
listing_type = 'version-1'
if 'marker' in req.params:
query.update({'marker': req.params['marker']})
resp = req.get_response(self.app, query=query)
objects = json.loads(resp.body)
# in order to judge that truncated is valid, check whether
# max_keys + 1 th element exists in swift.
is_truncated = max_keys > 0 and len(objects) > max_keys
objects = objects[:max_keys]
if listing_type == 'object-versions':
elem = Element('ListVersionsResult')
SubElement(elem, 'Name').text = req.container_name
SubElement(elem, 'Prefix').text = req.params.get('prefix')
SubElement(elem, 'KeyMarker').text = req.params.get('key-marker')
SubElement(
elem,
'VersionIdMarker').text = req.params.get('version-id-marker')
if is_truncated:
if 'name' in objects[-1]:
SubElement(elem, 'NextKeyMarker').text = \
objects[-1]['name']
if 'subdir' in objects[-1]:
SubElement(elem, 'NextKeyMarker').text = \
objects[-1]['subdir']
SubElement(elem, 'NextVersionIdMarker').text = 'null'
else:
elem = Element('ListBucketResult')
SubElement(elem, 'Name').text = req.container_name
SubElement(elem, 'Prefix').text = req.params.get('prefix')
if listing_type == 'version-1':
SubElement(elem, 'Marker').text = req.params.get('marker')
if is_truncated and 'delimiter' in req.params:
if 'name' in objects[-1]:
name = objects[-1]['name']
else:
name = objects[-1]['subdir']
if encoding_type == 'url':
name = quote(name)
SubElement(elem, 'NextMarker').text = name
elif listing_type == 'version-2':
if is_truncated:
if 'name' in objects[-1]:
SubElement(elem, 'NextContinuationToken').text = \
b64encode(objects[-1]['name'].encode('utf8'))
if 'subdir' in objects[-1]:
SubElement(elem, 'NextContinuationToken').text = \
b64encode(objects[-1]['subdir'].encode('utf8'))
if 'continuation-token' in req.params:
SubElement(elem, 'ContinuationToken').text = \
req.params['continuation-token']
if 'start-after' in req.params:
SubElement(elem, 'StartAfter').text = \
req.params['start-after']
SubElement(elem, 'KeyCount').text = str(len(objects))
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
if 'delimiter' in req.params:
SubElement(elem, 'Delimiter').text = req.params['delimiter']
if encoding_type == 'url':
SubElement(elem, 'EncodingType').text = encoding_type
SubElement(elem, 'IsTruncated').text = \
'true' if is_truncated else 'false'
for o in objects:
if 'subdir' not in o:
name = o['name']
if encoding_type == 'url':
name = quote(name.encode('utf-8'))
if listing_type == 'object-versions':
contents = SubElement(elem, 'Version')
SubElement(contents, 'Key').text = name
SubElement(contents, 'VersionId').text = 'null'
SubElement(contents, 'IsLatest').text = 'true'
else:
contents = SubElement(elem, 'Contents')
SubElement(contents, 'Key').text = name
SubElement(contents, 'LastModified').text = \
o['last_modified'][:-3] + 'Z'
if 's3_etag' in o:
# New-enough MUs are already in the right format
etag = o['s3_etag']
else:
etag = '"%s"' % o['hash']
SubElement(contents, 'ETag').text = etag
SubElement(contents, 'Size').text = str(o['bytes'])
if fetch_owner or listing_type != 'version-2':
owner = SubElement(contents, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
SubElement(contents, 'StorageClass').text = 'STANDARD'
for o in objects:
if 'subdir' in o:
common_prefixes = SubElement(elem, 'CommonPrefixes')
name = o['subdir']
if encoding_type == 'url':
name = quote(name.encode('utf-8'))
SubElement(common_prefixes, 'Prefix').text = name
body = tostring(elem)
return HTTPOk(body=body, content_type='application/xml')
def generate_signed_url(self, expiration, method='GET',
content_type=None,
generation=None, response_disposition=None,
response_type=None, client=None, credentials=None):
"""Generates a signed URL for this blob.
.. note::
If you are on Google Compute Engine, you can't generate a signed
URL. Follow `Issue 922`_ for updates on this. If you'd like to
be able to generate a signed URL from GCE, you can use a standard
service account from a JSON file rather than a GCE service account.
.. _Issue 922: https://github.com/GoogleCloudPlatform/\
google-cloud-python/issues/922
If you have a blob that you want to allow access to for a set
amount of time, you can use this method to generate a URL that
is only valid within a certain time period.
This is particularly useful if you don't want publicly
accessible blobs, but don't want to require users to explicitly
log in.
:type expiration: int, long, datetime.datetime, datetime.timedelta
:param expiration: When the signed URL should expire.
:type method: str
:param method: The HTTP verb that will be used when requesting the URL.
:type content_type: str
:param content_type: (Optional) The content type of the object
referenced by ``resource``.
:type generation: str
:param generation: (Optional) A value that indicates which generation
of the resource to fetch.
:type response_disposition: str
:param response_disposition: (Optional) Content disposition of
responses to requests for the signed URL.
For example, to enable the signed URL
to initiate a file of ``blog.png``, use
the value
``'attachment; filename=blob.png'``.
:type response_type: str
:param response_type: (Optional) Content type of responses to requests
for the signed URL. Used to over-ride the content
type of the underlying blob/object.
:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: (Optional) The client to use. If not passed, falls back
to the ``client`` stored on the blob's bucket.
:type credentials: :class:`oauth2client.client.OAuth2Credentials` or
:class:`NoneType`
:param credentials: (Optional) The OAuth2 credentials to use to sign
the URL. Defaults to the credentials stored on the
client used.
:rtype: str
:returns: A signed URL you can use to access the resource
until expiration.
"""
resource = '/{bucket_name}/{quoted_name}'.format(
bucket_name=self.bucket.name,
quoted_name=quote(self.name, safe=''))
if credentials is None:
client = self._require_client(client)
credentials = client._base_connection.credentials
return generate_signed_url(
credentials, resource=resource,
api_access_endpoint=_API_ACCESS_ENDPOINT,
expiration=expiration, method=method,
content_type=content_type,
response_type=response_type,
response_disposition=response_disposition,
generation=generation)
def arn_url_path(self):
"""Return an ARN quoted correctly for use in a URL."""
return '/' + urlparse.quote(self.arn())
def test_lists_for_email(self):
email = "*****@*****.**"
self.cl.stub_request("clients/%s/listsforemail.json?email=%s" % (self.cl.client_id, quote(email)), "listsforemail.json")
lists = self.cl.lists_for_email(email)
self.assertEquals(len(lists), 2)
self.assertEquals(lists[0].ListID, 'ab4a2b57c7c8f1ba62f898a1af1a575b')
self.assertEquals(lists[0].ListName, 'List Number One')
self.assertEquals(lists[0].SubscriberState, 'Active')
self.assertEquals(lists[0].DateSubscriberAdded, '2012-08-20 22:32:00')
def as_pagination_marker(self, user):
return urllib_parse.quote(user.name)
def get_tool(self, tool_id):
url = "%s/ga4gh/v1/tools/%s" % (self.base_url, quote(tool_id, safe=''))
return self._requests.get(url)
def create_http_request(self, api_client):
# verify the required parameter 'name' is set
if self.name is None:
raise ValueError(
"Missing the required parameter `name` when calling `copy_style`"
) # noqa: E501
# verify the required parameter 'style_copy' is set
if self.style_copy is None:
raise ValueError(
"Missing the required parameter `style_copy` when calling `copy_style`"
) # noqa: E501
path = '/v4.0/words/{name}/styles/copy'
path_params = {}
if self.name is not None:
path_params['name'] = self.name # noqa: E501
else:
path_params['name'] = '' # noqa: E501
# path parameters
collection_formats = {}
if path_params:
path_params = api_client.sanitize_for_serialization(path_params)
path_params = api_client.parameters_to_tuples(
path_params, collection_formats)
for k, v in path_params:
# specified safe chars, encode everything
path = path.replace(
'{%s}' % k,
quote(str(v),
safe=api_client.configuration.
safe_chars_for_path_param))
# remove optional path parameters
path = path.replace('//', '/')
query_params = []
if self.folder is not None:
query_params.append(('folder', self.folder)) # noqa: E501
if self.storage is not None:
query_params.append(('storage', self.storage)) # noqa: E501
if self.load_encoding is not None:
query_params.append(
('loadEncoding', self.load_encoding)) # noqa: E501
if self.password is not None:
query_params.append(('password', self.password)) # noqa: E501
if self.dest_file_name is not None:
query_params.append(
('destFileName', self.dest_file_name)) # noqa: E501
if self.revision_author is not None:
query_params.append(
('revisionAuthor', self.revision_author)) # noqa: E501
if self.revision_date_time is not None:
query_params.append(
('revisionDateTime', self.revision_date_time)) # noqa: E501
header_params = {}
# HTTP header `Content-Type`
header_params[
'Content-Type'] = api_client.select_header_content_type( # noqa: E501
['application/xml', 'application/json']) # noqa: E501
form_params = []
body_params = None
if self.style_copy is not None:
body_params = self.style_copy
return {
"method": "POST",
"path": path,
"query_params": query_params,
"header_params": header_params,
"form_params": form_params,
"body": body_params,
"collection_formats": collection_formats,
"response_type": 'StyleResponse' # noqa: E501
}
def as_pagination_marker(self, user):
return urllib_parse.quote('%s@%s' % (user.name, user.host))
def __call_api(self,
resource_path,
method,
path_params=None,
query_params=None,
header_params=None,
body=None,
post_params=None,
files=None,
response_type=None,
auth_settings=None,
_return_http_data_only=None,
collection_formats=None,
_preload_content=True,
_request_timeout=None):
config = self.configuration
# header parameters
header_params = header_params or {}
header_params.update(self.default_headers)
if self.cookie:
header_params['Cookie'] = self.cookie
if header_params:
header_params = self.sanitize_for_serialization(header_params)
header_params = dict(
self.parameters_to_tuples(header_params, collection_formats))
# path parameters
if path_params:
path_params = self.sanitize_for_serialization(path_params)
path_params = self.parameters_to_tuples(path_params,
collection_formats)
for k, v in path_params:
# specified safe chars, encode everything
resource_path = resource_path.replace(
'{%s}' % k,
quote(str(v), safe=config.safe_chars_for_path_param))
# query parameters
if query_params:
query_params = self.sanitize_for_serialization(query_params)
query_params = self.parameters_to_tuples(query_params,
collection_formats)
# post parameters
if post_params or files:
post_params = self.prepare_post_parameters(post_params, files)
post_params = self.sanitize_for_serialization(post_params)
post_params = self.parameters_to_tuples(post_params,
collection_formats)
# auth setting
self.update_params_for_auth(header_params, query_params, auth_settings)
# body
if body:
body = self.sanitize_for_serialization(body)
# request url
url = self.configuration.host + resource_path
# perform request and return response
response_data = self.request(method,
url,
query_params=query_params,
headers=header_params,
post_params=post_params,
body=body,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
self.last_response = response_data
return_data = response_data
if _preload_content:
# deserialize response data
if response_type:
return_data = self.deserialize(response_data, response_type)
else:
return_data = None
if _return_http_data_only:
return (return_data)
else:
return (return_data, response_data.status,
response_data.getheaders())
def get_tool(self, tool_id):
url = "{}/ga4gh/v1/tools/{}".format(self.base_url,
quote(tool_id, safe=''))
return self._requests.get(url)
def quote_unicode(value):
"""Quote a unicode value by encoding it to bytes first."""
if isinstance(value, text_type):
return quote(value.encode(default_encoding, 'replace'))
return quote(value)
def set_outlet_name(self, outlet=0, name="Unknown"):
""" Set the name of an outlet """
self.determine_outlet(outlet)
self.geturl(url='unitnames.cgi?outname%s=%s' % (outlet, quote(name)))
return self.get_outlet_name(outlet) == name
