def base_handle_copy_request(self, src_container, src_obj, dest_container, dest_object): """ Unified path for: PUT verb with X-Copy-From and COPY verb with Destination """ # Get an iterator over the source object source_path = "/%s/%s/%s/%s" % (self.api_version, self.account, src_container, src_obj) source_req = self.request.copy_get() source_req.headers.pop("X-Backend-Storage-Policy-Index", None) source_req.headers.pop("X-Run-Storlet", None) source_req.path_info = source_path source_req.headers["X-Newest"] = "true" src_resp = source_req.get_response(self.app) sreq = self._build_storlet_request(self.request, src_resp.headers, src_resp.app_iter) self.gather_extra_sources() sresp = self.gateway.invocation_flow(sreq, self.extra_sources) resp = self.handle_put_copy_response(sresp.user_metadata, sresp.data_iter) acct, path = src_resp.environ["PATH_INFO"].split("/", 3)[2:4] resp.headers["X-Storlet-Generated-From-Account"] = quote(acct) resp.headers["X-Storlet-Generated-From"] = quote(path) if "last-modified" in src_resp.headers: resp.headers["X-Storlet-Generated-From-Last-Modified"] = src_resp.headers["last-modified"] return resp
def test_image_already_generated_by_thumbor_2_times(self): with open( normalize_unicode_path(u'./tests/fixtures/images/alabama1_ap620é.jpg'), 'r' ) as f: self.context.modules.storage.put( quote("http://test.com/smart/alabama1_ap620é"), f.read() ) crypto = CryptoURL('ACME-SEC') image_url = self.get_url( crypto.generate( image_url=quote(self.get_url( crypto.generate( image_url=quote("http://test.com/smart/alabama1_ap620é") ) )) ) ) url = crypto.generate( image_url=quote(image_url) ) response = self.fetch(url) expect(response.code).to_equal(200)
def sites_linking_in(self, urls, count=MAX_SITES_LINKING_IN_COUNT, start=0): if count > self.MAX_SITES_LINKING_IN_COUNT: raise RuntimeError("Maximum SitesLinkingIn result count is %s." % self.MAX_SITES_LINKING_IN_COUNT) params = { "Action": "SitesLinkingIn" } if not isinstance(urls, (list, tuple)): params.update({ "Url": urlparse.quote(urls), "ResponseGroup": "SitesLinkingIn", "Count": count, "Start": start, }) else: if len(urls) > self.MAX_BATCH_REQUESTS: raise RuntimeError("Maximum number of batch URLs is %s." % self.MAX_BATCH_REQUESTS) params.update({ "SitesLinkingIn.Shared.ResponseGroup": "SitesLinkingIn", "SitesLinkingIn.Shared.Count": count, "SitesLinkingIn.Shared.Start": start, }) for i, url in enumerate(urls): params.update({"SitesLinkingIn.%d.Url" % (i + 1): urlparse.quote(url)}) return self.request(params)
def join_params(params, encode=True): return "&".join( [ "%s=%s" % (urlparse.quote(key, safe="") if encode else key, urlparse.quote(val, safe="") if encode else val) for key, val in six.iteritems(params) ] )
def _dump_common_links(self, obj): """Dump common links for deposits and records.""" links = {} m = obj.get('metadata', {}) doi = m.get('doi') if doi: links['badge'] = ui_link_for('badge', doi=quote(doi)) links['doi'] = idutils.to_url(doi, 'doi', 'https') conceptdoi = m.get('conceptdoi') if conceptdoi: links['conceptbadge'] = ui_link_for('badge', doi=quote(conceptdoi)) links['conceptdoi'] = idutils.to_url(conceptdoi, 'doi', 'https') files = m.get('_files', []) for f in files: if f.get('type') in thumbnail_exts: try: links['thumb250'] = self._thumbnail_url(f, 250) # First previewable image is used for preview. except RuntimeError: pass break return links
def list(self, stack_id, resource_name=None, **kwargs): """Get a list of events. :param stack_id: ID of stack the events belong to :param resource_name: Optional name of resources to filter events by :rtype: list of :class:`Event` """ params = {} if 'filters' in kwargs: filters = kwargs.pop('filters') params.update(filters) for key, value in kwargs.items(): if value: params[key] = value if resource_name is None: url = '/stacks/%s/events' % stack_id else: stack_id = self._resolve_stack_id(stack_id) url = '/stacks/%s/resources/%s/events' % ( parse.quote(stack_id), parse.quote(encodeutils.safe_encode(resource_name))) if params: # convert to a sorted dict for python3 predictible order params = collections.OrderedDict(sorted(params.items())) url += '?%s' % parse.urlencode(params, True) return self._list(url, 'events')
def quote_user_host(user, host): quoted = '' if host: quoted = parse.quote("%s@%s" % (user, host)) else: quoted = parse.quote("%s" % user) return quoted.replace('.', '%2e')
def create_container(self, req, container_path): """ Checks if the container exists and if not try to create it. :params container_path: an unquoted path to a container to be created :returns: True if created container, False if container exists :raises CreateContainerError: when unable to create container """ head_cont_req = make_subrequest( req.environ, method='HEAD', path=quote(container_path), headers={'X-Auth-Token': req.headers.get('X-Auth-Token')}, swift_source='EA') resp = head_cont_req.get_response(self.app) if resp.is_success: return False if resp.status_int == HTTP_NOT_FOUND: create_cont_req = make_subrequest( req.environ, method='PUT', path=quote(container_path), headers={'X-Auth-Token': req.headers.get('X-Auth-Token')}, swift_source='EA') resp = create_cont_req.get_response(self.app) if resp.is_success: return True raise CreateContainerError( "Create Container Failed: " + container_path, resp.status_int, resp.status)
def base_handle_copy_request(self, src_container, src_obj, dest_container, dest_object): """ Unified path for: PUT verb with X-Copy-From and COPY verb with Destination """ # Get an iterator over the source object source_path = '/%s/%s/%s/%s' % (self.api_version, self.account, src_container, src_obj) source_req = self.request.copy_get() source_req.headers.pop('X-Backend-Storage-Policy-Index', None) source_req.headers.pop('X-Run-Storlet', None) source_req.path_info = source_path source_req.headers['X-Newest'] = 'true' source_resp = source_req.get_response(self.app) # Do proxy copy flow (out_md, app_iter) = self.gateway.gatewayProxyCopyFlow(self.request, source_resp) resp = self.handle_put_copy_response(out_md, app_iter) acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4] resp.headers['X-Storlet-Generated-From-Account'] = quote(acct) resp.headers['X-Storlet-Generated-From'] = quote(path) if 'last-modified' in source_resp.headers: resp.headers['X-Storlet-Generated-From-Last-Modified'] = \ source_resp.headers['last-modified'] return resp
def test_check_symlink_header_invalid_format(self): def do_test(headers, status, err_msg): req = Request.blank('/v1/a/c/o', method='PUT', headers=headers) with self.assertRaises(swob.HTTPException) as cm: symlink._check_symlink_header(req) self.assertEqual(cm.exception.status, status) self.assertEqual(cm.exception.body, err_msg) do_test({'X-Symlink-Target': '/c1/o1'}, '412 Precondition Failed', 'X-Symlink-Target header must be of the ' 'form <container name>/<object name>') do_test({'X-Symlink-Target': 'c1o1'}, '412 Precondition Failed', 'X-Symlink-Target header must be of the ' 'form <container name>/<object name>') do_test({'X-Symlink-Target': 'c1/o1', 'X-Symlink-Target-Account': '/another'}, '412 Precondition Failed', 'Account name cannot contain slashes') do_test({'X-Symlink-Target': 'c1/o1', 'X-Symlink-Target-Account': 'an/other'}, '412 Precondition Failed', 'Account name cannot contain slashes') # url encoded case do_test({'X-Symlink-Target': '%2Fc1%2Fo1'}, '412 Precondition Failed', 'X-Symlink-Target header must be of the ' 'form <container name>/<object name>') do_test({'X-Symlink-Target': 'c1/o1', 'X-Symlink-Target-Account': '%2Fanother'}, '412 Precondition Failed', 'Account name cannot contain slashes') do_test({'X-Symlink-Target': 'c1/o1', 'X-Symlink-Target-Account': 'an%2Fother'}, '412 Precondition Failed', 'Account name cannot contain slashes') # with multi-bytes do_test( {'X-Symlink-Target': u'/\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3'}, '412 Precondition Failed', 'X-Symlink-Target header must be of the ' 'form <container name>/<object name>') target = u'/\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3' encoded_target = quote(target.encode('utf-8'), '') do_test( {'X-Symlink-Target': encoded_target}, '412 Precondition Failed', 'X-Symlink-Target header must be of the ' 'form <container name>/<object name>') account = u'\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3' encoded_account = quote(account.encode('utf-8'), '') do_test( {'X-Symlink-Target': 'c/o', 'X-Symlink-Target-Account': encoded_account}, '412 Precondition Failed', 'Account name cannot contain slashes')
def handle_COPY(self, req, start_response, account, container, obj): if not req.headers.get('Destination'): return HTTPPreconditionFailed(request=req, body='Destination header required' )(req.environ, start_response) dest_account = account if 'Destination-Account' in req.headers: dest_account = unquote(req.headers.get('Destination-Account')) dest_account = check_account_format(req, dest_account) req.headers['X-Copy-From-Account'] = quote(account) account = dest_account del req.headers['Destination-Account'] dest_container, dest_object = _check_destination_header(req) source = '/%s/%s' % (container, obj) container = dest_container obj = dest_object # re-write the existing request as a PUT instead of creating a new one req.method = 'PUT' # As this the path info is updated with destination container, # the proxy server app will use the right object controller # implementation corresponding to the container's policy type. ver, _junk = req.split_path(1, 2, rest_with_last=True) req.path_info = '/%s/%s/%s/%s' % ( ver, dest_account, dest_container, dest_object) req.headers['Content-Length'] = 0 req.headers['X-Copy-From'] = quote(source) del req.headers['Destination'] return self.handle_PUT(req, start_response)
def test_list_alarms_sort_by_asc_desc(self): alarm_definition_ids, expected_metric = self._create_alarms_for_test_alarms(num=3) resp, response_body = self.monasca_client.list_alarms('?metric_name=' + expected_metric['name']) self._verify_list_alarms_elements(resp, response_body, expect_num_elements=3) resp, response_body = self.monasca_client.list_alarms('?metric_name=' + expected_metric['name'] + '&sort_by=' + urlparse.quote('created_timestamp asc')) self._verify_list_alarms_elements(resp, response_body, expect_num_elements=3) elements = response_body['elements'] last_timestamp = elements[0]['created_timestamp'] for element in elements: assert element['created_timestamp'] >= last_timestamp,\ "Created_timestamps are not in ascending order {} came before {}".format(last_timestamp, element['created_timestamp']) last_timestamp = element['created_timestamp'] resp, response_body = self.monasca_client.list_alarms('?metric_name=' + expected_metric['name'] + '&sort_by=' + urlparse.quote('created_timestamp desc')) self._verify_list_alarms_elements(resp, response_body, expect_num_elements=3) elements = response_body['elements'] last_timestamp = elements[0]['created_timestamp'] for element in elements: assert element['created_timestamp'] <= last_timestamp,\ "Created_timestamps are not in descending order {} came before {}".format(last_timestamp, element['created_timestamp']) last_timestamp = element['created_timestamp']
def stack_path(self): """Return a URL-encoded path segment of a URL without a tenant. Returned in the form: <stack_name>/<stack_id> """ return "%s/%s" % (urlparse.quote(self.stack_name, ""), urlparse.quote(self.stack_id, ""))
def _listing_pages_iter(self, account_name, lcontainer, lprefix, env): marker = '' while True: lreq = make_pre_authed_request( env, method='GET', swift_source='VW', path='/v1/%s/%s' % (account_name, lcontainer)) lreq.environ['QUERY_STRING'] = \ 'format=json&prefix=%s&marker=%s' % (quote(lprefix), quote(marker)) lresp = lreq.get_response(self.app) if not is_success(lresp.status_int): if lresp.status_int == HTTP_NOT_FOUND: raise ListingIterNotFound() elif is_client_error(lresp.status_int): raise HTTPPreconditionFailed() else: raise ListingIterError() if not lresp.body: break sublisting = json.loads(lresp.body) if not sublisting: break marker = sublisting[-1]['name'].encode('utf-8') yield sublisting
def id(self): """Return the id of this sequence.""" if self.sub_children: id_ = ",".join(quote(child.id) for child in self.template.children()) else: id_ = quote(self.template.id) return id_
def make_query(*args, **kwargs): '''Construct a URL query string, with marshalling markup. If there are positional arguments, they must be dictionaries. They are combined with the dictionary of keyword arguments to form a dictionary of query names and values. Query names (the keys) must be strings. Values may be strings, integers, floats, or DateTimes, and they may also be lists or namespaces containing these types. Names and string values should not be URL-quoted. All arguments are marshalled with complex_marshal(). ''' d = {} for arg in args: d.update(arg) d.update(kwargs) qlist = complex_marshal(list(d.items())) for i in range(len(qlist)): k, m, v = qlist[i] if PY2 and isinstance(v, text_type): v = v.encode(_default_encoding()) qlist[i] = '%s%s=%s' % (quote(k), m, quote(str(v))) return '&'.join(qlist)
def getEditTileURL(tile, request): """Get the edit URL for the given tile. If the tile is transient, the URL will contain a `_tiledata` parameter with the tile data encoded in JSON. This avoids possible collisions between raw data coming from the edit form and tile data retrieved by the transient tile data manager. """ id = tile.id name = tile.__name__ context = tile.__parent__ if name is None or context is None: raise TypeError("Insufficient context to determine URL") url = str(getMultiAdapter((context, request), IAbsoluteURL)) tileFragment = "@@edit-tile/" + urlparse.quote(name.encode('utf-8'), _safe) if id: tileFragment += '/' + urlparse.quote(id.encode('utf-8'), _safe) url = '%s/%s' % (url, tileFragment,) if not IPersistentTile.providedBy(tile): data = ITileDataManager(tile).get() if data: tileType = queryUtility(ITileType, name=name) if tileType is not None and tileType.schema is not None: if '?' in url: url += '&' + '_tiledata=' + json.dumps(data) else: url += '?' + '_tiledata=' + json.dumps(data) return url
def object_request(self, req, api_version, account, container, obj, allow_versioned_writes): container_name = unquote(container) object_name = unquote(obj) orig_container = get_unversioned_container(container_name) if orig_container != container_name: orig_object, version = \ swift3_split_object_name_version(object_name) req.environ['oio.query'] = {'version': version} req.environ['PATH_INFO'] = '/%s/%s/%s/%s' % (api_version, account, quote(orig_container), quote(orig_object)) elif req.method == 'DELETE': ver_mode = req.headers.get('X-Backend-Versioning-Mode-Override', 'history') if ver_mode == 'stack': # Do not create a delete marker, delete the latest version obj_inf = get_object_info(req.environ, self.app, swift_source='VW') req.environ['oio.query'] = { 'version': obj_inf.get('sysmeta', {}).get('version-id') } resp = req.get_response(self.app) if req.method == 'HEAD': close_if_possible(resp.app_iter) return resp
def notify(self, check): url = self.channel.value_down if check.status == "up": url = self.channel.value_up if not url: # If the URL is empty then we do nothing return "no-op" # Replace variables with actual values. # There should be no bad translations if users use $ symbol in # check's name or tags, because $ gets urlencoded to %24 if "$CODE" in url: url = url.replace("$CODE", str(check.code)) if "$STATUS" in url: url = url.replace("$STATUS", check.status) if "$NAME" in url: url = url.replace("$NAME", quote(check.name)) if "$TAG" in url: for i, tag in enumerate(check.tags_list()): placeholder = "$TAG%d" % (i + 1) url = url.replace(placeholder, quote(tag)) return self.get(url)
def _db_create_special( client, source_db, dest_db, kwargs): # Determine server location kwargs['location'] = get_server_location( server_name=dest_db.server_name, resource_group_name=dest_db.resource_group_name) # Set create mode properties subscription_id = get_subscription_id() kwargs['source_database_id'] = ( '/subscriptions/{}/resourceGroups/{}/providers/Microsoft.Sql/servers/{}/databases/{}' .format(quote(subscription_id), quote(source_db.resource_group_name), quote(source_db.server_name), quote(source_db.database_name))) # Create return client.create_or_update( server_name=dest_db.server_name, resource_group_name=dest_db.resource_group_name, database_name=dest_db.database_name, parameters=kwargs)
def list(self, stack_id, resource_name=None, **kwargs): """Get a list of events. :param stack_id: ID of stack the events belong to :param resource_name: Optional name of resources to filter events by :rtype: list of :class:`Event` """ params = {} if 'filters' in kwargs: filters = kwargs.pop('filters') params.update(filters) for key, value in six.iteritems(kwargs): if value: params[key] = value if resource_name is None: url = '/stacks/%s/events' % stack_id else: stack_id = self._resolve_stack_id(stack_id) url = '/stacks/%s/resources/%s/events' % ( parse.quote(stack_id, ''), parse.quote(strutils.safe_encode(resource_name), '')) if params: url += '?%s' % parse.urlencode(params, True) return self._list(url, 'events')
def __call__(self, env, start_response): request = Request(env) if not request.path.startswith(self.endpoints_path): return self.app(env, start_response) if request.method != 'GET': return HTTPMethodNotAllowed( req=request, headers={"Allow": "GET"})(env, start_response) try: version, account, container, obj = self._parse_path(request) except ValueError as err: return HTTPBadRequest(str(err))(env, start_response) if account is not None: account = unquote(account) if container is not None: container = unquote(container) if obj is not None: obj = unquote(obj) storage_policy_index = None if obj is not None: container_info = get_container_info( {'PATH_INFO': '/v1/%s/%s' % (account, container)}, self.app, swift_source='LE') storage_policy_index = container_info['storage_policy'] obj_ring = self.get_object_ring(storage_policy_index) partition, nodes = obj_ring.get_nodes( account, container, obj) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}/{container}/{obj}' elif container is not None: partition, nodes = self.container_ring.get_nodes( account, container) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}/{container}' else: partition, nodes = self.account_ring.get_nodes( account) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}' endpoints = [] for node in nodes: endpoint = endpoint_template.format( ip=node['ip'], port=node['port'], device=node['device'], partition=partition, account=quote(account), container=quote(container or ''), obj=quote(obj or '')) endpoints.append(endpoint) resp = self.response_map[version]( request, endpoints=endpoints, storage_policy_index=storage_policy_index) return resp(env, start_response)
def percent_encode(values): rv = {} for k, v in values.items(): if isinstance(v, six.string_types): rv[k] = quote(v) else: rv[k] = [quote(s) for s in v] return rv
def ZScriptHTML_tryAction(self, REQUEST, argvars): """Apply the test parameters. """ vv = [] for argvar in argvars: if argvar.value: vv.append("%s=%s" % (quote(argvar.name), quote(argvar.value))) raise Redirect("%s?%s" % (REQUEST['URL1'], '&'.join(vv)))
def stack_path(self): ''' Return a URL-encoded path segment of a URL, in the form: <stack_name>/<stack_id> ''' return '%s/%s' % (urlparse.quote(self.stack_name, ''), urlparse.quote(self.stack_id, ''))
def url_quote(v, name='(Unknown name)', md={}): if six.PY2 and isinstance(v, unicode): # quote does not handle unicode. Encoding to a "safe" # intermediate encoding before quoting, then unencoding the result. return quote(v.encode('utf-8')).decode('utf-8') elif six.PY3 and isinstance(v, bytes): return quote(v.decode('utf-8')).encode('utf-8') return quote(str(v))
def _get_canonical_querystring(self, params): querystring = [] for param in sorted(params): querystring.append('{}={}'.format( quote(param, safe='-_.~'), quote(str(params[param]), safe='-_.~'), )) return '&'.join(querystring)
def _listing_pages_iter(self, account_name, lcontainer, lprefix, env, marker='', end_marker='', reverse=True): '''Get "pages" worth of objects that start with a prefix. The optional keyword arguments ``marker``, ``end_marker``, and ``reverse`` are used similar to how they are for containers. We're either coming: - directly from ``_listing_iter``, in which case none of the optional args are specified, or - from ``_in_proxy_reverse_listing``, in which case ``reverse`` is ``False`` and both ``marker`` and ``end_marker`` are specified (although they may still be blank). ''' while True: lreq = make_pre_authed_request( env, method='GET', swift_source='VW', path='/v1/%s/%s' % (account_name, lcontainer)) lreq.environ['QUERY_STRING'] = \ 'format=json&prefix=%s&marker=%s' % ( quote(lprefix), quote(marker)) if end_marker: lreq.environ['QUERY_STRING'] += '&end_marker=%s' % ( quote(end_marker)) if reverse: lreq.environ['QUERY_STRING'] += '&reverse=on' lresp = lreq.get_response(self.app) if not is_success(lresp.status_int): close_if_possible(lresp.app_iter) if lresp.status_int == HTTP_NOT_FOUND: raise ListingIterNotFound() elif is_client_error(lresp.status_int): raise HTTPPreconditionFailed() else: raise ListingIterError() if not lresp.body: break sublisting = json.loads(lresp.body) if not sublisting: break # When using the ``reverse`` param, check that the listing is # actually reversed first_item = sublisting[0]['name'].encode('utf-8') last_item = sublisting[-1]['name'].encode('utf-8') page_is_after_marker = marker and first_item > marker if reverse and (first_item < last_item or page_is_after_marker): # Apparently there's at least one pre-2.6.0 container server yield self._in_proxy_reverse_listing( account_name, lcontainer, lprefix, env, marker, sublisting) return marker = last_item yield sublisting
def download_spk(category, kernel, download_dir=None): """Download generic kernel SPK files from http://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/ Parameters: category: asteroids, comets, lagrange_point, planets, satellites, or stations kernel: Kernel name, e.g. `de430` will download `de430.bsp` download_dir: Directory to download file to. By default, this is to a platform-dependent astrodynamics directory: :py:const:`SPK_DIR` """ valid_categories = ['asteroids', 'comets', 'lagrange_point', 'planets', 'satellites', 'stations'] if category not in valid_categories: s = 'Invalid category. Valid categories: {}' raise InvalidCategoryError(s.format(', '.join(valid_categories))) # We only wanted the name, strip the extension. if kernel.endswith('.bsp'): kernel = kernel[:-4] urls = [ SPK_URL.format(category=category, kernel=quote(kernel)), SPK_OLD_URL.format(category=category, kernel=quote(kernel)) ] # Get last path component for filename. # e.g. 'de423_for_mercury_and_venus/de423' => 'de423.bsp' # 'de421' => 'de421.bsp' filename = kernel.split('/')[-1] + '.bsp' if download_dir: download_dir = Path(download_dir) filepath = download_dir / filename else: with suppress_file_exists_error(): SPK_DIR.mkdir(parents=True) filepath = SPK_DIR / filename downloaded = False for url in urls: try: download_file_with_progress(url, str(filepath)) except HTTPError as exc: if exc.response.status_code != 404: raise else: print('Kernel downloaded to', filepath) downloaded = True break if not downloaded: s = 'Kernel not found in the following locations:\n{}' raise KernelNotFoundError(s.format('\n'.join(prefix(' ', urls))))
def setUp(self): super(ErrorPageEmbedTest, self).setUp() self.project = self.create_project() self.project.update_option('sentry:origins', ['example.com']) self.key = self.create_project_key(self.project) self.event_id = uuid4().hex self.path = '%s?eventId=%s&dsn=%s' % ( reverse('sentry-error-page-embed'), quote(self.event_id), quote(self.key.dsn_public), )
def _upload_data_audit_artifacts(self, name): # type: (str) -> () logger = self._task.get_logger() pd_artifact = self._artifacts_container.get(name) pd_metadata = self._artifacts_container.get_metadata(name) # remove from artifacts watch list if name in self._unregister_request: try: self._unregister_request.remove(name) except KeyError: pass self._artifacts_container.unregister_artifact(name) if pd_artifact is None: return override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_csv = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) local_csv = Path(local_csv) pd_artifact.to_csv(local_csv.as_posix(), index=False, compression=self._compression) current_sha2, file_sha2 = sha256sum( local_csv.as_posix(), skip_header=32, block_size=Artifacts._hash_block_size) if name in self._last_artifacts_upload: previous_sha2 = self._last_artifacts_upload[name] if previous_sha2 == current_sha2: # nothing to do, we can skip the upload # noinspection PyBroadException try: local_csv.unlink() except Exception: pass return self._last_artifacts_upload[name] = current_sha2 # If old clearml-server, upload as debug image if not Session.check_min_api_version('2.3'): logger.report_image(title='artifacts', series=name, local_path=local_csv.as_posix(), delete_after_upload=True, iteration=self._task.get_last_iteration(), max_image_history=2) return # Find our artifact artifact = None for an_artifact in self._task_artifact_list: if an_artifact.key == name: artifact = an_artifact break file_size = local_csv.stat().st_size # upload file uri = self._upload_local_file(local_csv, name, delete_after_upload=True, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri) # update task artifacts with self._task_edit_lock: if not artifact: artifact = tasks.Artifact(key=name, type=self._pd_artifact_type) self._task_artifact_list.append(artifact) artifact_type_data = tasks.ArtifactTypeData() artifact_type_data.data_hash = current_sha2 artifact_type_data.content_type = "text/csv" artifact_type_data.preview = str(pd_artifact.__repr__( )) + '\n\n' + self._get_statistics({name: pd_artifact}) artifact.type_data = artifact_type_data artifact.uri = uri artifact.content_size = file_size artifact.hash = file_sha2 artifact.timestamp = int(time()) artifact.display_data = [(str(k), str(v)) for k, v in pd_metadata.items()] if pd_metadata else None self._task.set_artifacts(self._task_artifact_list)
def get_tool_descriptor(self, tool_id, version="latest", tool_type="CWL"): url = "%s/ga4gh/v1/tools/%s/versions/%s/%s/descriptor" % ( self.base_url, quote(tool_id, safe=''), version, tool_type) return self._requests.get(url)
def percent_encode(val): # see https://en.wikipedia.org/wiki/Percent-encoding return quote(val.encode('utf8', errors='replace')).replace('%7E', '~').replace( '/', '%2F')
def publish_sample(self, env, bytes_received, bytes_sent): path = urlparse.quote(env['PATH_INFO']) method = env['REQUEST_METHOD'] headers = {} for header in env: if header.startswith('HTTP_') and env[header]: key = header.strip('HTTP_') if isinstance(env[header], six.text_type): headers[key] = env[header].encode('utf-8') else: headers[key] = str(env[header]) try: container = obj = None version, account, remainder = path.replace( '/', '', 1).split('/', 2) if not version or not account: raise ValueError('Invalid path: %s' % path) if remainder: if '/' in remainder: container, obj = remainder.split('/', 1) else: container = remainder except ValueError: return now = timeutils.utcnow().isoformat() resource_metadata = { "path": path, "version": version, "container": container, "object": obj, } for header in self.metadata_headers: if header.upper() in headers: resource_metadata['http_header_%s' % header] = headers.get( header.upper()) with self.pipeline_manager.publisher( context.get_admin_context()) as publisher: if bytes_received: publisher([sample.Sample( name='storage.objects.incoming.bytes', type=sample.TYPE_DELTA, unit='B', volume=bytes_received, user_id=env.get('HTTP_X_USER_ID'), project_id=env.get('HTTP_X_TENANT_ID'), resource_id=account.partition(self.reseller_prefix)[2], timestamp=now, resource_metadata=resource_metadata)]) if bytes_sent: publisher([sample.Sample( name='storage.objects.outgoing.bytes', type=sample.TYPE_DELTA, unit='B', volume=bytes_sent, user_id=env.get('HTTP_X_USER_ID'), project_id=env.get('HTTP_X_TENANT_ID'), resource_id=account.partition(self.reseller_prefix)[2], timestamp=now, resource_metadata=resource_metadata)]) # publish the event for each request # request method will be recorded in the metadata resource_metadata['method'] = method.lower() publisher([sample.Sample( name='storage.api.request', type=sample.TYPE_DELTA, unit='request', volume=1, user_id=env.get('HTTP_X_USER_ID'), project_id=env.get('HTTP_X_TENANT_ID'), resource_id=account.partition(self.reseller_prefix)[2], timestamp=now, resource_metadata=resource_metadata)])
def create_http_request(self, api_client): # verify the required parameter 'name' is set if self.name is None: raise ValueError("Missing the required parameter `name` when calling `get_office_math_object`") # noqa: E501 # verify the required parameter 'index' is set if self.index is None: raise ValueError("Missing the required parameter `index` when calling `get_office_math_object`") # noqa: E501 path = '/v4.0/words/{name}/{nodePath}/OfficeMathObjects/{index}' path_params = {} if self.name is not None: path_params['name'] = self.name # noqa: E501 else: path_params['name'] = '' # noqa: E501 if self.index is not None: path_params['index'] = self.index # noqa: E501 else: path_params['index'] = '' # noqa: E501 if self.node_path is not None: path_params['nodePath'] = self.node_path # noqa: E501 else: path_params['nodePath'] = '' # noqa: E501 # path parameters collection_formats = {} if path_params: path_params = api_client.sanitize_for_serialization(path_params) path_params = api_client.parameters_to_tuples(path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything path = path.replace( '{%s}' % k, quote(str(v), safe=api_client.configuration.safe_chars_for_path_param) ) # remove optional path parameters path = path.replace('//', '/') query_params = [] if self.folder is not None: query_params.append(('folder', self.folder)) # noqa: E501 if self.storage is not None: query_params.append(('storage', self.storage)) # noqa: E501 if self.load_encoding is not None: query_params.append(('loadEncoding', self.load_encoding)) # noqa: E501 if self.password is not None: query_params.append(('password', self.password)) # noqa: E501 header_params = {} form_params = [] body_params = None return { "method": "GET", "path": path, "query_params": query_params, "header_params": header_params, "form_params": form_params, "body": body_params, "collection_formats": collection_formats, "response_type": 'OfficeMathObjectResponse' # noqa: E501 }
def _get_output_destination_suffix(self, extra_path=None): return '/'.join( quote(x, safe="'[]{}()$^,.; -_+-=") for x in (self.get_project_name(), '%s.%s' % (self.name, self.data.id), extra_path) if x)
def generate_signed_url(service_account_file, bucket_name, object_name, expiration, http_method='GET', query_parameters=None, headers=None): if expiration > 604800: print('Expiration Time can\'t be longer than 604800 seconds (7 days).') sys.exit(1) # [START storage_signed_url_canonical_uri] escaped_object_name = quote(six.ensure_binary(object_name), safe=b'/~') canonical_uri = '/{}/{}'.format(bucket_name, escaped_object_name) # [END storage_signed_url_canonical_uri] # [START storage_signed_url_canonical_datetime] datetime_now = datetime.datetime.utcnow() request_timestamp = datetime_now.strftime('%Y%m%dT%H%M%SZ') datestamp = datetime_now.strftime('%Y%m%d') # [END storage_signed_url_canonical_datetime] # [START storage_signed_url_credentials] # [START storage_signed_url_signer] google_credentials = service_account.Credentials.from_service_account_file( service_account_file) # [END storage_signed_url_signer] client_email = google_credentials.service_account_email credential_scope = '{}/auto/storage/goog4_request'.format(datestamp) credential = '{}/{}'.format(client_email, credential_scope) # [END storage_signed_url_credentials] if headers is None: headers = dict() # [START storage_signed_url_canonical_headers] headers['host'] = 'storage.googleapis.com' canonical_headers = '' ordered_headers = collections.OrderedDict(sorted(headers.items())) for k, v in ordered_headers.items(): lower_k = str(k).lower() strip_v = str(v).lower() canonical_headers += '{}:{}\n'.format(lower_k, strip_v) # [END storage_signed_url_canonical_headers] # [START storage_signed_url_signed_headers] signed_headers = '' for k, _ in ordered_headers.items(): lower_k = str(k).lower() signed_headers += '{};'.format(lower_k) signed_headers = signed_headers[:-1] # remove trailing ';' # [END storage_signed_url_signed_headers] if query_parameters is None: query_parameters = dict() # [START storage_signed_url_canonical_query_parameters] query_parameters['X-Goog-Algorithm'] = 'GOOG4-RSA-SHA256' query_parameters['X-Goog-Credential'] = credential query_parameters['X-Goog-Date'] = request_timestamp query_parameters['X-Goog-Expires'] = expiration query_parameters['X-Goog-SignedHeaders'] = signed_headers canonical_query_string = '' ordered_query_parameters = collections.OrderedDict( sorted(query_parameters.items())) for k, v in ordered_query_parameters.items(): encoded_k = quote(str(k), safe='') encoded_v = quote(str(v), safe='') canonical_query_string += '{}={}&'.format(encoded_k, encoded_v) canonical_query_string = canonical_query_string[:-1] # remove trailing ';' # [END storage_signed_url_canonical_query_parameters] # [START storage_signed_url_canonical_request] canonical_request = '\n'.join([ http_method, canonical_uri, canonical_query_string, canonical_headers, signed_headers, 'UNSIGNED-PAYLOAD' ]) # [END storage_signed_url_canonical_request] # [START storage_signed_url_hash] canonical_request_hash = hashlib.sha256( canonical_request.encode()).hexdigest() # [END storage_signed_url_hash] # [START storage_signed_url_string_to_sign] string_to_sign = '\n'.join([ 'GOOG4-RSA-SHA256', request_timestamp, credential_scope, canonical_request_hash ]) # [END storage_signed_url_string_to_sign] # [START storage_signed_url_signer] signature = binascii.hexlify( google_credentials.signer.sign(string_to_sign)).decode() # [END storage_signed_url_signer] # [START storage_signed_url_construction] host_name = 'https://storage.googleapis.com' signed_url = '{}{}?{}&X-Goog-Signature={}'.format(host_name, canonical_uri, canonical_query_string, signature) # [END storage_signed_url_construction] return signed_url
def login_post(): self.redirect_home_if_logged_in() """Authenticate users""" username = self.post_get('username') password = self.post_get('password') try: move_info = self.get_move_temp_info() except ValidationException as ve: self.flash_message('Login Failed: ' + str(ve)) self.redirect('/') return # if a collection is being moved, auth user # and then check for available space # if not enough space, don't continue with login if move_info and (self.manager.cork.is_authenticate( username, password)): if not self.manager.has_space_for_new_coll( username, move_info['from_user'], 'temp'): self.flash_message( 'Sorry, not enough space to import this Temporary Collection into your account.' ) self.redirect('/') return if not self.manager.cork.login(username, password): self.flash_message('Invalid Login. Please Try Again') redir_to = LOGIN_PATH self.redirect(redir_to) sesh = self.get_session() sesh.curr_user = username if move_info: try: new_title = self.manager.move_temp_coll( username, move_info) if new_title: self.flash_message( 'Collection <b>{0}</b> created!'.format(new_title), 'success') except: import traceback traceback.print_exc() remember_me = (self.post_get('remember_me') == '1') sesh.logged_in(remember_me) temp_prefix = self.manager.temp_prefix redir_to = request.headers.get('Referer') host = self.get_host() if redir_to and redir_to.startswith(host): redir_to = redir_to[len(host):] if not redir_to or redir_to.startswith(('/' + temp_prefix, '/_')): redir_to = self.get_path(username) if self.content_host: path = '/_clear_session?path=' + quote(redir_to) self.redir_host(self.content_host, path) else: self.redirect(redir_to)
def download_df(data_df): df = pd.DataFrame.from_dict(data_df, 'columns') csv_string = df.to_csv(index=False, encoding='utf-8') csv_string = "data:text/csv;charset=utf-8," + quote(csv_string) return csv_string
def test_unsuppress(self): email = "*****@*****.**" self.cl.stub_request("clients/%s/unsuppress.json?email=%s" % (self.cl.client_id, quote(email)), None) res = self.cl.unsuppress(email)
def test_set_primary_contact(self): email = '*****@*****.**' self.cl.stub_request("clients/%s/primarycontact.json?email=%s" % (self.cl.client_id, quote(email, '')), 'client_set_primary_contact.json') result = self.cl.set_primary_contact(email) self.assertEquals(email, result.EmailAddress)
def handle_extract_iter(self, req, compress_type, out_content_type='text/plain'): """ A generator that can be assigned to a swob Response's app_iter which, when iterated over, will extract and PUT the objects pulled from the request body. Will occasionally yield whitespace while request is being processed. When the request is completed will yield a response body that can be parsed to determine success. See above documentation for details. :params req: a swob Request :params compress_type: specifying the compression type of the tar. Accepts '', 'gz', or 'bz2' """ resp_dict = { 'Response Status': HTTPCreated().status, 'Response Body': '', 'Number Files Created': 0 } failed_files = [] last_yield = time() separator = '' containers_accessed = set() req.environ['eventlet.minimum_write_chunk_size'] = 0 try: if not out_content_type: raise HTTPNotAcceptable(request=req) if out_content_type.endswith('/xml'): yield '<?xml version="1.0" encoding="UTF-8"?>\n' if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) try: vrs, account, extract_base = req.split_path(2, 3, True) except ValueError: raise HTTPNotFound(request=req) extract_base = extract_base or '' extract_base = extract_base.rstrip('/') tar = tarfile.open(mode='r|' + compress_type, fileobj=req.body_file) failed_response_type = HTTPBadRequest containers_created = 0 while True: if last_yield + self.yield_frequency < time(): separator = '\r\n\r\n' last_yield = time() yield ' ' tar_info = next(tar) if tar_info is None or \ len(failed_files) >= self.max_failed_extractions: break if tar_info.isfile(): obj_path = tar_info.name if obj_path.startswith('./'): obj_path = obj_path[2:] obj_path = obj_path.lstrip('/') if extract_base: obj_path = extract_base + '/' + obj_path if '/' not in obj_path: continue # ignore base level file destination = '/'.join(['', vrs, account, obj_path]) container = obj_path.split('/', 1)[0] if not constraints.check_utf8(destination): failed_files.append([ quote(obj_path[:self.max_path_length]), HTTPPreconditionFailed().status ]) continue if tar_info.size > constraints.MAX_FILE_SIZE: failed_files.append([ quote(obj_path[:self.max_path_length]), HTTPRequestEntityTooLarge().status ]) continue container_failure = None if container not in containers_accessed: cont_path = '/'.join(['', vrs, account, container]) try: if self.create_container(req, cont_path): containers_created += 1 if containers_created > self.max_containers: raise HTTPBadRequest( 'More than %d containers to create ' 'from tar.' % self.max_containers) except CreateContainerError as err: # the object PUT to this container still may # succeed if acls are set container_failure = [ quote(cont_path[:self.max_path_length]), err.status ] if err.status_int == HTTP_UNAUTHORIZED: raise HTTPUnauthorized(request=req) except ValueError: failed_files.append([ quote(obj_path[:self.max_path_length]), HTTPBadRequest().status ]) continue tar_file = tar.extractfile(tar_info) create_headers = { 'Content-Length': tar_info.size, 'X-Auth-Token': req.headers.get('X-Auth-Token'), } create_obj_req = make_subrequest( req.environ, method='PUT', path=quote(destination), headers=create_headers, agent='%(orig)s BulkExpand', swift_source='EA') create_obj_req.environ['wsgi.input'] = tar_file for pax_key, pax_value in tar_info.pax_headers.items(): header_name = pax_key_to_swift_header(pax_key) if header_name: # Both pax_key and pax_value are unicode # strings; the key is already UTF-8 encoded, but # we still have to encode the value. create_obj_req.headers[header_name] = \ pax_value.encode("utf-8") resp = create_obj_req.get_response(self.app) containers_accessed.add(container) if resp.is_success: resp_dict['Number Files Created'] += 1 else: if container_failure: failed_files.append(container_failure) if resp.status_int == HTTP_UNAUTHORIZED: failed_files.append([ quote(obj_path[:self.max_path_length]), HTTPUnauthorized().status ]) raise HTTPUnauthorized(request=req) if resp.status_int // 100 == 5: failed_response_type = HTTPBadGateway failed_files.append([ quote(obj_path[:self.max_path_length]), resp.status ]) if failed_files: resp_dict['Response Status'] = failed_response_type().status elif not resp_dict['Number Files Created']: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files' except HTTPException as err: resp_dict['Response Status'] = err.status resp_dict['Response Body'] = err.body except (tarfile.TarError, zlib.error) as tar_error: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error except Exception: self.logger.exception('Error in extract archive.') resp_dict['Response Status'] = HTTPServerError().status yield separator + get_response_body(out_content_type, resp_dict, failed_files, 'extract')
def create_http_request(self, api_client): # verify the required parameter 'document' is set if self.document is None: raise ValueError( "Missing the required parameter `document` when calling `delete_border_online`" ) # noqa: E501 # verify the required parameter 'border_type' is set if self.border_type is None: raise ValueError( "Missing the required parameter `border_type` when calling `delete_border_online`" ) # noqa: E501 path = '/v4.0/words/online/delete/{nodePath}/borders/{borderType}' path_params = {} if self.border_type is not None: path_params['borderType'] = self.border_type # noqa: E501 else: path_params['borderType'] = '' # noqa: E501 if self.node_path is not None: path_params['nodePath'] = self.node_path # noqa: E501 else: path_params['nodePath'] = '' # noqa: E501 # path parameters collection_formats = {} if path_params: path_params = api_client.sanitize_for_serialization(path_params) path_params = api_client.parameters_to_tuples( path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything path = path.replace( '{%s}' % k, quote(str(v), safe=api_client.configuration. safe_chars_for_path_param)) # remove optional path parameters path = path.replace('//', '/') query_params = [] if self.load_encoding is not None: query_params.append( ('loadEncoding', self.load_encoding)) # noqa: E501 if self.password is not None: query_params.append(('password', self.password)) # noqa: E501 if self.dest_file_name is not None: query_params.append( ('destFileName', self.dest_file_name)) # noqa: E501 if self.revision_author is not None: query_params.append( ('revisionAuthor', self.revision_author)) # noqa: E501 if self.revision_date_time is not None: query_params.append( ('revisionDateTime', self.revision_date_time)) # noqa: E501 header_params = {} # HTTP header `Content-Type` header_params[ 'Content-Type'] = api_client.select_header_content_type( # noqa: E501 ['multipart/form-data']) # noqa: E501 form_params = [] if self.document is not None: form_params.append(['document', self.document, 'file']) # noqa: E501 body_params = None return { "method": "PUT", "path": path, "query_params": query_params, "header_params": header_params, "form_params": form_params, "body": body_params, "collection_formats": collection_formats, "response_type": 'DeleteBorderOnlineResponse' # noqa: E501 }
def upload_artifact(self, name, artifact_object=None, metadata=None, preview=None, delete_after_upload=False, auto_pickle=True, wait_on_upload=False): # type: (str, Optional[object], Optional[dict], Optional[str], bool, bool, bool) -> bool if not Session.check_min_api_version('2.3'): LoggerRoot.get_base_logger().warning('Artifacts not supported by your ClearML-server version, ' 'please upgrade to the latest server version') return False if name in self._artifacts_container: raise ValueError("Artifact by the name of {} is already registered, use register_artifact".format(name)) # cast preview to string if preview: preview = str(preview) # try to convert string Path object (it might reference a file/folder) # dont not try to serialize long texts. if isinstance(artifact_object, six.string_types) and len(artifact_object) < 2048: # noinspection PyBroadException try: artifact_path = Path(artifact_object) if artifact_path.exists(): artifact_object = artifact_path elif '*' in artifact_object or '?' in artifact_object: # hackish, detect wildcard in tr files folder = Path('').joinpath(*artifact_path.parts[:-1]) if folder.is_dir() and folder.parts: wildcard = artifact_path.parts[-1] if list(Path(folder).rglob(wildcard)): artifact_object = artifact_path except Exception: pass artifact_type_data = tasks.ArtifactTypeData() artifact_type_data.preview = '' override_filename_in_uri = None override_filename_ext_in_uri = None uri = None if np and isinstance(artifact_object, np.ndarray): artifact_type = 'numpy' artifact_type_data.content_type = 'application/numpy' artifact_type_data.preview = preview or str(artifact_object.__repr__()) override_filename_ext_in_uri = '.npz' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) np.savez_compressed(local_filename, **{name: artifact_object}) delete_after_upload = True elif pd and isinstance(artifact_object, pd.DataFrame): artifact_type = 'pandas' artifact_type_data.content_type = 'text/csv' artifact_type_data.preview = preview or str(artifact_object.__repr__()) override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.to_csv(local_filename, compression=self._compression) delete_after_upload = True elif isinstance(artifact_object, Image.Image): artifact_type = 'image' artifact_type_data.content_type = 'image/png' desc = str(artifact_object.__repr__()) artifact_type_data.preview = preview or desc[1:desc.find(' at ')] override_filename_ext_in_uri = '.png' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.save(local_filename) delete_after_upload = True elif isinstance(artifact_object, dict): artifact_type = 'JSON' artifact_type_data.content_type = 'application/json' json_text = json.dumps(artifact_object, sort_keys=True, indent=4) override_filename_ext_in_uri = '.json' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.write(fd, bytes(json_text.encode())) os.close(fd) preview = preview or json_text if len(preview) < self.max_preview_size_bytes: artifact_type_data.preview = preview else: artifact_type_data.preview = '# full json too large to store, storing first {}kb\n{}'.format( self.max_preview_size_bytes//1024, preview[:self.max_preview_size_bytes] ) delete_after_upload = True elif isinstance(artifact_object, (Path, pathlib_Path,) if pathlib_Path is not None else (Path,)): # check if single file artifact_object = Path(artifact_object) artifact_object.expanduser().absolute() # noinspection PyBroadException try: create_zip_file = not artifact_object.is_file() except Exception: # Hack for windows pathlib2 bug, is_file isn't valid. create_zip_file = True else: # We assume that this is not Windows os if artifact_object.is_dir(): # change to wildcard artifact_object /= '*' if create_zip_file: folder = Path('').joinpath(*artifact_object.parts[:-1]) if not folder.is_dir() or not folder.parts: raise ValueError("Artifact file/folder '{}' could not be found".format( artifact_object.as_posix())) wildcard = artifact_object.parts[-1] files = list(Path(folder).rglob(wildcard)) override_filename_ext_in_uri = '.zip' override_filename_in_uri = folder.parts[-1] + override_filename_ext_in_uri fd, zip_file = mkstemp( prefix=quote(folder.parts[-1], safe="") + '.', suffix=override_filename_ext_in_uri ) try: artifact_type_data.content_type = 'application/zip' archive_preview = 'Archive content {}:\n'.format(artifact_object.as_posix()) with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf: for filename in sorted(files): if filename.is_file(): relative_file_name = filename.relative_to(folder).as_posix() archive_preview += '{} - {}\n'.format( relative_file_name, format_size(filename.stat().st_size)) zf.write(filename.as_posix(), arcname=relative_file_name) except Exception as e: # failed uploading folder: LoggerRoot.get_base_logger().warning('Exception {}\nFailed zipping artifact folder {}'.format( folder, e)) return False finally: os.close(fd) artifact_type_data.preview = preview or archive_preview artifact_object = zip_file artifact_type = 'archive' artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0] local_filename = artifact_object delete_after_upload = True else: if not artifact_object.is_file(): raise ValueError("Artifact file '{}' could not be found".format(artifact_object.as_posix())) override_filename_in_uri = artifact_object.parts[-1] artifact_type_data.preview = preview or '{} - {}\n'.format( artifact_object, format_size(artifact_object.stat().st_size)) artifact_object = artifact_object.as_posix() artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0] local_filename = artifact_object elif ( isinstance(artifact_object, six.string_types) and len(artifact_object) < 4096 and urlparse(artifact_object).scheme in remote_driver_schemes ): # we should not upload this, just register local_filename = None uri = artifact_object artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0] if preview: artifact_type_data.preview = preview elif isinstance(artifact_object, six.string_types): # if we got here, we should store it as text file. artifact_type = 'string' artifact_type_data.content_type = 'text/plain' if preview: artifact_type_data.preview = preview elif len(artifact_object) < self.max_preview_size_bytes: artifact_type_data.preview = artifact_object else: artifact_type_data.preview = '# full text too large to store, storing first {}kb\n{}'.format( self.max_preview_size_bytes//1024, artifact_object[:self.max_preview_size_bytes] ) delete_after_upload = True override_filename_ext_in_uri = '.txt' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) # noinspection PyBroadException try: with open(local_filename, 'wt') as f: f.write(artifact_object) except Exception: # cleanup and raise exception os.unlink(local_filename) raise elif auto_pickle: # if we are here it means we do not know what to do with the object, so we serialize it with pickle. artifact_type = 'pickle' artifact_type_data.content_type = 'application/pickle' # noinspection PyBroadException try: artifact_type_data.preview = preview or str(artifact_object.__repr__())[:self.max_preview_size_bytes] except Exception: artifact_type_data.preview = preview or '' delete_after_upload = True override_filename_ext_in_uri = '.pkl' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) # noinspection PyBroadException try: with open(local_filename, 'wb') as f: pickle.dump(artifact_object, f) except Exception: # cleanup and raise exception os.unlink(local_filename) raise else: raise ValueError("Artifact type {} not supported".format(type(artifact_object))) # remove from existing list, if exists for artifact in self._task_artifact_list: if artifact.key == name: if artifact.type == self._pd_artifact_type: raise ValueError("Artifact of name {} already registered, " "use register_artifact instead".format(name)) self._task_artifact_list.remove(artifact) break if not local_filename: file_size = None file_hash = None else: # check that the file to upload exists local_filename = Path(local_filename).absolute() if not local_filename.exists() or not local_filename.is_file(): LoggerRoot.get_base_logger().warning('Artifact upload failed, cannot find file {}'.format( local_filename.as_posix())) return False file_hash, _ = sha256sum(local_filename.as_posix(), block_size=Artifacts._hash_block_size) file_size = local_filename.stat().st_size uri = self._upload_local_file(local_filename, name, delete_after_upload=delete_after_upload, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri, wait_on_upload=wait_on_upload) timestamp = int(time()) artifact = tasks.Artifact(key=name, type=artifact_type, uri=uri, content_size=file_size, hash=file_hash, timestamp=timestamp, type_data=artifact_type_data, display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None) # update task artifacts with self._task_edit_lock: self._task_artifact_list.append(artifact) self._task.set_artifacts(self._task_artifact_list) return True
def __call_api(self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): config = self.configuration # header parameters header_params = header_params or {} header_params.update(self.default_headers) if self.cookie: header_params['Cookie'] = self.cookie if header_params: header_params = self.sanitize_for_serialization(header_params) header_params = dict( self.parameters_to_tuples(header_params, collection_formats)) # path parameters if path_params: path_params = self.sanitize_for_serialization(path_params) path_params = self.parameters_to_tuples(path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything resource_path = resource_path.replace( '{%s}' % k, quote(str(v), safe=config.safe_chars_for_path_param)) # query parameters if query_params: query_params = self.sanitize_for_serialization(query_params) query_params = self.parameters_to_tuples(query_params, collection_formats) # post parameters if post_params or files: post_params = post_params if post_params else [] post_params = self.sanitize_for_serialization(post_params) post_params = self.parameters_to_tuples(post_params, collection_formats) post_params.extend(self.files_parameters(files)) # auth setting self.update_params_for_auth(header_params, query_params, auth_settings) # body if body: body = self.sanitize_for_serialization(body) # request url if _host is None: url = self.configuration.host + resource_path else: # use server/host defined in path or operation instead url = _host + resource_path try: # perform request and return response response_data = self.request(method, url, query_params=query_params, headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, _request_timeout=_request_timeout) except ApiException as e: e.body = e.body.decode('utf-8') if six.PY3 else e.body raise e content_type = response_data.getheader('content-type') self.last_response = response_data return_data = response_data if not _preload_content: return return_data if six.PY3 and response_type not in ["file", "bytes"]: match = None if content_type is not None: match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?", content_type) encoding = match.group(1) if match else "utf-8" response_data.data = response_data.data.decode(encoding) # deserialize response data if response_type: return_data = self.deserialize(response_data, response_type) else: return_data = None if _return_http_data_only: return (return_data) else: return (return_data, response_data.status, response_data.getheaders())
def percent_encode(val): # see https://en.wikipedia.org/wiki/Percent-encoding if isinstance(val, six.text_type): val = val.encode("utf8", errors="replace") return quote(val).replace("%7E", "~").replace("/", "%2F")
def get_tool_version(self, tool_id, version="latest"): url = "%s/ga4gh/v1/tools/%s/versions/%s" % ( self.base_url, quote(tool_id, safe=''), version) return self._requests.get(url)
def safe(s): return quote(s) if urlencode else s
def GET(self, req): """ Handle GET Bucket (List Objects) request """ max_keys = req.get_validated_param('max-keys', self.conf.max_bucket_listing) # TODO: Separate max_bucket_listing and default_bucket_listing tag_max_keys = max_keys max_keys = min(max_keys, self.conf.max_bucket_listing) encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) query = { 'format': 'json', 'limit': max_keys + 1, } if 'prefix' in req.params: query.update({'prefix': req.params['prefix']}) if 'delimiter' in req.params: query.update({'delimiter': req.params['delimiter']}) fetch_owner = False if 'versions' in req.params: listing_type = 'object-versions' if 'key-marker' in req.params: query.update({'marker': req.params['key-marker']}) elif 'version-id-marker' in req.params: err_msg = ('A version-id marker cannot be specified without ' 'a key marker.') raise InvalidArgument('version-id-marker', req.params['version-id-marker'], err_msg) elif int(req.params.get('list-type', '1')) == 2: listing_type = 'version-2' if 'start-after' in req.params: query.update({'marker': req.params['start-after']}) # continuation-token overrides start-after if 'continuation-token' in req.params: decoded = b64decode(req.params['continuation-token']) query.update({'marker': decoded}) if 'fetch-owner' in req.params: fetch_owner = config_true_value(req.params['fetch-owner']) else: listing_type = 'version-1' if 'marker' in req.params: query.update({'marker': req.params['marker']}) resp = req.get_response(self.app, query=query) objects = json.loads(resp.body) # in order to judge that truncated is valid, check whether # max_keys + 1 th element exists in swift. is_truncated = max_keys > 0 and len(objects) > max_keys objects = objects[:max_keys] if listing_type == 'object-versions': elem = Element('ListVersionsResult') SubElement(elem, 'Name').text = req.container_name SubElement(elem, 'Prefix').text = req.params.get('prefix') SubElement(elem, 'KeyMarker').text = req.params.get('key-marker') SubElement( elem, 'VersionIdMarker').text = req.params.get('version-id-marker') if is_truncated: if 'name' in objects[-1]: SubElement(elem, 'NextKeyMarker').text = \ objects[-1]['name'] if 'subdir' in objects[-1]: SubElement(elem, 'NextKeyMarker').text = \ objects[-1]['subdir'] SubElement(elem, 'NextVersionIdMarker').text = 'null' else: elem = Element('ListBucketResult') SubElement(elem, 'Name').text = req.container_name SubElement(elem, 'Prefix').text = req.params.get('prefix') if listing_type == 'version-1': SubElement(elem, 'Marker').text = req.params.get('marker') if is_truncated and 'delimiter' in req.params: if 'name' in objects[-1]: name = objects[-1]['name'] else: name = objects[-1]['subdir'] if encoding_type == 'url': name = quote(name) SubElement(elem, 'NextMarker').text = name elif listing_type == 'version-2': if is_truncated: if 'name' in objects[-1]: SubElement(elem, 'NextContinuationToken').text = \ b64encode(objects[-1]['name'].encode('utf8')) if 'subdir' in objects[-1]: SubElement(elem, 'NextContinuationToken').text = \ b64encode(objects[-1]['subdir'].encode('utf8')) if 'continuation-token' in req.params: SubElement(elem, 'ContinuationToken').text = \ req.params['continuation-token'] if 'start-after' in req.params: SubElement(elem, 'StartAfter').text = \ req.params['start-after'] SubElement(elem, 'KeyCount').text = str(len(objects)) SubElement(elem, 'MaxKeys').text = str(tag_max_keys) if 'delimiter' in req.params: SubElement(elem, 'Delimiter').text = req.params['delimiter'] if encoding_type == 'url': SubElement(elem, 'EncodingType').text = encoding_type SubElement(elem, 'IsTruncated').text = \ 'true' if is_truncated else 'false' for o in objects: if 'subdir' not in o: name = o['name'] if encoding_type == 'url': name = quote(name.encode('utf-8')) if listing_type == 'object-versions': contents = SubElement(elem, 'Version') SubElement(contents, 'Key').text = name SubElement(contents, 'VersionId').text = 'null' SubElement(contents, 'IsLatest').text = 'true' else: contents = SubElement(elem, 'Contents') SubElement(contents, 'Key').text = name SubElement(contents, 'LastModified').text = \ o['last_modified'][:-3] + 'Z' if 's3_etag' in o: # New-enough MUs are already in the right format etag = o['s3_etag'] else: etag = '"%s"' % o['hash'] SubElement(contents, 'ETag').text = etag SubElement(contents, 'Size').text = str(o['bytes']) if fetch_owner or listing_type != 'version-2': owner = SubElement(contents, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id SubElement(contents, 'StorageClass').text = 'STANDARD' for o in objects: if 'subdir' in o: common_prefixes = SubElement(elem, 'CommonPrefixes') name = o['subdir'] if encoding_type == 'url': name = quote(name.encode('utf-8')) SubElement(common_prefixes, 'Prefix').text = name body = tostring(elem) return HTTPOk(body=body, content_type='application/xml')
def generate_signed_url(self, expiration, method='GET', content_type=None, generation=None, response_disposition=None, response_type=None, client=None, credentials=None): """Generates a signed URL for this blob. .. note:: If you are on Google Compute Engine, you can't generate a signed URL. Follow `Issue 922`_ for updates on this. If you'd like to be able to generate a signed URL from GCE, you can use a standard service account from a JSON file rather than a GCE service account. .. _Issue 922: https://github.com/GoogleCloudPlatform/\ google-cloud-python/issues/922 If you have a blob that you want to allow access to for a set amount of time, you can use this method to generate a URL that is only valid within a certain time period. This is particularly useful if you don't want publicly accessible blobs, but don't want to require users to explicitly log in. :type expiration: int, long, datetime.datetime, datetime.timedelta :param expiration: When the signed URL should expire. :type method: str :param method: The HTTP verb that will be used when requesting the URL. :type content_type: str :param content_type: (Optional) The content type of the object referenced by ``resource``. :type generation: str :param generation: (Optional) A value that indicates which generation of the resource to fetch. :type response_disposition: str :param response_disposition: (Optional) Content disposition of responses to requests for the signed URL. For example, to enable the signed URL to initiate a file of ``blog.png``, use the value ``'attachment; filename=blob.png'``. :type response_type: str :param response_type: (Optional) Content type of responses to requests for the signed URL. Used to over-ride the content type of the underlying blob/object. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type credentials: :class:`oauth2client.client.OAuth2Credentials` or :class:`NoneType` :param credentials: (Optional) The OAuth2 credentials to use to sign the URL. Defaults to the credentials stored on the client used. :rtype: str :returns: A signed URL you can use to access the resource until expiration. """ resource = '/{bucket_name}/{quoted_name}'.format( bucket_name=self.bucket.name, quoted_name=quote(self.name, safe='')) if credentials is None: client = self._require_client(client) credentials = client._base_connection.credentials return generate_signed_url( credentials, resource=resource, api_access_endpoint=_API_ACCESS_ENDPOINT, expiration=expiration, method=method, content_type=content_type, response_type=response_type, response_disposition=response_disposition, generation=generation)
def arn_url_path(self): """Return an ARN quoted correctly for use in a URL.""" return '/' + urlparse.quote(self.arn())
def test_lists_for_email(self): email = "*****@*****.**" self.cl.stub_request("clients/%s/listsforemail.json?email=%s" % (self.cl.client_id, quote(email)), "listsforemail.json") lists = self.cl.lists_for_email(email) self.assertEquals(len(lists), 2) self.assertEquals(lists[0].ListID, 'ab4a2b57c7c8f1ba62f898a1af1a575b') self.assertEquals(lists[0].ListName, 'List Number One') self.assertEquals(lists[0].SubscriberState, 'Active') self.assertEquals(lists[0].DateSubscriberAdded, '2012-08-20 22:32:00')
def as_pagination_marker(self, user): return urllib_parse.quote(user.name)
def get_tool(self, tool_id): url = "%s/ga4gh/v1/tools/%s" % (self.base_url, quote(tool_id, safe='')) return self._requests.get(url)
def create_http_request(self, api_client): # verify the required parameter 'name' is set if self.name is None: raise ValueError( "Missing the required parameter `name` when calling `copy_style`" ) # noqa: E501 # verify the required parameter 'style_copy' is set if self.style_copy is None: raise ValueError( "Missing the required parameter `style_copy` when calling `copy_style`" ) # noqa: E501 path = '/v4.0/words/{name}/styles/copy' path_params = {} if self.name is not None: path_params['name'] = self.name # noqa: E501 else: path_params['name'] = '' # noqa: E501 # path parameters collection_formats = {} if path_params: path_params = api_client.sanitize_for_serialization(path_params) path_params = api_client.parameters_to_tuples( path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything path = path.replace( '{%s}' % k, quote(str(v), safe=api_client.configuration. safe_chars_for_path_param)) # remove optional path parameters path = path.replace('//', '/') query_params = [] if self.folder is not None: query_params.append(('folder', self.folder)) # noqa: E501 if self.storage is not None: query_params.append(('storage', self.storage)) # noqa: E501 if self.load_encoding is not None: query_params.append( ('loadEncoding', self.load_encoding)) # noqa: E501 if self.password is not None: query_params.append(('password', self.password)) # noqa: E501 if self.dest_file_name is not None: query_params.append( ('destFileName', self.dest_file_name)) # noqa: E501 if self.revision_author is not None: query_params.append( ('revisionAuthor', self.revision_author)) # noqa: E501 if self.revision_date_time is not None: query_params.append( ('revisionDateTime', self.revision_date_time)) # noqa: E501 header_params = {} # HTTP header `Content-Type` header_params[ 'Content-Type'] = api_client.select_header_content_type( # noqa: E501 ['application/xml', 'application/json']) # noqa: E501 form_params = [] body_params = None if self.style_copy is not None: body_params = self.style_copy return { "method": "POST", "path": path, "query_params": query_params, "header_params": header_params, "form_params": form_params, "body": body_params, "collection_formats": collection_formats, "response_type": 'StyleResponse' # noqa: E501 }
def as_pagination_marker(self, user): return urllib_parse.quote('%s@%s' % (user.name, user.host))
def __call_api(self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None): config = self.configuration # header parameters header_params = header_params or {} header_params.update(self.default_headers) if self.cookie: header_params['Cookie'] = self.cookie if header_params: header_params = self.sanitize_for_serialization(header_params) header_params = dict( self.parameters_to_tuples(header_params, collection_formats)) # path parameters if path_params: path_params = self.sanitize_for_serialization(path_params) path_params = self.parameters_to_tuples(path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything resource_path = resource_path.replace( '{%s}' % k, quote(str(v), safe=config.safe_chars_for_path_param)) # query parameters if query_params: query_params = self.sanitize_for_serialization(query_params) query_params = self.parameters_to_tuples(query_params, collection_formats) # post parameters if post_params or files: post_params = self.prepare_post_parameters(post_params, files) post_params = self.sanitize_for_serialization(post_params) post_params = self.parameters_to_tuples(post_params, collection_formats) # auth setting self.update_params_for_auth(header_params, query_params, auth_settings) # body if body: body = self.sanitize_for_serialization(body) # request url url = self.configuration.host + resource_path # perform request and return response response_data = self.request(method, url, query_params=query_params, headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, _request_timeout=_request_timeout) self.last_response = response_data return_data = response_data if _preload_content: # deserialize response data if response_type: return_data = self.deserialize(response_data, response_type) else: return_data = None if _return_http_data_only: return (return_data) else: return (return_data, response_data.status, response_data.getheaders())
def get_tool(self, tool_id): url = "{}/ga4gh/v1/tools/{}".format(self.base_url, quote(tool_id, safe='')) return self._requests.get(url)
def quote_unicode(value): """Quote a unicode value by encoding it to bytes first.""" if isinstance(value, text_type): return quote(value.encode(default_encoding, 'replace')) return quote(value)
def set_outlet_name(self, outlet=0, name="Unknown"): """ Set the name of an outlet """ self.determine_outlet(outlet) self.geturl(url='unitnames.cgi?outname%s=%s' % (outlet, quote(name))) return self.get_outlet_name(outlet) == name