def test_HALNavigator__getitem_gauntlet(): with httprettify(): index_uri = 'http://www.example.com/' index_regex = re.compile(index_uri + '.*') template_href = 'http://www.example.com/{?max,page}' index_links = {'first': { 'href': template_href, 'templated': True }} register_hal(index_regex, index_links) N = HN.HALNavigator(index_uri) expanded_nav = N['first', 'page':0, 'max':1] assert expanded_nav.uri == uritemplate.expand(template_href, {'max': 1, 'page': '0'}) assert N['first'].expand(page=0, max=1) == expanded_nav assert N['first']['page': 0].uri == uritemplate \ .expand(template_href, {'page': '0'}) assert N['first', :].uri == uritemplate.expand( template_href, variables={}) first_page_expanded = uritemplate.expand(template_href, {'page': '0'}) first_null_expanded = uritemplate.expand(template_href, {}) first_both_expanded = uritemplate.expand( template_href, {'page': '0', 'max': 4}) # (somewhat) exhaustive combinations N_first = N['first'] with pytest.raises(TypeError): assert N['page': 0] assert N_first['page':0].uri == first_page_expanded assert N[...].uri == N.uri with pytest.raises(TypeError): assert N['page': 0, ...] assert N_first['page':0, ...].uri == first_page_expanded assert N_first['page':0, ...].templated with pytest.raises(TypeError): assert N[:] assert N_first[:].uri == first_null_expanded with pytest.raises(TypeError): assert N['page':0, :] assert N_first['page':0, :].uri == first_page_expanded assert not N_first['page':0, :].templated with pytest.raises(SyntaxError): assert N[:, ...] with pytest.raises(SyntaxError): assert N['page':0, :, ...] assert N['first'].template_uri == template_href assert N['first', 'page': 0].uri == first_page_expanded assert N['first', ...].template_uri == template_href assert N['first', 'page':0, ...].template_uri == template_href assert N['first', 'page':0, ...].templated assert N['first', 'page':0, ...]['max': 4].uri == first_both_expanded assert N['first', :].uri == first_null_expanded assert not N['first', :].templated assert N['first', 'page':0, :].uri == first_page_expanded assert not N['first', 'page':0, :].templated with pytest.raises(SyntaxError): assert N['first', :, ...] with pytest.raises(SyntaxError): assert N['first', 'page': 0, :, ...]
def get_touched_branch_files(payload, github_auth): ''' Return a set of files modified between master and payload head. ''' branch_sha = payload['head_commit']['id'] compare1_url = payload['repository']['compare_url'] compare1_url = expand(compare1_url, dict(base='master', head=branch_sha)) current_app.logger.debug('Compare URL 1 {}'.format(compare1_url)) compare1 = get(compare1_url, auth=github_auth).json() merge_base_sha = compare1['merge_base_commit']['sha'] # That's no branch. if merge_base_sha == branch_sha: return set() compare2_url = payload['repository']['compare_url'] compare2_url = expand(compare2_url, dict(base=merge_base_sha, head=branch_sha)) current_app.logger.debug('Compare URL 2 {}'.format(compare2_url)) compare2 = get(compare2_url, auth=github_auth).json() touched = set([file['filename'] for file in compare2['files']]) current_app.logger.debug('Touched files {}'.format(', '.join(touched))) return touched
def __init__(self, bb, mode='fetch', **kwargs): self.bb = bb if mode: self.url = [uritemplate.expand(x, **kwargs) for x in self.uri] self.data = {} if mode == 'fetch': for arg in kwargs: setattr(self, arg, kwargs[arg]) self.data = {} for url in self.url: self.data.update(self.get(url)) elif mode == 'list': self.instances = [] for instance in self.get(self.url[0]): kw = kwargs.copy() kw.update(instance) instance = type(self)(self.bb, mode=None, **kw) instance.url = [uritemplate.expand(x, **kw) for x in instance.uri] self.instances.append(instance) else: self.data = kwargs for datum in self.data: if datum == 'data': setattr(self, '_' + datum, self.data[datum]) else: setattr(self, datum, self.data[datum])
def get_route_from_fk(resource_type, pk=None): """Gets a fully qualified URL for a given resource_type, pk""" routes = requests.get(zc_settings.GATEWAY_ROOT_PATH).json() for route in routes.iterkeys(): if 'resource_type' in routes[route] and routes[route]['resource_type'] == resource_type: if isinstance(pk, (list, set)): expanded = '{}?filter[id__in]={}'.format(expand(route, {}), ','.join([str(x) for x in pk])) else: expanded = expand(route, {'id': pk}) return '{0}{1}'.format(routes[route]['domain'], expanded) raise RouteNotFoundException('No route for resource_type: "{0}"'.format(resource_type))
def get_odes_extracts(db, api_key): ''' ''' odeses, extracts = list(), list() vars = dict(api_key=api_key) extracts_url = uritemplate.expand(odes_extracts_url, vars) resp = requests.get(extracts_url) if resp.status_code in range(200, 299): odeses.extend([ data.ODES(str(oj['id']), status=oj['status'], bbox=oj['bbox'], links=oj.get('download_links', {}), processed_at=(parse_datetime(oj['processed_at']) if oj['processed_at'] else None), created_at=(parse_datetime(oj['created_at']) if oj['created_at'] else None)) for oj in resp.json() ]) for odes in sorted(odeses, key=attrgetter('created_at'), reverse=True): extract = data.get_extract(db, odes=odes) if extract is None: extract = data.Extract(None, None, None, odes, None, None, None) extracts.append(extract) return extracts
def create(cls, payload, client=None): """Create a new consumer. :param payload: the options for creating the new consumer :type payload: ConsumerPayload :param client: the configured connection to Bitbucket. If not provided, assumes an Anonymous connection. :type client: bitbucket.Client :returns: the new consumer object. :rtype: Consumer :raises: ValueError """ client = client or Client() owner = client.get_username() if not owner: raise ValueError('owner is required') data = payload.validate().build() templates = cls.extract_templates_from_json() api_url = expand(templates['create'], { 'bitbucket_url': client.get_bitbucket_url(), 'username': owner }) # Note: This Bitbucket API expects a urlencoded-form, not json. # Hence, use `data` instead of `json`. return cls.post(api_url, data=data, client=client)
def build(serviceName, version, http=None, discoveryServiceUrl=DISCOVERY_URI, developerKey=None, model=JsonModel(), requestBuilder=HttpRequest): params = { 'api': serviceName, 'apiVersion': version } if http is None: http = httplib2.Http() requested_url = uritemplate.expand(discoveryServiceUrl, params) logging.info('URL being requested: %s' % requested_url) resp, content = http.request(requested_url) service = simplejson.loads(content) fn = os.path.join(os.path.dirname(__file__), "contrib", serviceName, "future.json") try: f = file(fn, "r") d = simplejson.load(f) f.close() future = d['resources'] auth_discovery = d['auth'] except IOError: future = {} auth_discovery = {} base = urlparse.urljoin(discoveryServiceUrl, service['restBasePath']) resources = service['resources'] class Service(object): """Top level interface for a service""" def __init__(self, http=http): self._http = http self._baseUrl = base self._model = model self._developerKey = developerKey self._requestBuilder = requestBuilder def auth_discovery(self): return auth_discovery def createMethod(theclass, methodName, methodDesc, futureDesc): def method(self): return createResource(self._http, self._baseUrl, self._model, self._requestBuilder, methodName, self._developerKey, methodDesc, futureDesc) setattr(method, '__doc__', 'A description of how to use this function') setattr(method, '__is_resource__', True) setattr(theclass, methodName, method) for methodName, methodDesc in resources.iteritems(): createMethod(Service, methodName, methodDesc, future.get(methodName, {})) return Service()
def format_href(self, **kw): """ Default implementation for href, which expands the uri_template with the provided arguments """ return expand(self.uri_template, **kw)
async def asset_uploader( gh: AioGitHubApi, queue: "asyncio.Queue[Optional[AssetUploadWork]]", ) -> bool: success = True while True: work = await queue.get() if not work: queue.task_done() return success logger.info("uploading %s", work.name) try: with open(work.path, "rb") as f: await gh.upload( uritemplate.expand(work.upload_url, name=work.name), f, work.content_type, ) except Exception: logger.exception("uploading %s failed", work.name) success = False else: logger.info("uploaded %s", work.name) finally: queue.task_done()
def widget(): """ Finds issues based on the given label. Render them in the widget """ # Get optional parameters labels = request.args.get("labels", None) org_name = request.args.get("organization_name") org_type = request.args.get("org_type") number = request.args.get("number") tracking_status = request.args.get("tracking") # Build the url if org_name and labels: issues_path_template = "organizations{/org_name}/issues/labels{/labels}{?query*}" elif org_name: issues_path_template = "organizations{/org_name}/issues{?query*}" elif labels: issues_path_template = "issues/labels{/labels}{?query*}" else: issues_path_template = "issues{?query*}" issues_url_template = urljoin(CFAPI_BASE, issues_path_template) issues_url_kwargs = ("organization_type", org_type), ("per_page", number) url_args = dict(org_name=org_name, labels=labels, query={k: v for (k, v) in issues_url_kwargs if v}) issues_url = expand(issues_url_template, url_args) # Get the actual issues from the API try: issues_response = get(issues_url, timeout=5) except Timeout: return render_template("widget.html", error=True) except ConnectionError, e: return render_template("widget.html", error=True)
def main(): '''Main logic''' import argparse parser = argparse.ArgumentParser('Create GitHub relase') parser.add_argument('--token', '-t', required=True, help='GitHub authentication token') parser.add_argument('--user', '-u', required=True, help='GitHub user account') parser.add_argument('--repo', '-r', required=True, help='GitHub repo name') parser.add_argument('--version', '-v', required=True, help='Version to create') parser.add_argument('--file', '-f', help='File to upload to release') parser.add_argument('--content-type', '-c', help='Content type of file') args = parser.parse_args() api_url = 'https://api.github.com/repos/{user}/{repo}/releases' api_url = api_url.format(user=args.user, repo=args.repo) # Create release release_json = create_release(api_url, args.token, args.version) # Upload file if args.file: upload_url = expand(release_json['upload_url'], {'name': os.path.basename(args.file)}) upload_file(upload_url, args.token, args.file, args.content_type)
def get_odes_extract(db, id, api_key): ''' ''' extract, odes = data.get_extract(db, extract_id=id), None if extract is None: # Nothing by that name in the database, so ask the ODES API. vars = dict(id=id, api_key=api_key) extract_url = uritemplate.expand(odes_extracts_url, vars) resp = requests.get(extract_url) if resp.status_code in range(200, 299): oj = resp.json() odes = data.ODES(str(oj['id']), status=oj['status'], bbox=oj['bbox'], links=oj.get('download_links', {}), processed_at=(parse_datetime(oj['processed_at']) if oj['processed_at'] else None), created_at=(parse_datetime(oj['created_at']) if oj['created_at'] else None)) if odes is None: # Nothing at all for this ID anywhere. return None if odes is None: # A DB extract was found, but nothing in ODES - very weird! return get_odes_extract(db, extract.odes.id, api_key) # We have a known ODES, so look for it in the database. extract = data.get_extract(db, odes=odes) if extract is None: # Known ODES, but nothing in the DB so make one up. return data.Extract(None, None, None, odes, None, None, None) return extract
def request_odes_extract(extract, request, url_for, api_key): ''' ''' env = Environment(loader=PackageLoader(__name__, 'templates')) args = dict( name = extract.name or extract.wof.name or 'an unnamed place', link = urljoin(util.get_base_url(request), url_for('ODES.get_extract', extract_id=extract.id)), extracts_link = urljoin(util.get_base_url(request), url_for('ODES.get_extracts')), created = extract.created ) email = dict( email_subject=env.get_template('email-subject.txt').render(**args), email_body_text=env.get_template('email-body.txt').render(**args), email_body_html=env.get_template('email-body.html').render(**args) ) params = {key: extract.envelope.bbox[i] for (i, key) in enumerate(('bbox_w', 'bbox_s', 'bbox_e', 'bbox_n'))} params.update(email) post_url = uritemplate.expand(odes_extracts_url, dict(api_key=api_key)) resp = requests.post(post_url, data=params) oj = resp.json() if 'error' in oj: raise util.KnownUnknown("Error: {}".format(oj['error'])) elif resp.status_code != 200: raise Exception("Bad ODES status code: {}".format(resp.status_code)) return data.ODES(str(oj['id']), status=oj['status'], bbox=oj['bbox'], links=oj.get('download_links', {}), processed_at=(parse_datetime(oj['processed_at']) if oj['processed_at'] else None), created_at=(parse_datetime(oj['created_at']) if oj['created_at'] else None))
def setup_class(cls): template = (Bitbucket(client=cls.test_client).data.get( '_links', {}).get('snippetsForRole', {}).get('href')) cls.url = expand(template, { 'bitbucket_url': cls.test_client.get_bitbucket_url(), 'role': cls.role })
def _get_related_url(self, property_name, element_type, **kwargs): if property_name in self: template = self[property_name] url = uritemplate.expand(template, kwargs) return self._client.get_list_absolute_url( url, element_type, fetch_params=self._get_related_fetch_params()) else: template = self._default_urls[property_name].format( **self._fetch_params, **self) url = uritemplate.expand(template, kwargs) return self._client.get_list_relative_url( url, element_type, fetch_params=self._get_related_fetch_params())
def expand_uri(self, name, **kwargs): link = self.schema.get_link(name) href = link.get('href', '') context = dict(self.data, **kwargs) return uritemplate.expand(href, context)
def renewnodevice(self, end): link = self.get_link(self.RENEW) pend = datetime.utcfromtimestamp(end).isoformat() + 'Z' if link['type'] == License.STATUS_MIMETYPE and link[ 'templated'] == True: regurl = expand(link['href'], {'end': pend}) self.status = json.loads(self._put(regurl))
def _get_build_content( serviceName, version, discoveryServiceUrl=DISCOVERY_URI, cache_discovery=True, cache=None, proxy_info=None, ): params = {"api": serviceName, "apiVersion": version} discovery_http = httplib2.Http(timeout=60) if proxy_info: discovery_http.proxy_info = httplib2.ProxyInfo( httplib2.socks.PROXY_TYPE_HTTP_NO_TUNNEL, proxy_info["host"], proxy_info["port"], proxy_user=proxy_info["user"], proxy_pass=proxy_info["password"], ) discovery_http.disable_ssl_certificate_validation = True for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI): requested_url = uritemplate.expand(discovery_url, params) try: return _retrieve_discovery_doc(requested_url, discovery_http, cache_discovery, cache) except HttpError as e: if e.resp.status == 404: continue else: raise e raise Exception("name: %s version: %s" % (serviceName, version))
def generate_iri(self, values, default_representation=HYDRA.BasicRepresentation): """ Generate an IRI according to this template. This method takes care of formatting the provided values according to the template's variableRepresentation. :param values: a dict whose keys are properties, and values are RDF terms :return: the IRI as a string NB: if required properties are missing from `values`, a ValueError is raised. """ # ensure that keys are URIRefs in values graph = self.graph values = {URIRef(key): value for key, value in values.items()} representation = self.get_variable_representation( default=default_representation) mode = 0 if representation is HYDRA.BasicRepresentation else 1 data, msg = self._map_properties(values) if data is None: raise ValueError(msg) data = { key: _format_variable(value, mode) for key, value in data.items() } return expand(self.template, data)
def format_data_href(self, **kw): """ Default implementation for data-href, same as format_href, with uri_template_json instead. """ return expand(self.uri_template_json, **kw)
def __get_discovery_doc(self, service_name, version, http=None, discovery_service_url=DISCOVERY_URI): # Returned a cached copy if we have it. cached = memcache.get("discovery_doc") if cached: return cached logging.info("Cache miss in discovery document.") params = {'api': service_name, 'apiVersion': version} requested_url = uritemplate.expand(discovery_service_url, params) # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment # variable that contains the network address of the client sending the # request. If it exists then add that to the request for the discovery # document to avoid exceeding the quota on discovery requests. if 'REMOTE_ADDR' in os.environ: requested_url = _add_query_parameter(requested_url, 'userIp', os.environ['REMOTE_ADDR']) http = http or httplib2.Http() resp, content = http.request(requested_url) if resp.status >= 400: raise HttpError(resp, content, uri=requested_url) # Store it in the memcache. memcache.set("discovery_doc", content, time=60 * 60 * 24) return content
def get_auth(self): self.token_uri = expand('{+server_base_uri}/site/oauth2/access_token', {'server_base_uri': self.server_base_uri}) token_response = """ { "access_token":"2YotnFZFEjr1zCsicMWpAA", "token_type":"bearer", "expires_in":3600, "refresh_token":"tGzv3JOkF0XG5Qx2TlKWIA", "example_parameter":"example_value", "scope":"repository" } """ httpretty.HTTPretty.allow_net_connect = False httpretty.register_uri(httpretty.POST, self.token_uri, content_type='application/json', body=token_response, status=200) a = OAuth2Authenticator(self.client_id, self.client_secret, self.email, self.grant, redirect_uris=self.redirect_uris, server_base_uri=self.server_base_uri, client_name=self.client_name, client_description=self.client_description) return a
def create_repository(username, repository_name, fork_policy, is_private, scm=None, name=None, description=None, language=None, has_issues=None, has_wiki=None, client=Client()): template = ('{+bitbucket_url}' + '/2.0/repositories/{username}/{repository_name}') url = expand( template, { 'bitbucket_url': client.get_bitbucket_url(), 'username': username, 'repository_name': repository_name }) payload = Repository.make_new_repository_payload( fork_policy, is_private, scm, name, description, language, has_issues, has_wiki) response = client.session.post(url, data=payload) Client.expect_ok(response) return client.convert_to_object(response.json())
def RelativeName(self, url_escape=False): """Relative resource name. A URI path ([path-noscheme](http://tools.ietf.org/html/rfc3986#appendix-A)) without the leading "/". It identifies a resource within the API service. For example: "shelves/shelf1/books/book2" Args: url_escape: bool, if true would url escape each parameter. Returns: Unescaped part of SelfLink which is essentially base_url + relative_name. For example if SelfLink is https://pubsub.googleapis.com/v1/projects/myprj/topics/mytopic then relative name is projects/myprj/topics/mytopic. """ escape_func = urllib.parse.quote if url_escape else lambda x, safe: x effective_params = dict( [(k, escape_func(getattr(self, k), safe='')) for k in self._params]) return urllib.parse.unquote( uritemplate.expand(self._path, effective_params))
def setup_class(cls): cls.url = expand( Hook.templates['create'], { 'bitbucket_url': cls.test_client.get_bitbucket_url(), 'owner': cls.test_client.get_username(), 'repository_name': cls.repository_name, })
def get_gcs_client(self) -> luigi.contrib.gcs.GCSClient: if (not os.path.isfile(self.discover_cache_local_path)): with open(self.discover_cache_local_path, "w") as f: try: fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) params = {"api": "storage", "apiVersion": "v1"} discovery_http = build_http() for discovery_url in (self._DISCOVERY_URI, self._V2_DISCOVERY_URI): requested_url = uritemplate.expand( discovery_url, params) try: content = _retrieve_discovery_doc( requested_url, discovery_http, False) except HttpError as e: if e.resp.status == http_client.NOT_FOUND: continue else: raise e break f.write(content) fcntl.flock(f, fcntl.LOCK_UN) except IOError: # try to read pass with open(self.discover_cache_local_path, "r") as f: fcntl.flock(f, fcntl.LOCK_SH) descriptor = f.read() fcntl.flock(f, fcntl.LOCK_UN) return luigi.contrib.gcs.GCSClient( oauth_credentials=self._load_oauth_credentials(), descriptor=descriptor)
def document_api(name, version, uri): """Document the given API. Args: name: string, Name of the API. version: string, Version of the API. uri: string, URI of the API's discovery document """ try: service = build(name, version) except UnknownApiNameOrVersion as e: print("Warning: {} {} found but could not be built.".format( name, version)) return except HttpError as e: print("Warning: {} {} returned {}.".format(name, version, e)) return http = build_http() response, content = http.request( uri or uritemplate.expand(FLAGS.discovery_uri_template, { "api": name, "apiVersion": version })) discovery = json.loads(content) version = safe_version(version) document_collection_recursive(service, "{}_{}.".format(name, version), discovery, discovery)
def retrieve_discovery_doc(service_name, version, http=None, discovery_service_url=DISCOVERY_URI): """Retrieves the discovery doc.""" params = {'api': service_name, 'apiVersion': version} requested_url = uritemplate.expand(discovery_service_url, params) # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment # variable that contains the network address of the client sending the # request. If it exists then add that to the request for the discovery # document to avoid exceeding the quota on discovery requests. if 'REMOTE_ADDR' in os.environ: requested_url = _add_query_parameter(requested_url, 'userIp', os.environ['REMOTE_ADDR']) http = http or httplib2.Http() resp, content = http.request(requested_url) if resp.status >= 400: raise HttpError(resp, content, uri=requested_url) try: json.loads(content) except ValueError: raise InvalidJsonError( 'Bad JSON: %s from %s.' % (content, requested_url)) # we return content instead of the JSON deserialized service because # build_from_document() consumes a string rather than a dictionary return content
def setup_class(cls): cls.url = expand( BranchRestriction.templates['create'], { 'bitbucket_url': cls.test_client.get_bitbucket_url(), 'owner': cls.owner, 'repository_name': cls.repository_name })
def delete(self): if not hasattr(self, 'url'): self.url = [ uritemplate.expand(x, user=self.owner['username']) for x in self.uri ] self.delete_(self.url[0] + '/%d' % self.pk)
def find_consumers(client=Client()): url = expand( Consumer.get_link_template('consumers'), { 'bitbucket_url': client.get_bitbucket_url(), 'username': client.get_username() }) return client.remote_relationship(url)
def __init__(self, collection_info, subcollection, param_values, endpoint_url): """Create a Resource object that may be partially resolved. To allow resolving of unknown params to happen after parse-time, the param resolution code is in this class rather than the _ResourceParser class. Args: collection_info: resource_util.CollectionInfo, The collection description for this resource. subcollection: str, id for subcollection of this collection. param_values: {param->value}, A list of values for parameters. endpoint_url: str, override service endpoint url for this resource. If None default base url of collection api will be used. Raises: RequiredFieldOmittedException: if param_values have None value. """ self._collection_info = collection_info self._endpoint_url = endpoint_url or collection_info.base_url self._subcollection = subcollection self._path = collection_info.GetPath(subcollection) self._params = collection_info.GetParams(subcollection) for param, value in param_values.iteritems(): if value is None: raise RequiredFieldOmittedException(collection_info.full_name, param) setattr(self, param, value) self._self_link = '{0}{1}'.format( self._endpoint_url, uritemplate.expand(self._path, self.AsDict())) if (self._collection_info.api_name in ('compute', 'clouduseraccounts', 'storage')): # TODO(b/15425944): Unquote URLs for now for these apis. self._self_link = urllib.unquote(self._self_link) self._initialized = True
def test_pagination(self): self.client.auto_paginate = True url = uritemplate.expand(self.client.url, {'param': 'foo'}) h1 = { 'Link': '<'+url+'?page=2&per_page=100>; rel="next"', 'X-RateLimit-Remaining': '56', 'X-RateLimit-Reset': '1446804464', 'X-RateLimit-Limit': '60' } h2 = { 'Link': '<'+url+'?page=3&per_page=100>; rel="next"', 'X-RateLimit-Remaining': '56', 'X-RateLimit-Reset': '1446804464', 'X-RateLimit-Limit': '60' } h3 = { 'X-RateLimit-Remaining': '56', 'X-RateLimit-Reset': '1446804464', 'X-RateLimit-Limit': '60' } res1 = '["a","b"]' res2 = '["c","d"]' res3 = '["e","f"]' self.adapter.register_uri('GET', url, headers=h1, text=res1) self.adapter.register_uri('GET', url+'?page=2', headers=h2, text=res2) self.adapter.register_uri('GET', url+'?page=3', headers=h3, text=res3) response = self.client.paginate(param='foo') resultSchema = [r.schema for r in response.schema] expectedSchema = ['a', 'b', 'c', 'd', 'e', 'f'] self.assertEqual(resultSchema, expectedSchema)
def build(serviceName, version, http=None, discoveryServiceUrl=DISCOVERY_URI, developerKey=None, model=None, requestBuilder=HttpRequest): """Construct a Resource for interacting with an API. Construct a Resource object for interacting with an API. The serviceName and version are the names from the Discovery service. Args: serviceName: string, name of the service version: string, the version of the service discoveryServiceUrl: string, a URI Template that points to the location of the discovery service. It should have two parameters {api} and {apiVersion} that when filled in produce an absolute URI to the discovery document for that service. developerKey: string, key obtained from https://code.google.com/apis/console model: apiclient.Model, converts to and from the wire format requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP request Returns: A Resource object with methods for interacting with the service. """ params = {'api': serviceName, 'apiVersion': version} if http is None: http = httplib2.Http() requested_url = uritemplate.expand(discoveryServiceUrl, params) # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment # variable that contains the network address of the client sending the # request. If it exists then add that to the request for the discovery # document to avoid exceeding the quota on discovery requests. if 'REMOTE_ADDR' in os.environ: requested_url = _add_query_parameter(requested_url, 'userIp', os.environ['REMOTE_ADDR']) logging.info('URL being requested: %s' % requested_url) resp, content = http.request(requested_url) if resp.status == 404: raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName, version)) if resp.status >= 400: raise HttpError(resp, content, requested_url) try: service = simplejson.loads(content) except ValueError, e: logging.error('Failed to parse as JSON: ' + content) raise InvalidJsonError()
def document_api(name, version): """Document the given API. Args: name: string, Name of the API. version: string, Version of the API. """ try: service = build(name, version) except UnknownApiNameOrVersion as e: print 'Warning: {} {} found but could not be built.'.format( name, version) return response, content = http.request( uritemplate.expand(FLAGS.discovery_uri_template, { 'api': name, 'apiVersion': version })) discovery = json.loads(content) version = safe_version(version) document_collection_recursive(service, '%s_%s.' % (name, version), discovery, discovery)
def create_buildstatus(owner, repository_name, revision, key, state, url, name=None, description=None, client=Client()): template = ('{+bitbucket_url}' + '/2.0/repositories{/owner,repository_name}' + '/commit{/revision}/statuses/build') # owner, repository_name, and revision are required api_url = expand( template, { 'bitbucket_url': client.get_bitbucket_url(), 'owner': owner, 'repository_name': repository_name, 'revision': revision }) payload = BuildStatus.make_payload(key=key, state=state, url=url, name=name, description=description) return BuildStatus.post(api_url, json=payload, client=client)
def _refresh(self, http_request): """Refreshes the access_token. Skip all the storage hoops and just refresh using the API. Args: http_request: callable, a callable that matches the method signature of httplib2.Http.request, used to make the refresh request. Raises: AccessTokenRefreshError: When the refresh fails. """ uri = uritemplate.expand(META, {'scope': self.scope}) response, content = http_request(uri) if response.status == 200: try: d = simplejson.loads(content) except Exception as e: raise AccessTokenRefreshError(str(e)) self.access_token = d['accessToken'] else: if response.status == 404: content = content + (' This can occur if a VM was created' ' with no service account or scopes.') raise AccessTokenRefreshError(content)
def retrieve_discovery_doc(serviceName, version, discoveryServiceUrl=DISCOVERY_URI): params = {'api': serviceName, 'apiVersion': version} requested_url = uritemplate.expand(discoveryServiceUrl, params) # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment # variable that contains the network address of the client sending the # request. If it exists then add that to the request for the discovery # document to avoid exceeding the quota on discovery requests. if 'REMOTE_ADDR' in os.environ: requested_url = _add_query_parameter(requested_url, 'userIp', os.environ['REMOTE_ADDR']) http = httplib2.Http() resp, content = http.request(requested_url) if resp.status >= 400: raise HttpError(resp, content, uri=requested_url) try: service = json.loads(content) except ValueError: raise InvalidJsonError( 'Bad JSON: %s from %s.' % (content, requested_url)) # We return content instead of the JSON deserialized service because # build_from_document() consumes a string rather than a dictionary. return content
def expand(self, _keep_templated=False, **kwargs): '''Expand template args in a templated Navigator. if :_keep_templated: is True, the resulting Navigator can be further expanded. A Navigator created this way is not part of the id map. ''' if not self.templated: raise TypeError( "This Navigator isn't templated! You can't expand it.") for k, v in kwargs.iteritems(): if v == 0: kwargs[k] = '0' # uritemplate expands 0's to empty string if self.template_args is not None: kwargs.update(self.template_args) cp = self._copy( uri=uritemplate.expand(self.template_uri, kwargs), templated=_keep_templated, ) if not _keep_templated: cp.template_uri = None cp.template_args = None else: cp.template_args = kwargs return cp
def load_toots(mastodon_whoami_url, mastodon_statuses_url, mastodon_header, max_count=50): ''' Load recent toots from a Mastodon instance and return list of Toot objects. These are assumed to be in reverse-chronological order. ''' got1 = requests.get(mastodon_whoami_url, headers=mastodon_header) mastodon_id = got1.json().get('id') mastodon_toots = list() url = uritemplate.expand(mastodon_statuses_url, dict(id=mastodon_id)) while len(mastodon_toots) < max_count: print('Get', url, '...', file=sys.stderr) got2 = requests.get(url, headers=mastodon_header) for status in got2.json(): url = status['url'] soup = bs4.BeautifulSoup(status['content'], 'html.parser') links = [a['href'] for a in soup.find_all('a')] mastodon_toots.append(Toot(url, links)) print(mastodon_toots[-1], file=sys.stderr) if 'next' in got2.links: url = got2.links['next']['url'] else: break return mastodon_toots
def _delete(self, **kwargs): # TODO: Refactor to resolve circular dependency. from pluct import resource self.href = expand(self.href, self.resource.data) response = requests.delete(url=self.href, headers=self.get_headers()) return resource.from_response(response)
def get_odes_extracts(db, api_key): ''' ''' odeses, extracts = list(), list() vars = dict(api_key=api_key) extracts_url = uritemplate.expand(odes_extracts_url, vars) resp = requests.get(extracts_url) if resp.status_code in range(200, 299): odeses.extend([data.ODES(str(oj['id']), status=oj['status'], bbox=oj['bbox'], links=oj.get('download_links', {}), processed_at=(parse_datetime(oj['processed_at']) if oj['processed_at'] else None), created_at=(parse_datetime(oj['created_at']) if oj['created_at'] else None)) for oj in resp.json()]) for odes in sorted(odeses, key=attrgetter('created_at'), reverse=True): extract = data.get_extract(db, odes=odes) if extract is None: extract = data.Extract(None, None, None, odes, None, None, None) extracts.append(extract) return extracts
def document_api(name, version): """Document the given API. Args: name: string, Name of the API. version: string, Version of the API. """ try: service = build(name, version) except UnknownApiNameOrVersion as e: print 'Warning: {} {} found but could not be built.'.format(name, version) return response, content = http.request( uritemplate.expand( FLAGS.discovery_uri_template, { 'api': name, 'apiVersion': version}) ) discovery = json.loads(content) version = safe_version(version) document_collection_recursive( service, '%s_%s.' % (name, version), discovery, discovery)
def build(service, version, http=httplib2.Http(), discoveryServiceUrl = DISCOVERY_URI, auth = None, model = JsonModel()): params = { 'api': service, 'apiVersion': version } resp, content = http.request(uritemplate.expand(discoveryServiceUrl, params)) d = simplejson.loads(content) service = d['data'][service][version] base = service['baseUrl'] resources = service['resources'] class Service(object): """Top level interface for a service""" def __init__(self, http=http): self._http = http self._baseUrl = base self._model = model def createMethod(theclass, methodName, methodDesc): def method(self, **kwargs): return createResource(self._http, self._baseUrl, self._model, methodName, methodDesc) setattr(method, '__doc__', 'A description of how to use this function') setattr(theclass, methodName, method) for methodName, methodDesc in resources.iteritems(): createMethod(Service, methodName, methodDesc) return Service()
def build( serviceName, version, http=None, discoveryServiceUrl=DISCOVERY_URI, developerKey=None, model=None, requestBuilder=HttpRequest, ): """Construct a Resource for interacting with an API. Construct a Resource object for interacting with an API. The serviceName and version are the names from the Discovery service. Args: serviceName: string, name of the service version: string, the version of the service discoveryServiceUrl: string, a URI Template that points to the location of the discovery service. It should have two parameters {api} and {apiVersion} that when filled in produce an absolute URI to the discovery document for that service. developerKey: string, key obtained from https://code.google.com/apis/console model: apiclient.Model, converts to and from the wire format requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP request Returns: A Resource object with methods for interacting with the service. """ params = {"api": serviceName, "apiVersion": version} if http is None: http = httplib2.Http() requested_url = uritemplate.expand(discoveryServiceUrl, params) # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment # variable that contains the network address of the client sending the # request. If it exists then add that to the request for the discovery # document to avoid exceeding the quota on discovery requests. if "REMOTE_ADDR" in os.environ: requested_url = _add_query_parameter(requested_url, "userIp", os.environ["REMOTE_ADDR"]) logging.info("URL being requested: %s" % requested_url) resp, content = http.request(requested_url) if resp.status == 404: raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName, version)) if resp.status >= 400: raise HttpError(resp, content, requested_url) try: service = simplejson.loads(content) except ValueError, e: logging.error("Failed to parse as JSON: " + content) raise InvalidJsonError()
def expand_uri(template, args): ''' ''' new_args = {k: v for (k, v) in args.items() if not hasattr(v, 'encode')} new_args.update({k: v.encode('utf8') for (k, v) in args.items() if hasattr(v, 'encode')}) return uritemplate.expand(template, new_args)
def expand(self, _keep_templated=False, **kwargs): '''Expand template args in a templated Navigator. if :_keep_templated: is True, the resulting Navigator can be further expanded. A Navigator created this way is not part of the id map. ''' if not self.templated: raise TypeError( "This Navigator isn't templated! You can't expand it.") for k, v in kwargs.iteritems(): if v == 0: kwargs[k] = '0' # uritemplate expands 0's to empty string if self.template_args is not None: kwargs.update(self.template_args) cp = self._copy(uri=uritemplate.expand(self.template_uri, kwargs), templated=_keep_templated, ) if not _keep_templated: cp.template_uri = None cp.template_args = None else: cp.template_args = kwargs return cp
def _put(self, **kwargs): self.href = expand(self.href, self.resource.data) data = kwargs.pop('data') response = self._remote().put(url=self.href, data=ujson.dumps(data), headers=self.get_headers(), timeout=self.resource.timeout) return from_response(self.resource.__class__, response)
def _expand_path_params(url, path_params): """ Given a templated URL and some parameters that have been provided, expand the URL. """ if path_params: return uritemplate.expand(url, path_params) return url
def issue(self, repo, number, oldData=None): """Return github issue and event data""" url = uritemplate.expand(repo.data['issues_url'], { 'number': number }) issue = self.get(url) # TODO: only get events up to the most recent in oldData? issue['events'] = self.getAll(issue['events_url']) logging.info('Found %d events for #%s' % (len(issue['events']), number)) return issue
def _get_related_url(self, property_name, element_type, **kwargs): if property_name in self: template = self[property_name] url = uritemplate.expand(template, kwargs) return self._client.get_list_absolute_url( url, element_type, fetch_params=self._get_related_fetch_params() ) else: template = self._default_urls[property_name].format( **self._fetch_params, **self ) url = uritemplate.expand(template, kwargs) return self._client.get_list_relative_url( url, element_type, fetch_params=self._get_related_fetch_params() )
def make(self, name, params={}): try: route = self.routes[name] parts = urlparse(self.host) return uritemplate.expand("{proto}://{host}{+path}", dict(proto=parts.scheme, host=parts.netloc, path=route.template(params))) except KeyError: return None
def json_test_function(self): actual = unicode(expand(template, variables)) msg = '%(template)r did not expand as expected, got %(actual)r.'\ % locals() if type(expected) == type([]): self.assertTrue (actual in expected, msg) else: self.assertEqual(expected, actual)
def expand(self, link): if ':' in link: (name, rel) = link.split(':', 1) try: return expand(self.map[name], {'rel': rel}) except KeyError: pass return link