def test_HALNavigator__getitem_gauntlet():
with httprettify():
index_uri = 'http://www.example.com/'
index_regex = re.compile(index_uri + '.*')
template_href = 'http://www.example.com/{?max,page}'
index_links = {'first': {
'href': template_href,
'templated': True
}}
register_hal(index_regex, index_links)
N = HN.HALNavigator(index_uri)
expanded_nav = N['first', 'page':0, 'max':1]
assert expanded_nav.uri == uritemplate.expand(template_href,
{'max': 1, 'page': '0'})
assert N['first'].expand(page=0, max=1) == expanded_nav
assert N['first']['page': 0].uri == uritemplate \
.expand(template_href, {'page': '0'})
assert N['first', :].uri == uritemplate.expand(
template_href, variables={})
first_page_expanded = uritemplate.expand(template_href, {'page': '0'})
first_null_expanded = uritemplate.expand(template_href, {})
first_both_expanded = uritemplate.expand(
template_href, {'page': '0', 'max': 4})
# (somewhat) exhaustive combinations
N_first = N['first']
with pytest.raises(TypeError):
assert N['page': 0]
assert N_first['page':0].uri == first_page_expanded
assert N[...].uri == N.uri
with pytest.raises(TypeError):
assert N['page': 0, ...]
assert N_first['page':0, ...].uri == first_page_expanded
assert N_first['page':0, ...].templated
with pytest.raises(TypeError):
assert N[:]
assert N_first[:].uri == first_null_expanded
with pytest.raises(TypeError):
assert N['page':0, :]
assert N_first['page':0, :].uri == first_page_expanded
assert not N_first['page':0, :].templated
with pytest.raises(SyntaxError):
assert N[:, ...]
with pytest.raises(SyntaxError):
assert N['page':0, :, ...]
assert N['first'].template_uri == template_href
assert N['first', 'page': 0].uri == first_page_expanded
assert N['first', ...].template_uri == template_href
assert N['first', 'page':0, ...].template_uri == template_href
assert N['first', 'page':0, ...].templated
assert N['first', 'page':0, ...]['max': 4].uri == first_both_expanded
assert N['first', :].uri == first_null_expanded
assert not N['first', :].templated
assert N['first', 'page':0, :].uri == first_page_expanded
assert not N['first', 'page':0, :].templated
with pytest.raises(SyntaxError):
assert N['first', :, ...]
with pytest.raises(SyntaxError):
assert N['first', 'page': 0, :, ...]
def get_touched_branch_files(payload, github_auth):
''' Return a set of files modified between master and payload head.
'''
branch_sha = payload['head_commit']['id']
compare1_url = payload['repository']['compare_url']
compare1_url = expand(compare1_url, dict(base='master', head=branch_sha))
current_app.logger.debug('Compare URL 1 {}'.format(compare1_url))
compare1 = get(compare1_url, auth=github_auth).json()
merge_base_sha = compare1['merge_base_commit']['sha']
# That's no branch.
if merge_base_sha == branch_sha:
return set()
compare2_url = payload['repository']['compare_url']
compare2_url = expand(compare2_url, dict(base=merge_base_sha, head=branch_sha))
current_app.logger.debug('Compare URL 2 {}'.format(compare2_url))
compare2 = get(compare2_url, auth=github_auth).json()
touched = set([file['filename'] for file in compare2['files']])
current_app.logger.debug('Touched files {}'.format(', '.join(touched)))
return touched
def __init__(self, bb, mode='fetch', **kwargs):
self.bb = bb
if mode:
self.url = [uritemplate.expand(x, **kwargs) for x in self.uri]
self.data = {}
if mode == 'fetch':
for arg in kwargs:
setattr(self, arg, kwargs[arg])
self.data = {}
for url in self.url:
self.data.update(self.get(url))
elif mode == 'list':
self.instances = []
for instance in self.get(self.url[0]):
kw = kwargs.copy()
kw.update(instance)
instance = type(self)(self.bb, mode=None, **kw)
instance.url = [uritemplate.expand(x, **kw) for x in instance.uri]
self.instances.append(instance)
else:
self.data = kwargs
for datum in self.data:
if datum == 'data':
setattr(self, '_' + datum, self.data[datum])
else:
setattr(self, datum, self.data[datum])
def get_route_from_fk(resource_type, pk=None):
"""Gets a fully qualified URL for a given resource_type, pk"""
routes = requests.get(zc_settings.GATEWAY_ROOT_PATH).json()
for route in routes.iterkeys():
if 'resource_type' in routes[route] and routes[route]['resource_type'] == resource_type:
if isinstance(pk, (list, set)):
expanded = '{}?filter[id__in]={}'.format(expand(route, {}), ','.join([str(x) for x in pk]))
else:
expanded = expand(route, {'id': pk})
return '{0}{1}'.format(routes[route]['domain'], expanded)
raise RouteNotFoundException('No route for resource_type: "{0}"'.format(resource_type))
def get_odes_extracts(db, api_key):
'''
'''
odeses, extracts = list(), list()
vars = dict(api_key=api_key)
extracts_url = uritemplate.expand(odes_extracts_url, vars)
resp = requests.get(extracts_url)
if resp.status_code in range(200, 299):
odeses.extend([
data.ODES(str(oj['id']),
status=oj['status'],
bbox=oj['bbox'],
links=oj.get('download_links', {}),
processed_at=(parse_datetime(oj['processed_at'])
if oj['processed_at'] else None),
created_at=(parse_datetime(oj['created_at'])
if oj['created_at'] else None))
for oj in resp.json()
])
for odes in sorted(odeses, key=attrgetter('created_at'), reverse=True):
extract = data.get_extract(db, odes=odes)
if extract is None:
extract = data.Extract(None, None, None, odes, None, None, None)
extracts.append(extract)
return extracts
def create(cls, payload, client=None):
"""Create a new consumer.
:param payload: the options for creating the new consumer
:type payload: ConsumerPayload
:param client: the configured connection to Bitbucket.
If not provided, assumes an Anonymous connection.
:type client: bitbucket.Client
:returns: the new consumer object.
:rtype: Consumer
:raises: ValueError
"""
client = client or Client()
owner = client.get_username()
if not owner:
raise ValueError('owner is required')
data = payload.validate().build()
templates = cls.extract_templates_from_json()
api_url = expand(templates['create'], {
'bitbucket_url': client.get_bitbucket_url(),
'username': owner
})
# Note: This Bitbucket API expects a urlencoded-form, not json.
# Hence, use `data` instead of `json`.
return cls.post(api_url, data=data, client=client)
def build(serviceName, version,
http=None,
discoveryServiceUrl=DISCOVERY_URI,
developerKey=None,
model=JsonModel(),
requestBuilder=HttpRequest):
params = {
'api': serviceName,
'apiVersion': version
}
if http is None:
http = httplib2.Http()
requested_url = uritemplate.expand(discoveryServiceUrl, params)
logging.info('URL being requested: %s' % requested_url)
resp, content = http.request(requested_url)
service = simplejson.loads(content)
fn = os.path.join(os.path.dirname(__file__), "contrib",
serviceName, "future.json")
try:
f = file(fn, "r")
d = simplejson.load(f)
f.close()
future = d['resources']
auth_discovery = d['auth']
except IOError:
future = {}
auth_discovery = {}
base = urlparse.urljoin(discoveryServiceUrl, service['restBasePath'])
resources = service['resources']
class Service(object):
"""Top level interface for a service"""
def __init__(self, http=http):
self._http = http
self._baseUrl = base
self._model = model
self._developerKey = developerKey
self._requestBuilder = requestBuilder
def auth_discovery(self):
return auth_discovery
def createMethod(theclass, methodName, methodDesc, futureDesc):
def method(self):
return createResource(self._http, self._baseUrl, self._model,
self._requestBuilder, methodName,
self._developerKey, methodDesc, futureDesc)
setattr(method, '__doc__', 'A description of how to use this function')
setattr(method, '__is_resource__', True)
setattr(theclass, methodName, method)
for methodName, methodDesc in resources.iteritems():
createMethod(Service, methodName, methodDesc, future.get(methodName, {}))
return Service()
def format_href(self, **kw):
"""
Default implementation for href, which expands the uri_template
with the provided arguments
"""
return expand(self.uri_template, **kw)
async def asset_uploader(
gh: AioGitHubApi,
queue: "asyncio.Queue[Optional[AssetUploadWork]]",
) -> bool:
success = True
while True:
work = await queue.get()
if not work:
queue.task_done()
return success
logger.info("uploading %s", work.name)
try:
with open(work.path, "rb") as f:
await gh.upload(
uritemplate.expand(work.upload_url, name=work.name),
f,
work.content_type,
)
except Exception:
logger.exception("uploading %s failed", work.name)
success = False
else:
logger.info("uploaded %s", work.name)
finally:
queue.task_done()
def widget():
"""
Finds issues based on the given label. Render them in the widget
"""
# Get optional parameters
labels = request.args.get("labels", None)
org_name = request.args.get("organization_name")
org_type = request.args.get("org_type")
number = request.args.get("number")
tracking_status = request.args.get("tracking")
# Build the url
if org_name and labels:
issues_path_template = "organizations{/org_name}/issues/labels{/labels}{?query*}"
elif org_name:
issues_path_template = "organizations{/org_name}/issues{?query*}"
elif labels:
issues_path_template = "issues/labels{/labels}{?query*}"
else:
issues_path_template = "issues{?query*}"
issues_url_template = urljoin(CFAPI_BASE, issues_path_template)
issues_url_kwargs = ("organization_type", org_type), ("per_page", number)
url_args = dict(org_name=org_name, labels=labels, query={k: v for (k, v) in issues_url_kwargs if v})
issues_url = expand(issues_url_template, url_args)
# Get the actual issues from the API
try:
issues_response = get(issues_url, timeout=5)
except Timeout:
return render_template("widget.html", error=True)
except ConnectionError, e:
return render_template("widget.html", error=True)
def main():
'''Main logic'''
import argparse
parser = argparse.ArgumentParser('Create GitHub relase')
parser.add_argument('--token', '-t', required=True,
help='GitHub authentication token')
parser.add_argument('--user', '-u', required=True,
help='GitHub user account')
parser.add_argument('--repo', '-r', required=True,
help='GitHub repo name')
parser.add_argument('--version', '-v', required=True,
help='Version to create')
parser.add_argument('--file', '-f',
help='File to upload to release')
parser.add_argument('--content-type', '-c',
help='Content type of file')
args = parser.parse_args()
api_url = 'https://api.github.com/repos/{user}/{repo}/releases'
api_url = api_url.format(user=args.user, repo=args.repo)
# Create release
release_json = create_release(api_url, args.token, args.version)
# Upload file
if args.file:
upload_url = expand(release_json['upload_url'],
{'name': os.path.basename(args.file)})
upload_file(upload_url, args.token, args.file, args.content_type)
def get_odes_extract(db, id, api_key):
'''
'''
extract, odes = data.get_extract(db, extract_id=id), None
if extract is None:
# Nothing by that name in the database, so ask the ODES API.
vars = dict(id=id, api_key=api_key)
extract_url = uritemplate.expand(odes_extracts_url, vars)
resp = requests.get(extract_url)
if resp.status_code in range(200, 299):
oj = resp.json()
odes = data.ODES(str(oj['id']), status=oj['status'], bbox=oj['bbox'],
links=oj.get('download_links', {}),
processed_at=(parse_datetime(oj['processed_at']) if oj['processed_at'] else None),
created_at=(parse_datetime(oj['created_at']) if oj['created_at'] else None))
if odes is None:
# Nothing at all for this ID anywhere.
return None
if odes is None:
# A DB extract was found, but nothing in ODES - very weird!
return get_odes_extract(db, extract.odes.id, api_key)
# We have a known ODES, so look for it in the database.
extract = data.get_extract(db, odes=odes)
if extract is None:
# Known ODES, but nothing in the DB so make one up.
return data.Extract(None, None, None, odes, None, None, None)
return extract
def request_odes_extract(extract, request, url_for, api_key):
'''
'''
env = Environment(loader=PackageLoader(__name__, 'templates'))
args = dict(
name = extract.name or extract.wof.name or 'an unnamed place',
link = urljoin(util.get_base_url(request), url_for('ODES.get_extract', extract_id=extract.id)),
extracts_link = urljoin(util.get_base_url(request), url_for('ODES.get_extracts')),
created = extract.created
)
email = dict(
email_subject=env.get_template('email-subject.txt').render(**args),
email_body_text=env.get_template('email-body.txt').render(**args),
email_body_html=env.get_template('email-body.html').render(**args)
)
params = {key: extract.envelope.bbox[i] for (i, key) in enumerate(('bbox_w', 'bbox_s', 'bbox_e', 'bbox_n'))}
params.update(email)
post_url = uritemplate.expand(odes_extracts_url, dict(api_key=api_key))
resp = requests.post(post_url, data=params)
oj = resp.json()
if 'error' in oj:
raise util.KnownUnknown("Error: {}".format(oj['error']))
elif resp.status_code != 200:
raise Exception("Bad ODES status code: {}".format(resp.status_code))
return data.ODES(str(oj['id']), status=oj['status'], bbox=oj['bbox'],
links=oj.get('download_links', {}),
processed_at=(parse_datetime(oj['processed_at']) if oj['processed_at'] else None),
created_at=(parse_datetime(oj['created_at']) if oj['created_at'] else None))
def setup_class(cls):
template = (Bitbucket(client=cls.test_client).data.get(
'_links', {}).get('snippetsForRole', {}).get('href'))
cls.url = expand(template, {
'bitbucket_url': cls.test_client.get_bitbucket_url(),
'role': cls.role
})
def _get_related_url(self, property_name, element_type, **kwargs):
if property_name in self:
template = self[property_name]
url = uritemplate.expand(template, kwargs)
return self._client.get_list_absolute_url(
url,
element_type,
fetch_params=self._get_related_fetch_params())
else:
template = self._default_urls[property_name].format(
**self._fetch_params, **self)
url = uritemplate.expand(template, kwargs)
return self._client.get_list_relative_url(
url,
element_type,
fetch_params=self._get_related_fetch_params())
def expand_uri(self, name, **kwargs):
link = self.schema.get_link(name)
href = link.get('href', '')
context = dict(self.data, **kwargs)
return uritemplate.expand(href, context)
def renewnodevice(self, end):
link = self.get_link(self.RENEW)
pend = datetime.utcfromtimestamp(end).isoformat() + 'Z'
if link['type'] == License.STATUS_MIMETYPE and link[
'templated'] == True:
regurl = expand(link['href'], {'end': pend})
self.status = json.loads(self._put(regurl))
def _get_build_content(
serviceName,
version,
discoveryServiceUrl=DISCOVERY_URI,
cache_discovery=True,
cache=None,
proxy_info=None,
):
params = {"api": serviceName, "apiVersion": version}
discovery_http = httplib2.Http(timeout=60)
if proxy_info:
discovery_http.proxy_info = httplib2.ProxyInfo(
httplib2.socks.PROXY_TYPE_HTTP_NO_TUNNEL,
proxy_info["host"],
proxy_info["port"],
proxy_user=proxy_info["user"],
proxy_pass=proxy_info["password"],
)
discovery_http.disable_ssl_certificate_validation = True
for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI):
requested_url = uritemplate.expand(discovery_url, params)
try:
return _retrieve_discovery_doc(requested_url, discovery_http,
cache_discovery, cache)
except HttpError as e:
if e.resp.status == 404:
continue
else:
raise e
raise Exception("name: %s version: %s" % (serviceName, version))
def generate_iri(self,
values,
default_representation=HYDRA.BasicRepresentation):
"""
Generate an IRI according to this template.
This method takes care of formatting the provided values according to the template's variableRepresentation.
:param values: a dict whose keys are properties, and values are RDF terms
:return: the IRI as a string
NB: if required properties are missing from `values`, a ValueError is raised.
"""
# ensure that keys are URIRefs in values
graph = self.graph
values = {URIRef(key): value for key, value in values.items()}
representation = self.get_variable_representation(
default=default_representation)
mode = 0 if representation is HYDRA.BasicRepresentation else 1
data, msg = self._map_properties(values)
if data is None:
raise ValueError(msg)
data = {
key: _format_variable(value, mode)
for key, value in data.items()
}
return expand(self.template, data)
def format_data_href(self, **kw):
"""
Default implementation for data-href, same as format_href, with
uri_template_json instead.
"""
return expand(self.uri_template_json, **kw)
def __get_discovery_doc(self, service_name, version, http=None,
discovery_service_url=DISCOVERY_URI):
# Returned a cached copy if we have it.
cached = memcache.get("discovery_doc")
if cached:
return cached
logging.info("Cache miss in discovery document.")
params = {'api': service_name, 'apiVersion': version}
requested_url = uritemplate.expand(discovery_service_url, params)
# REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
# variable that contains the network address of the client sending the
# request. If it exists then add that to the request for the discovery
# document to avoid exceeding the quota on discovery requests.
if 'REMOTE_ADDR' in os.environ:
requested_url = _add_query_parameter(requested_url, 'userIp',
os.environ['REMOTE_ADDR'])
http = http or httplib2.Http()
resp, content = http.request(requested_url)
if resp.status >= 400:
raise HttpError(resp, content, uri=requested_url)
# Store it in the memcache.
memcache.set("discovery_doc", content, time=60 * 60 * 24)
return content
def get_auth(self):
self.token_uri = expand('{+server_base_uri}/site/oauth2/access_token',
{'server_base_uri': self.server_base_uri})
token_response = """
{
"access_token":"2YotnFZFEjr1zCsicMWpAA",
"token_type":"bearer",
"expires_in":3600,
"refresh_token":"tGzv3JOkF0XG5Qx2TlKWIA",
"example_parameter":"example_value",
"scope":"repository"
}
"""
httpretty.HTTPretty.allow_net_connect = False
httpretty.register_uri(httpretty.POST,
self.token_uri,
content_type='application/json',
body=token_response,
status=200)
a = OAuth2Authenticator(self.client_id,
self.client_secret,
self.email,
self.grant,
redirect_uris=self.redirect_uris,
server_base_uri=self.server_base_uri,
client_name=self.client_name,
client_description=self.client_description)
return a
def create_repository(username,
repository_name,
fork_policy,
is_private,
scm=None,
name=None,
description=None,
language=None,
has_issues=None,
has_wiki=None,
client=Client()):
template = ('{+bitbucket_url}' +
'/2.0/repositories/{username}/{repository_name}')
url = expand(
template, {
'bitbucket_url': client.get_bitbucket_url(),
'username': username,
'repository_name': repository_name
})
payload = Repository.make_new_repository_payload(
fork_policy, is_private, scm, name, description, language,
has_issues, has_wiki)
response = client.session.post(url, data=payload)
Client.expect_ok(response)
return client.convert_to_object(response.json())
def RelativeName(self, url_escape=False):
"""Relative resource name.
A URI path ([path-noscheme](http://tools.ietf.org/html/rfc3986#appendix-A))
without the leading "/". It identifies a resource within the API service.
For example:
"shelves/shelf1/books/book2"
Args:
url_escape: bool, if true would url escape each parameter.
Returns:
Unescaped part of SelfLink which is essentially base_url + relative_name.
For example if SelfLink is
https://pubsub.googleapis.com/v1/projects/myprj/topics/mytopic
then relative name is
projects/myprj/topics/mytopic.
"""
escape_func = urllib.parse.quote if url_escape else lambda x, safe: x
effective_params = dict(
[(k, escape_func(getattr(self, k), safe=''))
for k in self._params])
return urllib.parse.unquote(
uritemplate.expand(self._path, effective_params))
def main():
'''Main logic'''
import argparse
parser = argparse.ArgumentParser('Create GitHub relase')
parser.add_argument('--token',
'-t',
required=True,
help='GitHub authentication token')
parser.add_argument('--user',
'-u',
required=True,
help='GitHub user account')
parser.add_argument('--repo', '-r', required=True, help='GitHub repo name')
parser.add_argument('--version',
'-v',
required=True,
help='Version to create')
parser.add_argument('--file', '-f', help='File to upload to release')
parser.add_argument('--content-type', '-c', help='Content type of file')
args = parser.parse_args()
api_url = 'https://api.github.com/repos/{user}/{repo}/releases'
api_url = api_url.format(user=args.user, repo=args.repo)
# Create release
release_json = create_release(api_url, args.token, args.version)
# Upload file
if args.file:
upload_url = expand(release_json['upload_url'],
{'name': os.path.basename(args.file)})
upload_file(upload_url, args.token, args.file, args.content_type)
def expand_uri(self, name, **kwargs):
link = self.schema.get_link(name)
href = link.get('href', '')
context = dict(self.data, **kwargs)
return uritemplate.expand(href, context)
def setup_class(cls):
cls.url = expand(
Hook.templates['create'], {
'bitbucket_url': cls.test_client.get_bitbucket_url(),
'owner': cls.test_client.get_username(),
'repository_name': cls.repository_name,
})
def get_gcs_client(self) -> luigi.contrib.gcs.GCSClient:
if (not os.path.isfile(self.discover_cache_local_path)):
with open(self.discover_cache_local_path, "w") as f:
try:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
params = {"api": "storage", "apiVersion": "v1"}
discovery_http = build_http()
for discovery_url in (self._DISCOVERY_URI,
self._V2_DISCOVERY_URI):
requested_url = uritemplate.expand(
discovery_url, params)
try:
content = _retrieve_discovery_doc(
requested_url, discovery_http, False)
except HttpError as e:
if e.resp.status == http_client.NOT_FOUND:
continue
else:
raise e
break
f.write(content)
fcntl.flock(f, fcntl.LOCK_UN)
except IOError:
# try to read
pass
with open(self.discover_cache_local_path, "r") as f:
fcntl.flock(f, fcntl.LOCK_SH)
descriptor = f.read()
fcntl.flock(f, fcntl.LOCK_UN)
return luigi.contrib.gcs.GCSClient(
oauth_credentials=self._load_oauth_credentials(),
descriptor=descriptor)
def document_api(name, version, uri):
"""Document the given API.
Args:
name: string, Name of the API.
version: string, Version of the API.
uri: string, URI of the API's discovery document
"""
try:
service = build(name, version)
except UnknownApiNameOrVersion as e:
print("Warning: {} {} found but could not be built.".format(
name, version))
return
except HttpError as e:
print("Warning: {} {} returned {}.".format(name, version, e))
return
http = build_http()
response, content = http.request(
uri or uritemplate.expand(FLAGS.discovery_uri_template, {
"api": name,
"apiVersion": version
}))
discovery = json.loads(content)
version = safe_version(version)
document_collection_recursive(service, "{}_{}.".format(name, version),
discovery, discovery)
def retrieve_discovery_doc(service_name, version, http=None,
discovery_service_url=DISCOVERY_URI):
"""Retrieves the discovery doc."""
params = {'api': service_name, 'apiVersion': version}
requested_url = uritemplate.expand(discovery_service_url, params)
# REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
# variable that contains the network address of the client sending the
# request. If it exists then add that to the request for the discovery
# document to avoid exceeding the quota on discovery requests.
if 'REMOTE_ADDR' in os.environ:
requested_url = _add_query_parameter(requested_url, 'userIp',
os.environ['REMOTE_ADDR'])
http = http or httplib2.Http()
resp, content = http.request(requested_url)
if resp.status >= 400:
raise HttpError(resp, content, uri=requested_url)
try:
json.loads(content)
except ValueError:
raise InvalidJsonError(
'Bad JSON: %s from %s.' % (content, requested_url))
# we return content instead of the JSON deserialized service because
# build_from_document() consumes a string rather than a dictionary
return content
def RelativeName(self, url_escape=False):
"""Relative resource name.
A URI path ([path-noscheme](http://tools.ietf.org/html/rfc3986#appendix-A))
without the leading "/". It identifies a resource within the API service.
For example:
"shelves/shelf1/books/book2"
Args:
url_escape: bool, if true would url escape each parameter.
Returns:
Unescaped part of SelfLink which is essentially base_url + relative_name.
For example if SelfLink is
https://pubsub.googleapis.com/v1/projects/myprj/topics/mytopic
then relative name is
projects/myprj/topics/mytopic.
"""
escape_func = urllib.parse.quote if url_escape else lambda x, safe: x
effective_params = dict(
[(k, escape_func(getattr(self, k), safe=''))
for k in self._params])
return urllib.parse.unquote(
uritemplate.expand(self._path, effective_params))
def setup_class(cls):
cls.url = expand(
BranchRestriction.templates['create'], {
'bitbucket_url': cls.test_client.get_bitbucket_url(),
'owner': cls.owner,
'repository_name': cls.repository_name
})
def delete(self):
if not hasattr(self, 'url'):
self.url = [
uritemplate.expand(x, user=self.owner['username'])
for x in self.uri
]
self.delete_(self.url[0] + '/%d' % self.pk)
def find_consumers(client=Client()):
url = expand(
Consumer.get_link_template('consumers'), {
'bitbucket_url': client.get_bitbucket_url(),
'username': client.get_username()
})
return client.remote_relationship(url)
def __init__(self, collection_info, subcollection, param_values,
endpoint_url):
"""Create a Resource object that may be partially resolved.
To allow resolving of unknown params to happen after parse-time, the
param resolution code is in this class rather than the _ResourceParser
class.
Args:
collection_info: resource_util.CollectionInfo, The collection description
for this resource.
subcollection: str, id for subcollection of this collection.
param_values: {param->value}, A list of values for parameters.
endpoint_url: str, override service endpoint url for this resource. If
None default base url of collection api will be used.
Raises:
RequiredFieldOmittedException: if param_values have None value.
"""
self._collection_info = collection_info
self._endpoint_url = endpoint_url or collection_info.base_url
self._subcollection = subcollection
self._path = collection_info.GetPath(subcollection)
self._params = collection_info.GetParams(subcollection)
for param, value in param_values.iteritems():
if value is None:
raise RequiredFieldOmittedException(collection_info.full_name, param)
setattr(self, param, value)
self._self_link = '{0}{1}'.format(
self._endpoint_url, uritemplate.expand(self._path, self.AsDict()))
if (self._collection_info.api_name
in ('compute', 'clouduseraccounts', 'storage')):
# TODO(b/15425944): Unquote URLs for now for these apis.
self._self_link = urllib.unquote(self._self_link)
self._initialized = True
def __init__(self, collection_info, subcollection, param_values,
endpoint_url):
"""Create a Resource object that may be partially resolved.
To allow resolving of unknown params to happen after parse-time, the
param resolution code is in this class rather than the _ResourceParser
class.
Args:
collection_info: resource_util.CollectionInfo, The collection description
for this resource.
subcollection: str, id for subcollection of this collection.
param_values: {param->value}, A list of values for parameters.
endpoint_url: str, override service endpoint url for this resource. If
None default base url of collection api will be used.
Raises:
RequiredFieldOmittedException: if param_values have None value.
"""
self._collection_info = collection_info
self._endpoint_url = endpoint_url or collection_info.base_url
self._subcollection = subcollection
self._path = collection_info.GetPath(subcollection)
self._params = collection_info.GetParams(subcollection)
for param, value in param_values.iteritems():
if value is None:
raise RequiredFieldOmittedException(collection_info.full_name, param)
setattr(self, param, value)
self._self_link = '{0}{1}'.format(
self._endpoint_url, uritemplate.expand(self._path, self.AsDict()))
if (self._collection_info.api_name
in ('compute', 'clouduseraccounts', 'storage')):
# TODO(b/15425944): Unquote URLs for now for these apis.
self._self_link = urllib.unquote(self._self_link)
self._initialized = True
def test_pagination(self):
self.client.auto_paginate = True
url = uritemplate.expand(self.client.url, {'param': 'foo'})
h1 = {
'Link': '<'+url+'?page=2&per_page=100>; rel="next"',
'X-RateLimit-Remaining': '56',
'X-RateLimit-Reset': '1446804464',
'X-RateLimit-Limit': '60'
}
h2 = {
'Link': '<'+url+'?page=3&per_page=100>; rel="next"',
'X-RateLimit-Remaining': '56',
'X-RateLimit-Reset': '1446804464',
'X-RateLimit-Limit': '60'
}
h3 = {
'X-RateLimit-Remaining': '56',
'X-RateLimit-Reset': '1446804464',
'X-RateLimit-Limit': '60'
}
res1 = '["a","b"]'
res2 = '["c","d"]'
res3 = '["e","f"]'
self.adapter.register_uri('GET', url, headers=h1, text=res1)
self.adapter.register_uri('GET', url+'?page=2', headers=h2, text=res2)
self.adapter.register_uri('GET', url+'?page=3', headers=h3, text=res3)
response = self.client.paginate(param='foo')
resultSchema = [r.schema for r in response.schema]
expectedSchema = ['a', 'b', 'c', 'd', 'e', 'f']
self.assertEqual(resultSchema, expectedSchema)
def build(serviceName,
version,
http=None,
discoveryServiceUrl=DISCOVERY_URI,
developerKey=None,
model=None,
requestBuilder=HttpRequest):
"""Construct a Resource for interacting with an API.
Construct a Resource object for interacting with
an API. The serviceName and version are the
names from the Discovery service.
Args:
serviceName: string, name of the service
version: string, the version of the service
discoveryServiceUrl: string, a URI Template that points to
the location of the discovery service. It should have two
parameters {api} and {apiVersion} that when filled in
produce an absolute URI to the discovery document for
that service.
developerKey: string, key obtained
from https://code.google.com/apis/console
model: apiclient.Model, converts to and from the wire format
requestBuilder: apiclient.http.HttpRequest, encapsulator for
an HTTP request
Returns:
A Resource object with methods for interacting with
the service.
"""
params = {'api': serviceName, 'apiVersion': version}
if http is None:
http = httplib2.Http()
requested_url = uritemplate.expand(discoveryServiceUrl, params)
# REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
# variable that contains the network address of the client sending the
# request. If it exists then add that to the request for the discovery
# document to avoid exceeding the quota on discovery requests.
if 'REMOTE_ADDR' in os.environ:
requested_url = _add_query_parameter(requested_url, 'userIp',
os.environ['REMOTE_ADDR'])
logging.info('URL being requested: %s' % requested_url)
resp, content = http.request(requested_url)
if resp.status == 404:
raise UnknownApiNameOrVersion("name: %s version: %s" %
(serviceName, version))
if resp.status >= 400:
raise HttpError(resp, content, requested_url)
try:
service = simplejson.loads(content)
except ValueError, e:
logging.error('Failed to parse as JSON: ' + content)
raise InvalidJsonError()
def document_api(name, version):
"""Document the given API.
Args:
name: string, Name of the API.
version: string, Version of the API.
"""
try:
service = build(name, version)
except UnknownApiNameOrVersion as e:
print 'Warning: {} {} found but could not be built.'.format(
name, version)
return
response, content = http.request(
uritemplate.expand(FLAGS.discovery_uri_template, {
'api': name,
'apiVersion': version
}))
discovery = json.loads(content)
version = safe_version(version)
document_collection_recursive(service, '%s_%s.' % (name, version),
discovery, discovery)
def create_buildstatus(owner,
repository_name,
revision,
key,
state,
url,
name=None,
description=None,
client=Client()):
template = ('{+bitbucket_url}' +
'/2.0/repositories{/owner,repository_name}' +
'/commit{/revision}/statuses/build')
# owner, repository_name, and revision are required
api_url = expand(
template, {
'bitbucket_url': client.get_bitbucket_url(),
'owner': owner,
'repository_name': repository_name,
'revision': revision
})
payload = BuildStatus.make_payload(key=key,
state=state,
url=url,
name=name,
description=description)
return BuildStatus.post(api_url, json=payload, client=client)
def _refresh(self, http_request):
"""Refreshes the access_token.
Skip all the storage hoops and just refresh using the API.
Args:
http_request: callable, a callable that matches the method signature of
httplib2.Http.request, used to make the refresh request.
Raises:
AccessTokenRefreshError: When the refresh fails.
"""
uri = uritemplate.expand(META, {'scope': self.scope})
response, content = http_request(uri)
if response.status == 200:
try:
d = simplejson.loads(content)
except Exception as e:
raise AccessTokenRefreshError(str(e))
self.access_token = d['accessToken']
else:
if response.status == 404:
content = content + (' This can occur if a VM was created'
' with no service account or scopes.')
raise AccessTokenRefreshError(content)
def retrieve_discovery_doc(serviceName, version,
discoveryServiceUrl=DISCOVERY_URI):
params = {'api': serviceName, 'apiVersion': version}
requested_url = uritemplate.expand(discoveryServiceUrl, params)
# REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
# variable that contains the network address of the client sending the
# request. If it exists then add that to the request for the discovery
# document to avoid exceeding the quota on discovery requests.
if 'REMOTE_ADDR' in os.environ:
requested_url = _add_query_parameter(requested_url, 'userIp',
os.environ['REMOTE_ADDR'])
http = httplib2.Http()
resp, content = http.request(requested_url)
if resp.status >= 400:
raise HttpError(resp, content, uri=requested_url)
try:
service = json.loads(content)
except ValueError:
raise InvalidJsonError(
'Bad JSON: %s from %s.' % (content, requested_url))
# We return content instead of the JSON deserialized service because
# build_from_document() consumes a string rather than a dictionary.
return content
def expand(self, _keep_templated=False, **kwargs):
'''Expand template args in a templated Navigator.
if :_keep_templated: is True, the resulting Navigator can be further
expanded. A Navigator created this way is not part of the id map.
'''
if not self.templated:
raise TypeError(
"This Navigator isn't templated! You can't expand it.")
for k, v in kwargs.iteritems():
if v == 0:
kwargs[k] = '0' # uritemplate expands 0's to empty string
if self.template_args is not None:
kwargs.update(self.template_args)
cp = self._copy(
uri=uritemplate.expand(self.template_uri, kwargs),
templated=_keep_templated,
)
if not _keep_templated:
cp.template_uri = None
cp.template_args = None
else:
cp.template_args = kwargs
return cp
def load_toots(mastodon_whoami_url,
mastodon_statuses_url,
mastodon_header,
max_count=50):
''' Load recent toots from a Mastodon instance and return list of Toot objects.
These are assumed to be in reverse-chronological order.
'''
got1 = requests.get(mastodon_whoami_url, headers=mastodon_header)
mastodon_id = got1.json().get('id')
mastodon_toots = list()
url = uritemplate.expand(mastodon_statuses_url, dict(id=mastodon_id))
while len(mastodon_toots) < max_count:
print('Get', url, '...', file=sys.stderr)
got2 = requests.get(url, headers=mastodon_header)
for status in got2.json():
url = status['url']
soup = bs4.BeautifulSoup(status['content'], 'html.parser')
links = [a['href'] for a in soup.find_all('a')]
mastodon_toots.append(Toot(url, links))
print(mastodon_toots[-1], file=sys.stderr)
if 'next' in got2.links:
url = got2.links['next']['url']
else:
break
return mastodon_toots
def get_odes_extract(db, id, api_key):
'''
'''
extract, odes = data.get_extract(db, extract_id=id), None
if extract is None:
# Nothing by that name in the database, so ask the ODES API.
vars = dict(id=id, api_key=api_key)
extract_url = uritemplate.expand(odes_extracts_url, vars)
resp = requests.get(extract_url)
if resp.status_code in range(200, 299):
oj = resp.json()
odes = data.ODES(str(oj['id']), status=oj['status'], bbox=oj['bbox'],
links=oj.get('download_links', {}),
processed_at=(parse_datetime(oj['processed_at']) if oj['processed_at'] else None),
created_at=(parse_datetime(oj['created_at']) if oj['created_at'] else None))
if odes is None:
# Nothing at all for this ID anywhere.
return None
if odes is None:
# A DB extract was found, but nothing in ODES - very weird!
return get_odes_extract(db, extract.odes.id, api_key)
# We have a known ODES, so look for it in the database.
extract = data.get_extract(db, odes=odes)
if extract is None:
# Known ODES, but nothing in the DB so make one up.
return data.Extract(None, None, None, odes, None, None, None)
return extract
def _delete(self, **kwargs):
# TODO: Refactor to resolve circular dependency.
from pluct import resource
self.href = expand(self.href, self.resource.data)
response = requests.delete(url=self.href, headers=self.get_headers())
return resource.from_response(response)
def request_odes_extract(extract, request, url_for, api_key):
'''
'''
env = Environment(loader=PackageLoader(__name__, 'templates'))
args = dict(
name = extract.name or extract.wof.name or 'an unnamed place',
link = urljoin(util.get_base_url(request), url_for('ODES.get_extract', extract_id=extract.id)),
extracts_link = urljoin(util.get_base_url(request), url_for('ODES.get_extracts')),
created = extract.created
)
email = dict(
email_subject=env.get_template('email-subject.txt').render(**args),
email_body_text=env.get_template('email-body.txt').render(**args),
email_body_html=env.get_template('email-body.html').render(**args)
)
params = {key: extract.envelope.bbox[i] for (i, key) in enumerate(('bbox_w', 'bbox_s', 'bbox_e', 'bbox_n'))}
params.update(email)
post_url = uritemplate.expand(odes_extracts_url, dict(api_key=api_key))
resp = requests.post(post_url, data=params)
oj = resp.json()
if 'error' in oj:
raise util.KnownUnknown("Error: {}".format(oj['error']))
elif resp.status_code != 200:
raise Exception("Bad ODES status code: {}".format(resp.status_code))
return data.ODES(str(oj['id']), status=oj['status'], bbox=oj['bbox'],
links=oj.get('download_links', {}),
processed_at=(parse_datetime(oj['processed_at']) if oj['processed_at'] else None),
created_at=(parse_datetime(oj['created_at']) if oj['created_at'] else None))
def get_odes_extracts(db, api_key):
'''
'''
odeses, extracts = list(), list()
vars = dict(api_key=api_key)
extracts_url = uritemplate.expand(odes_extracts_url, vars)
resp = requests.get(extracts_url)
if resp.status_code in range(200, 299):
odeses.extend([data.ODES(str(oj['id']), status=oj['status'], bbox=oj['bbox'],
links=oj.get('download_links', {}),
processed_at=(parse_datetime(oj['processed_at']) if oj['processed_at'] else None),
created_at=(parse_datetime(oj['created_at']) if oj['created_at'] else None))
for oj in resp.json()])
for odes in sorted(odeses, key=attrgetter('created_at'), reverse=True):
extract = data.get_extract(db, odes=odes)
if extract is None:
extract = data.Extract(None, None, None, odes, None, None, None)
extracts.append(extract)
return extracts
def document_api(name, version):
"""Document the given API.
Args:
name: string, Name of the API.
version: string, Version of the API.
"""
try:
service = build(name, version)
except UnknownApiNameOrVersion as e:
print 'Warning: {} {} found but could not be built.'.format(name, version)
return
response, content = http.request(
uritemplate.expand(
FLAGS.discovery_uri_template, {
'api': name,
'apiVersion': version})
)
discovery = json.loads(content)
version = safe_version(version)
document_collection_recursive(
service, '%s_%s.' % (name, version), discovery, discovery)
def build(service, version, http=httplib2.Http(),
discoveryServiceUrl = DISCOVERY_URI, auth = None, model = JsonModel()):
params = {
'api': service,
'apiVersion': version
}
resp, content = http.request(uritemplate.expand(discoveryServiceUrl, params))
d = simplejson.loads(content)
service = d['data'][service][version]
base = service['baseUrl']
resources = service['resources']
class Service(object):
"""Top level interface for a service"""
def __init__(self, http=http):
self._http = http
self._baseUrl = base
self._model = model
def createMethod(theclass, methodName, methodDesc):
def method(self, **kwargs):
return createResource(self._http, self._baseUrl, self._model,
methodName, methodDesc)
setattr(method, '__doc__', 'A description of how to use this function')
setattr(theclass, methodName, method)
for methodName, methodDesc in resources.iteritems():
createMethod(Service, methodName, methodDesc)
return Service()
def build(
serviceName,
version,
http=None,
discoveryServiceUrl=DISCOVERY_URI,
developerKey=None,
model=None,
requestBuilder=HttpRequest,
):
"""Construct a Resource for interacting with an API.
Construct a Resource object for interacting with
an API. The serviceName and version are the
names from the Discovery service.
Args:
serviceName: string, name of the service
version: string, the version of the service
discoveryServiceUrl: string, a URI Template that points to
the location of the discovery service. It should have two
parameters {api} and {apiVersion} that when filled in
produce an absolute URI to the discovery document for
that service.
developerKey: string, key obtained
from https://code.google.com/apis/console
model: apiclient.Model, converts to and from the wire format
requestBuilder: apiclient.http.HttpRequest, encapsulator for
an HTTP request
Returns:
A Resource object with methods for interacting with
the service.
"""
params = {"api": serviceName, "apiVersion": version}
if http is None:
http = httplib2.Http()
requested_url = uritemplate.expand(discoveryServiceUrl, params)
# REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
# variable that contains the network address of the client sending the
# request. If it exists then add that to the request for the discovery
# document to avoid exceeding the quota on discovery requests.
if "REMOTE_ADDR" in os.environ:
requested_url = _add_query_parameter(requested_url, "userIp", os.environ["REMOTE_ADDR"])
logging.info("URL being requested: %s" % requested_url)
resp, content = http.request(requested_url)
if resp.status == 404:
raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName, version))
if resp.status >= 400:
raise HttpError(resp, content, requested_url)
try:
service = simplejson.loads(content)
except ValueError, e:
logging.error("Failed to parse as JSON: " + content)
raise InvalidJsonError()
def expand_uri(template, args):
'''
'''
new_args = {k: v for (k, v) in args.items() if not hasattr(v, 'encode')}
new_args.update({k: v.encode('utf8') for (k, v) in args.items() if hasattr(v, 'encode')})
return uritemplate.expand(template, new_args)
def expand(self, _keep_templated=False, **kwargs):
'''Expand template args in a templated Navigator.
if :_keep_templated: is True, the resulting Navigator can be further
expanded. A Navigator created this way is not part of the id map.
'''
if not self.templated:
raise TypeError(
"This Navigator isn't templated! You can't expand it.")
for k, v in kwargs.iteritems():
if v == 0:
kwargs[k] = '0' # uritemplate expands 0's to empty string
if self.template_args is not None:
kwargs.update(self.template_args)
cp = self._copy(uri=uritemplate.expand(self.template_uri, kwargs),
templated=_keep_templated,
)
if not _keep_templated:
cp.template_uri = None
cp.template_args = None
else:
cp.template_args = kwargs
return cp
def _put(self, **kwargs):
self.href = expand(self.href, self.resource.data)
data = kwargs.pop('data')
response = self._remote().put(url=self.href,
data=ujson.dumps(data),
headers=self.get_headers(),
timeout=self.resource.timeout)
return from_response(self.resource.__class__, response)
def _expand_path_params(url, path_params):
"""
Given a templated URL and some parameters that have been provided,
expand the URL.
"""
if path_params:
return uritemplate.expand(url, path_params)
return url
def issue(self, repo, number, oldData=None):
"""Return github issue and event data"""
url = uritemplate.expand(repo.data['issues_url'], { 'number': number })
issue = self.get(url)
# TODO: only get events up to the most recent in oldData?
issue['events'] = self.getAll(issue['events_url'])
logging.info('Found %d events for #%s' % (len(issue['events']), number))
return issue
def _get_related_url(self, property_name, element_type, **kwargs):
if property_name in self:
template = self[property_name]
url = uritemplate.expand(template, kwargs)
return self._client.get_list_absolute_url(
url, element_type,
fetch_params=self._get_related_fetch_params()
)
else:
template = self._default_urls[property_name].format(
**self._fetch_params, **self
)
url = uritemplate.expand(template, kwargs)
return self._client.get_list_relative_url(
url, element_type,
fetch_params=self._get_related_fetch_params()
)
def make(self, name, params={}):
try:
route = self.routes[name]
parts = urlparse(self.host)
return uritemplate.expand("{proto}://{host}{+path}", dict(proto=parts.scheme, host=parts.netloc,
path=route.template(params)))
except KeyError:
return None
def json_test_function(self):
actual = unicode(expand(template, variables))
msg = '%(template)r did not expand as expected, got %(actual)r.'\
% locals()
if type(expected) == type([]):
self.assertTrue (actual in expected, msg)
else:
self.assertEqual(expected, actual)
def expand(self, link):
if ':' in link:
(name, rel) = link.split(':', 1)
try:
return expand(self.map[name], {'rel': rel})
except KeyError:
pass
return link
