def get_media_url(self, host, media_id):
web_url = self.get_url(host, media_id)
rurl = urllib_parse.urljoin(web_url, '/')
headers = {'User-Agent': common.FF_USER_AGENT, 'Referer': rurl}
html = self.net.http_GET(web_url, headers=headers).content
r = re.search(r"btn--primary'\s*href='([^']+)", html)
if r:
common.kodi.sleep(7000)
g = self.net.http_GET(r.group(1), headers=headers)
ghtml = g.content
gurl = g.get_url()
data = helpers.get_hidden(ghtml)
headers.update({
'Origin': urllib_parse.urljoin(gurl, '/')[:-1],
'Referer': gurl,
'X-Requested-With': 'XMLHttpRequest'
})
purl = re.findall('<form.+?action="([^"]+)', ghtml)[0]
if purl.startswith('/'):
purl = urllib_parse.urljoin(gurl, purl)
common.kodi.sleep(5000)
html = self.net.http_POST(purl, form_data=data,
headers=headers).content
jd = json.loads(html)
if jd.get('status') == "success":
headers.pop('X-Requested-With')
return jd.get('url') + helpers.append_headers(headers)
raise ResolverError('File Not Found or removed')
def call_recommendation_server(server, client_id_or_guid, data, verb='get'):
"""Call taar `server` to get recommendations for a given
`client_id_or_guid`.
`data` is a dict containing either query parameters to be passed in the URL
if we're calling the server through GET, or the data we'll pass through
POST as json.
The HTTP verb to use is either "get" or "post", controlled through `verb`,
which defaults to "get"."""
request_kwargs = {
'timeout': settings.RECOMMENDATION_ENGINE_TIMEOUT
}
if verb == 'get':
params = OrderedDict(sorted(data.items(), key=lambda t: t[0]))
endpoint = urljoin(server, '%s/%s%s' % (
client_id_or_guid, '?' if params else '', urlencode(params)))
else:
endpoint = urljoin(server, '%s/' % client_id_or_guid)
request_kwargs['json'] = data
log.debug(u'Calling recommendation server: {0}'.format(endpoint))
try:
with statsd.timer('services.recommendations'):
response = getattr(requests, verb)(endpoint, **request_kwargs)
if response.status_code != 200:
raise requests.exceptions.RequestException()
except requests.exceptions.RequestException as e:
log.error(u'Calling recommendation engine failed: {0}'.format(e))
statsd.incr('services.recommendations.fail')
return None
else:
statsd.incr('services.recommendations.success')
return json.loads(response.content).get('results', None)
def get_media_url(self, host, media_id):
if '$$' in media_id:
media_id, referer = media_id.split('$$')
referer = urllib_parse.urljoin(referer, '/')
else:
referer = False
web_url = self.get_url(host, media_id)
rurl = urllib_parse.urljoin(web_url, '/')
if not referer:
referer = rurl
domain = 'aHR0cHM6Ly9zdHJlYW1yYXBpZC5ydTo0NDM.'
headers = {'User-Agent': common.FF_USER_AGENT, 'Referer': referer}
html = self.net.http_GET(web_url, headers).content
token = helpers.girc(html, rurl, domain)
number = re.findall(r"recaptchaNumber\s*=\s*'(\d+)", html)
if token and number:
eid, media_id = media_id.split('/')
surl = 'https://streamrapid.ru/ajax/embed-{0}/getSources'.format(
eid)
if '?' in media_id:
media_id = media_id.split('?')[0]
data = {'_number': number[0], 'id': media_id, '_token': token}
headers.update({'X-Requested-With': 'XMLHttpRequest'})
shtml = self.net.http_GET('{0}?{1}'.format(
surl, urllib_parse.urlencode(data)),
headers=headers).content
sources = json.loads(shtml).get('sources')
if sources:
source = sources[0].get('file')
headers.pop('X-Requested-With')
return source + helpers.append_headers(headers)
raise ResolverError('File Not Found or removed')
def get_media_url(self, host, media_id):
if '$$' in media_id:
media_id, referer = media_id.split('$$')
referer = urllib_parse.urljoin(referer, '/')
else:
referer = False
web_url = self.get_url(host, media_id)
if not referer:
referer = urllib_parse.urljoin(web_url, '/')
headers = {'User-Agent': common.FF_USER_AGENT, 'Referer': referer}
html = self.net.http_GET(web_url, headers=headers).content
params = re.search(r'video:\s*([^;]+)', html)
if params:
params = params.group(1)
server = re.findall(r'server:\s*"([^"]+)', params)[0][::-1]
server = base64.b64decode(server.encode('ascii')).decode('ascii')
ids = re.search(r'cdn_id:\s*"([^"]+)', params)
if ids:
id1, id2 = ids.group(1).split('_')
sources = json.loads(
re.findall(r'cdn_files:\s*([^}]+})', params)[0])
sources = [(key[4:], 'https://{0}/videos/{1}/{2}/{3}'.format(
server, id1, id2, sources[key].replace('.',
'.mp4?extra=')))
for key in list(sources.keys())]
else:
vid = re.findall(r'id:\s*"([^"]+)', params)[0]
ekeys = json.loads(
re.findall(r'quality":\s*([^}]+})', params)[0])
data = {
'token':
re.findall(r'access_token:\s*"([^"]+)', params)[0],
'videos':
vid,
'ckey':
re.findall(r'c_key:\s*"([^"]+)', params)[0],
'credentials':
re.findall(r'credentials:\s*"([^"]+)', params)[0]
}
vurl = 'https://{0}/method/video.get/{1}?{2}'.format(
server, vid, urllib_parse.urlencode(data))
headers.update({'Origin': referer[:-1]})
vhtml = self.net.http_GET(vurl, headers=headers).content
sources = json.loads(vhtml).get('response').get(
'items')[0].get('files')
sources = [(key[4:], sources[key] +
'&videos={0}&extra_key={1}&videos={0}'.format(
vid, ekeys[key[4:]]))
for key in list(sources.keys())
if key[4:] in ekeys.keys()]
source = helpers.pick_source(sorted(sources, reverse=True))
if 'extra_key' in source:
source = source.replace('https://',
'https://{0}/'.format(server))
return source + helpers.append_headers(headers)
raise ResolverError('No playable video found.')
def send_confirm_notification(self, queue, subscription, conf,
project=None, expires=None,
api_version=None, is_unsubscribed=False):
# NOTE(flwang): If the confirmation feature isn't enabled, just do
# nothing. Here we're getting the require_confirmation from conf
# object instead of using self.require_confirmation, because the
# variable from self object really depends on the kwargs when
# initializing the NotifierDriver object. See bug 1655812 for more
# information.
if not conf.notification.require_confirmation:
return
key = conf.signed_url.secret_key
if not key:
LOG.error("Can't send confirm notification due to the value of"
" secret_key option is None")
return
url = "/%s/queues/%s/subscriptions/%s/confirm" % (api_version, queue,
subscription['id'])
pre_url = urls.create_signed_url(key, [url], project=project,
expires=expires, methods=['PUT'])
message = None
if is_unsubscribed:
message_type = MessageType.UnsubscribeConfirmation.name
message = ('You have unsubscribed successfully to the queue: %s, '
'you can resubscribe it by using confirmed=True.'
% queue)
else:
message_type = MessageType.SubscriptionConfirmation.name
message = 'You have chosen to subscribe to the queue: %s' % queue
messages = {}
endpoint_dict = auth.get_public_endpoint()
if endpoint_dict:
wsgi_endpoint = endpoint_dict.get('zaqar')
if wsgi_endpoint:
wsgi_subscribe_url = urllib_parse.urljoin(
wsgi_endpoint, url)
messages['WSGISubscribeURL'] = wsgi_subscribe_url
websocket_endpoint = endpoint_dict.get('zaqar-websocket')
if websocket_endpoint:
websocket_subscribe_url = urllib_parse.urljoin(
websocket_endpoint, url)
messages['WebSocketSubscribeURL'] = websocket_subscribe_url
messages.update({'Message_Type': message_type,
'Message': message,
'URL-Signature': pre_url['signature'],
'URL-Methods': pre_url['methods'][0],
'URL-Paths': pre_url['paths'][0],
'X-Project-ID': pre_url['project'],
'URL-Expires': pre_url['expires'],
'SubscribeBody': {'confirmed': True},
'UnsubscribeBody': {'confirmed': False}})
s_type = urllib_parse.urlparse(subscription['subscriber']).scheme
LOG.info('Begin to send %(type)s confirm/unsubscribe notification.'
' The request body is %(messages)s',
{'type': s_type, 'messages': messages})
self._execute(s_type, subscription, [messages], conf)
def edit(self):
'''Generate Edit template for client from a Job object'''
f = Response()
f.entries = [
FormEntry(name='name',
type='text',
value=self.name,
label='Name',
placeholder='Name for Job...',
required=True),
FormEntry(name='id',
type='text',
value=self.id,
label='Id',
hidden=True),
FormEntry(name='notebook name',
type='text',
value=self.meta.notebook.name,
label='Notebook',
required=True,
readonly=True),
FormEntry(name='notebook',
type='text',
value=self.meta.notebook.id,
required=True,
readonly=True,
hidden=True),
FormEntry(name='starttime',
value=self.meta.start_time.strftime('%Y-%m-%dT%H:%M'),
type='datetime',
label='Start Time/Date',
required=True),
FormEntry(name='interval',
type='select',
value=self.meta.interval,
label='Interval',
options=_INTERVAL_TYPES,
required=True),
FormEntry(name='level',
type='select',
value=self.meta.level,
label='Level',
options=_SERVICE_LEVELS,
required=True),
FormEntry(name='reports',
type='text',
value=str(self.meta.reports),
readonly=True),
FormEntry(name='save',
type='submit',
value='save',
url=urljoin(self.config.apiurl, 'jobs?action=save')),
FormEntry(name='delete',
type='submit',
value='delete',
url=urljoin(self.config.apiurl, 'jobs?action=delete'))
]
return f.to_json()
def download_get_basefiles(self, url):
done = False
pagecnt = 1
# existing_cnt = 0
while not done:
self.log.info('Result page #%s (%s)' % (pagecnt, url))
resp = requests.get(url)
mainsoup = BeautifulSoup(resp.text)
for link in mainsoup.find_all(href=re.compile("/sb/d/108/a/")):
desc = link.find_next_sibling("span", "info").get_text(strip=True)
tmpurl = urljoin(url, link['href'])
# use a strict regex first, then a more forgiving
m = self.re_basefile_strict.search(desc)
if not m:
m = self.re_basefile_lax.search(desc)
if not m:
self.log.warning(
"Can't find Document ID from %s, forced to download doc page" % desc)
resp = requests.get(tmpurl)
subsoup = BeautifulSoup(resp.text)
for a in subsoup.find("div", "doc").find_all("li", "pdf"):
text = a.get_text(strip=True)
m = self.re_basefile_lax.search(text)
if m:
break
else:
self.log.error("Cannot possibly find docid for %s" % tmpurl)
continue
else:
self.log.warning(
"%s (%s) not using preferred form: '%s'" % (m.group(1), tmpurl, m.group(0)))
basefile = m.group(1)
# Extra checking -- sometimes ids like 2003/2004:45
# are used (should be 2003/04:45)
if (":" in basefile and "/" in basefile):
(y1, y2, o) = re.split("[:/]", basefile)
# 1999/2000:45 is a special case
if len(y2) == 4 and y1 != "1999":
self.log.warning(
"%s (%s) using incorrect year format, should be '%s/%s:%s'" %
(basefile, tmpurl, y1, y2[2:], o))
basefile = "%s/%s:%s" % (y1, y2[2:], o)
yield basefile, urljoin(url, link['href'])
pagecnt += 1
next = mainsoup.find("a", text="Nästa sida")
if next:
url = urljoin(url, next['href'])
else:
done = True
def download_from_atom(self):
refresh = self.config.force
feed_url = self.start_url
ns = 'http://www.w3.org/2005/Atom'
done = False
biggraph = Graph()
biggraph.bind("dct", self.ns['dct'])
biggraph.bind("rpubl", self.ns['rpubl'])
while not done:
self.log.info("Feed: %s" % feed_url)
tree = etree.parse(requests.get(feed_url).text)
for entry in tree.findall('{%s}entry' % (ns)):
try:
self.log.info(" Examining entry")
rdf_url = None
for node in entry:
if (node.tag == "{%s}link" % ns and
node.get('type') == 'application/rdf+xml'):
rdf_url = urljoin(feed_url, node.get("href"))
elif (node.tag == "{%s}content" % ns and
node.get('type') == 'application/rdf+xml'):
rdf_url = urljoin(feed_url, node.get("src"))
if rdf_url:
self.log.info(" RDF: %s" % rdf_url)
g = Graph()
g.parse(requests.get(rdf_url).text)
for triple in g:
s, p, o = triple
if (not isinstance(o, URIRef) or
not str(o).startswith(self.config.url)):
g.remove(triple)
self.log.debug(" Adding %s triples" % len(g))
biggraph += g
except KeyboardInterrupt:
raise
except:
e = sys.exc_info()[1]
self.log.error("ERROR: %s" % e)
done = True
for link in list(tree.findall('{%s}link' % (ns))):
self.log.info(" Examining link")
if link.get('rel') == 'prev-archive':
feed_url = urljoin(feed_url, link.get("href"))
done = False
# done = True
self.log.info("Done downloading")
with self.store.open_downloaded("biggraph", "wb") as fp:
fp.write(biggraph.serialize(format="nt"))
def _update_tags(self):
""" Synchronize tags with Bleemeo API
"""
current_tags = set(self.core.config.get('tags', []))
old_tags = set(self.core.state.get('tags_uuid', {}))
if current_tags == old_tags:
return
response = requests.get(
urllib_parse.urljoin(self.bleemeo_base_url,
'/v1/agent/%s/' % self.agent_uuid),
params={'fields': 'tags'},
auth=(self.agent_username, self.agent_password),
headers={
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': self.core.http_user_agent,
},
)
if response.status_code != 200:
logging.debug('Fetching current agent tags failed: %s',
response.content)
return
current_api_tags = set(x['name'] for x in response.json()['tags'])
deleted_tags = (old_tags - current_tags)
tags = (current_api_tags - deleted_tags).union(current_tags)
response = requests.patch(
urllib_parse.urljoin(self.bleemeo_base_url,
'/v1/agent/%s/' % self.agent_uuid),
data=json.dumps({'tags': [{
'name': x
} for x in tags]}),
auth=(self.agent_username, self.agent_password),
headers={
'X-Requested-With': 'XMLHttpRequest',
'Content-type': 'application/json',
'User-Agent': self.core.http_user_agent,
},
)
if response.status_code > 400:
logging.debug('Updating current agent tags failed: %s',
response.content)
return
tags_uuid = {}
for tag in response.json()['tags']:
if tag['name'] in current_tags:
tags_uuid[tag['name']] = tag['id']
self.core.state.set('tags_uuid', tags_uuid)
def form(self):
'''Generate Form template for client from a Report object'''
f = Response()
f.entries = [
FormEntry(name='name',
type='text',
label='Name',
placeholder='Name for Report...',
required=True),
FormEntry(name='notebook',
type='autocomplete',
label='Notebook',
url=urljoin(self.config.apiurl,
'autocomplete?type=notebooks&partial='),
required=True),
FormEntry(name='job',
type='autocomplete',
label='Job',
url=urljoin(self.config.apiurl,
'autocomplete?type=jobs&partial='),
required=True),
FormEntry(name='parameters',
type='textarea',
label='Parameters',
placeholder='JSON Parameters...'),
FormEntry(name='type',
type='select',
label='Type',
options=_REPORT_TYPES,
required=True),
FormEntry(name='output',
type='select',
label='Output',
options=_OUTPUT_TYPES,
required=True),
FormEntry(name='code',
type='select',
label='Strip Code',
options=['yes', 'no'],
required=True),
FormEntry(name='template',
type='text',
label='Template',
required=False),
FormEntry(name='submit',
type='submit',
value='save',
url=urljoin(self.config.apiurl, 'reports?action=save'))
]
return f.to_json()
def download_resource(self, resource_location, location):
"""Download the resource in the specified location
:param resource_script:
Is relative to the /argus/resources/ directory.
:param location:
The location on the instance.
"""
base_resource = CONFIG.argus.resources
if not base_resource.endswith("/"):
base_resource = urlparse.urljoin(CONFIG.argus.resources,
"resources/")
uri = urlparse.urljoin(base_resource, resource_location)
self.download(uri, location)
def download_resource(self, resource_location, location):
"""Download the resource in the specified location
:param resource_script:
Is relative to the /argus/resources/ directory.
:param location:
The location on the instance.
"""
base_resource = CONFIG.argus.resources
if not base_resource.endswith("/"):
base_resource = urlparse.urljoin(CONFIG.argus.resources,
"resources/")
uri = urlparse.urljoin(base_resource, resource_location)
self.download(uri, location)
def __getTrakt(url, post=None):
try:
url = urllib_parse.urljoin(BASE_URL, url)
post = json.dumps(post) if post else None
headers = {'Content-Type': 'application/json', 'trakt-api-key': V2_API_KEY, 'trakt-api-version': 2}
if getTraktCredentialsInfo():
headers.update({'Authorization': 'Bearer %s' % control.addon('plugin.video.koditvr').getSetting('trakt.token')})
result = client.request(url, post=post, headers=headers, output='extended', error=True)
result = utils.byteify(result)
resp_code = result[1]
resp_header = result[2]
result = result[0]
if resp_code in ['423', '500', '502', '503', '504', '520', '521', '522', '524']:
log_utils.log('Trakt Error: %s' % str(resp_code))
control.infoDialog('Trakt Error: ' + str(resp_code), sound=True)
return
elif resp_code in ['429']:
log_utils.log('Trakt Rate Limit Reached: %s' % str(resp_code))
control.infoDialog('Trakt Rate Limit Reached: ' + str(resp_code), sound=True)
return
elif resp_code in ['404']:
log_utils.log('Object Not Found : %s' % str(resp_code))
return
if resp_code not in ['401', '405']:
return result, resp_header
koditvr = urllib_parse.urljoin(BASE_URL, '/koditvr/token')
opost = {'client_id': V2_API_KEY, 'client_secret': CLIENT_SECRET, 'redirect_uri': REDIRECT_URI, 'grant_type': 'refresh_token', 'refresh_token': control.addon('plugin.video.koditvr').getSetting('trakt.refresh')}
result = client.request(koditvr, post=json.dumps(opost), headers=headers)
result = utils.json_loads_as_str(result)
token, refresh = result['access_token'], result['refresh_token']
print('Info - ' + str(token))
control.addon('plugin.video.koditvr').setSetting(id='trakt.token', value=token)
control.addon('plugin.video.koditvr').setSetting(id='trakt.refresh', value=refresh)
headers['Authorization'] = 'Bearer %s' % token
result = client.request(url, post=post, headers=headers, output='extended', error=True)
result = utils.byteify(result)
return result[0], result[2]
except:
log_utils.log('getTrakt Error', 1)
pass
def paste(self, s):
"""Upload to pastebin via json interface."""
url = urljoin(self.url, '/json/new')
payload = {
'code': s,
'lexer': 'pycon',
'expiry': self.expiry
}
try:
response = requests.post(url, data=payload, verify=True)
response.raise_for_status()
except requests.exceptions.RequestException as exc:
raise PasteFailed(exc.message)
data = response.json()
paste_url_template = Template(self.show_url)
paste_id = urlquote(data['paste_id'])
paste_url = paste_url_template.safe_substitute(paste_id=paste_id)
removal_url_template = Template(self.removal_url)
removal_id = urlquote(data['removal_id'])
removal_url = removal_url_template.safe_substitute(
removal_id=removal_id)
return (paste_url, removal_url)
def _get_application(self, app_id):
url = urllib_parse.urljoin(self._api_url + 'applications/', str(app_id) + '.json')
resp = requests.get(url, headers=self._headers).json()
if 'application' in resp:
# pick 1st application
return resp['application'] if resp['application'] else None
return None
def get_api_data(self, target_url=None, query_params=None):
"""retrieves the Jenkins API specific data from the specified URL
:param str target_url:
Full URL to the REST API endpoint to be queried. If not provided,
data will be loaded from the default 'url' for this object
:param str query_params:
optional set of query parameters to customize the returned data
:returns:
The set of Jenkins attributes, converted to Python objects,
associated with the given URL.
:rtype: :class:`dict`
"""
if target_url is None:
target_url = self.url
temp_url = urllib_parse.urljoin(target_url, "api/json")
if query_params is not None:
# TODO: Update this to pass 'params' key to get method
temp_url += "?" + query_params
req = requests.get(
temp_url,
auth=self._creds,
verify=self._ssl_cert)
req.raise_for_status()
retval = req.json()
self._log.debug(json.dumps(retval, indent=4))
return retval
def _register_services(self):
""" Check for any unregistered services and register them
Also check for changed services and update them
"""
base_url = self.bleemeo_base_url
registration_url = urllib_parse.urljoin(base_url, '/v1/service/')
for key, service_info in self.core.services.items():
(service_name, instance) = key
entry = {
'listen_addresses':
get_listen_addresses(service_info),
'label': service_name,
'exe_path': service_info.get('exe_path', ''),
}
if instance is not None:
entry['instance'] = instance
if key in self.services_uuid:
entry['uuid'] = self.services_uuid[key]['uuid']
# check for possible update
if self.services_uuid[key] == entry:
continue
method = requests.put
service_uuid = self.services_uuid[key]['uuid']
url = registration_url + str(service_uuid) + '/'
expected_code = 200
else:
method = requests.post
url = registration_url
expected_code = 201
payload = entry.copy()
payload.update({
'account': self.account_id,
'agent': self.agent_uuid,
})
response = method(
url,
data=json.dumps(payload),
auth=(self.agent_username, self.agent_password),
headers={
'X-Requested-With': 'XMLHttpRequest',
'Content-type': 'application/json',
},
)
if response.status_code != expected_code:
logging.debug(
'Service registration failed. Server response = %s',
response.content
)
continue
entry['uuid'] = response.json()['id']
self.services_uuid[key] = entry
self.core.state.set_complex_dict(
'services_uuid', self.services_uuid
)
def _ComputePaths(package, version, discovery_doc):
full_path = urllib_parse.urljoin(discovery_doc["rootUrl"], discovery_doc["servicePath"])
api_path_component = "/".join((package, version, ""))
if api_path_component not in full_path:
return full_path, ""
prefix, _, suffix = full_path.rpartition(api_path_component)
return prefix + api_path_component, suffix
def make_fasta_dna_url(
ensembl_release,
species,
contig,
server=ENSEMBL_FTP_SERVER):
"""
Construct URL to FASTA file with full sequence of a particular chromosome.
Returns server_url/subdir and filename as tuple result.
"""
ensembl_release, species, reference_name = _normalize_release_properties(
ensembl_release, species)
subdir = _species_subdir(
ensembl_release,
species=species,
filetype="fasta",
server=server,)
server_subdir = urllib_parse.urljoin(server, subdir)
server_sequence_subdir = join(server_subdir, "dna")
filename = FASTA_DNA_CHROMOSOME_FILENAME_TEMPLATE % {
"Species": species.capitalize(),
"reference": reference_name,
"release": ensembl_release,
"sequence_type": "dna",
"contig": contig
}
return join(server_sequence_subdir, filename)
def notify_new_email(email, user):
""" Ask the user to confirm to the email belong to them.
"""
root_url = pagure_config.get("APP_URL", flask.request.url_root)
url = urljoin(
root_url or flask.request.url_root,
flask.url_for("ui_ns.confirm_email", token=email.token),
)
text = """Dear %(username)s,
You have registered a new email on pagure at %(root_url)s.
To finish your validate this registration, please click on the following
link or copy/paste it in your browser, this link will remain valid only 2 days:
%(url)s
The email will not be activated until you finish this step.
Sincerely,
Your pagure admin.
""" % (
{"username": user.username, "url": url, "root_url": root_url}
)
send_email(
text,
"Confirm new email",
email.email,
user_from=user.fullname or user.user,
)
def urls_from_urlset_or_sitemapindex(response):
""" Yields URLs from ``<urlset>`` or ``<sitemapindex>`` elements as per
`sitemaps.org <http://www.sitemaps.org/protocol.html>`_.
"""
sitemap = URL(response.url).fragment_dict.get('sitemap')
content_subtypes = response.headers.get_content_subtype().split('+')
if not sitemap and not 'xml' in content_subtypes:
return
root = None
for _, elem in iterparse(decode(response)):
if root is None:
root = elem.getroottree().getroot()
if not (root.tag.endswith('}sitemapindex') or
root.tag.endswith('}urlset')):
# root element has wrong tag - give up
break
if elem.tag.endswith('}loc') and elem.text is not None:
text = elem.text.strip()
if text:
# http://www.sitemaps.org/protocol.html#locdef
url = URL(urljoin(response.url, text))
if elem.getparent().tag.endswith('}sitemap'):
# set sitemap=True to help downstream processing
url = url.update_fragment_dict(sitemap=True)
yield "url", url
if elem.getparent() is root:
# release memory for previous elements
while elem.getprevious() is not None:
del root[0]
def _generate_client_conf():
auth_strategy = os.environ.get('OS_AUTH_STRATEGY', 'noauth')
if auth_strategy == 'keystone':
args = _get_credential_args()
# FIXME(flwang): Now we're hardcode the keystone auth version, since
# there is a 'bug' with the osc-config which is returning the auth_url
# without version. This should be fixed as long as the bug is fixed.
parsed_url = urllib_parse.urlparse(args['auth_url'])
auth_url = args['auth_url']
if not parsed_url.path or parsed_url.path == '/':
auth_url = urllib_parse.urljoin(args['auth_url'], 'v2.0')
conf = {
'auth_opts': {
'backend': 'keystone',
'options': {
'os_username': args['username'],
'os_password': args['password'],
'os_project_name': args['project_name'],
'os_auth_url': auth_url,
'insecure': '',
},
},
}
else:
conf = {
'auth_opts': {
'backend': 'noauth',
'options': {
'os_project_id': 'my-lovely-benchmark',
},
},
}
print("Using '{0}' authentication method".format(conf['auth_opts']
['backend']))
return conf
def verify_proxy_ticket(ticket, service):
"""Verifies CAS 2.0+ XML-based proxy ticket.
Returns username on success and None on failure.
"""
try:
from xml.etree import ElementTree
except ImportError:
from elementtree import ElementTree
params = {'ticket': ticket, 'service': service}
url = (urljoin(settings.CAS_SERVER_URL, 'proxyValidate') + '?' +
urlencode(params))
page = urlopen(url)
try:
response = page.read()
tree = ElementTree.fromstring(response)
if tree[0].tag.endswith('authenticationSuccess'):
username = tree[0][0].text
proxies = []
if len(tree[0]) > 1:
for element in tree[0][1]:
proxies.append(element.text)
return {"username": username, "proxies": proxies}, None
else:
return None, None
finally:
page.close()
def get_file_urls(self, name):
query = (
select([
release_files.c.name,
release_files.c.filename,
release_files.c.python_version,
release_files.c.md5_digest,
])
.where(release_files.c.name == name)
.order_by(release_files.c.filename.desc())
)
with self.engine.connect() as conn:
results = conn.execute(query)
return [
FileURL(
filename=r["filename"],
url=urllib_parse.urljoin(
"/".join([
"../../packages",
r["python_version"],
r["name"][0],
r["name"],
r["filename"],
]),
"#md5={}".format(r["md5_digest"]),
),
)
for r in results
]
def remove_user_from_uri(apps, schema_editor):
Formula = apps.get_model('formulas', 'Formula')
for formula in Formula.objects.all():
url_bits = urlsplit(formula.uri)
# don't do it if it's an ssh formula
if 'ssh' in url_bits.scheme:
continue
if url_bits.username:
formula.git_username = url_bits.username
if url_bits.port:
new_netloc = '{}:{}'.format(url_bits.hostname, url_bits.port)
else:
new_netloc = url_bits.hostname
formula.uri = urljoin((
url_bits.scheme,
new_netloc,
url_bits.path,
url_bits.query,
url_bits.fragment,
))
formula.save()
def build_uri(self, base, matches):
if not base:
return None
if self.uriTemplate:
expanded = str(self.uriTemplate)
elif self.fragmentTemplate:
if "#" in base:
base += self.space.fragmentSeparator
else:
base += "#"
expanded = base + str(self.fragmentTemplate)
else:
return None
expanded = expanded.replace("{+base}", base)
for var, value in matches.items():
slug = self.transform_value(value)
expanded = expanded.replace("{%s}" % var, slug)
# if base is eg "http://localhost/res/" and expanded is a
# /-prefixed relative uri like "/sfs/9999:998", urljoin
# results in "http://localhost/sfs/9999:998/", not
# "http://localhost/res/" like you'd expect. So we work
# around.
if expanded[0] == "/":
expanded = expanded[1:]
if expanded.startswith("http://") or expanded.startswith("https://"):
return urljoin(base, expanded)
else:
# see the test integrationLegalURI.CustomCoinstruct.test_1845_50_s.1
return "%s/%s" % (base, expanded)
def rename(self, new_job_name):
"""Changes the name of this job
:param str new_job_name:
new name to assign to this job
"""
args = {
"params": {
"newName": new_job_name
}
}
self._api.post(self._api.url + "doRename", args=args)
# NOTE: In order to properly support jobs that may contain nested
# jobs we have to do some URL manipulations to extrapolate the
# REST API endpoint for the parent object to which the cloned
# view is to be contained.
parts = urllib_parse.urlsplit(self._api.url).path.split("/")
parts = [cur_part for cur_part in parts if cur_part.strip()]
assert len(parts) >= 2
assert parts[-2] == "job"
new_url = urllib_parse.urljoin(
self._api.url, "/" + "/".join(parts[:-2]))
new_url += "/job/" + new_job_name
self._api = self._api.clone(new_url)
assert self.name == new_job_name
def get_proxy_ticket_for(self, service):
"""Verifies CAS 2.0+ XML-based authentication ticket.
Returns username on success and None on failure.
"""
if not settings.CAS_PROXY_CALLBACK:
raise CasConfigException("No proxy callback set in settings")
try:
from xml.etree import ElementTree
except ImportError:
from elementtree import ElementTree
params = {'pgt': self.tgt, 'targetService': service}
url = (urljoin(settings.CAS_SERVER_URL, 'proxy') + '?' +
urlencode(params))
page = urlopen(url)
try:
response = page.read()
tree = ElementTree.fromstring(response)
if tree[0].tag.endswith('proxySuccess'):
return tree[0][0].text
else:
raise CasTicketException("Failed to get proxy ticket")
finally:
page.close()
def static_url(app, filename):
"""
static_url('css/bootstrap.css')
"""
static_dir = os.path.join(
os.path.dirname(os.path.abspath(warehouse.__file__)),
"static",
"compiled",
)
filepath = os.path.join(static_dir, filename)
manifest_path = os.path.join(os.path.dirname(filepath), ".manifest.json")
if not app.config.debug:
# Load our on disk manifest
with open(manifest_path) as fp:
manifest = json.load(fp)
# Get the base name for this file
basename = manifest.get(os.path.basename(filename))
# If we were able to get a base name, then create a filename with it
if basename is not None:
filename = os.path.join(os.path.dirname(filename), basename)
return urllib_parse.urljoin("/static/", filename)
def get_eureka_urls(self):
if self.eureka_url:
return [self.eureka_url]
elif self.use_dns:
zone_dns_map = self.get_zones_from_dns()
zones = zone_dns_map.keys()
assert len(zones) > 0, "No availability zones found for, please add them explicitly"
if self.prefer_same_zone:
if self.get_instance_zone() in zones:
zones = [zones.pop(zones.index(self.get_instance_zone()))] + zones # Add our zone as the first element
else:
logger.warn("No match for the zone %s in the list of available zones %s" % (
self.get_instance_zone(), zones)
)
service_urls = []
for zone in zones:
eureka_instances = zone_dns_map[zone]
random.shuffle(eureka_instances) # Shuffle order for load balancing
for eureka_instance in eureka_instances:
server_uri = "http://%s" % eureka_instance
if self.eureka_port:
server_uri += ":%s" % self.eureka_port
eureka_instance_url = urljoin(server_uri, self.context, "/")
if not eureka_instance_url.endswith("/"):
eureka_instance_url = "%s/" % eureka_instance_url
service_urls.append(eureka_instance_url)
primary_server = service_urls.pop(0)
random.shuffle(service_urls)
service_urls.insert(0, primary_server)
logger.info("This client will talk to the following serviceUrls in order: %s" % service_urls)
return service_urls
def reportIncorrectImage(self, task_id):
request = {"clientKey": self.client_key,
"taskId": task_id
}
response = self.session.post(urljoin(self.base_url, self.REPORT_IMAGE_URL), json=request).json()
self._check_response(response)
return response.get('status', False) != False
def get_headers(self, path=None):
"""gets the HTTP header attributes from a Jenkins URL
:param str path:
optional extension path to append to the root
URL managed by this object when performing the
get operation
:returns: dictionary of HTTP header attributes with their associated values
:rtype: :class:`dict`
"""
temp_path = self._url
if path is not None:
temp_path = urllib_parse.urljoin(temp_path, path.lstrip("/\\"))
if temp_path in DataRequester._header_cache:
return DataRequester._header_cache[temp_path]
self._log.debug("Header cache miss: " + temp_path)
req = requests.get(temp_path, auth=self._credentials)
if req.status_code != 200:
req.raise_for_status()
if DataRequester.ENABLE_CACHING:
DataRequester._header_cache[temp_path] = req.headers
return req.headers
def get_api_data(self, query_params=None):
"""Convenience method that retrieves the Jenkins API specific data from the specified URL
:param str query_params: optional set of query parameters to customize the returned data
:returns:
The set of Jenkins attributes, converted to Python objects, associated
with the given URL.
:rtype: :class:`object`
"""
cache_url = urllib_parse.urljoin(self._url, "api/python")
if cache_url in DataRequester._api_data_cache:
# NOTE: The assumption here is that the query parameters have no impact on the cached data
# while this may not be true in general, for most use cases intended to be supported this
# shouldn't be an issue
return DataRequester._api_data_cache[cache_url]
temp_url = cache_url
if query_params is not None:
temp_url += "?" + query_params
txt = self._get_raw_text(temp_url)
retval = eval(txt) # pylint: disable=eval-used
if DataRequester.ENABLE_CACHING:
DataRequester._api_data_cache[cache_url] = retval
return retval
def download_www(self, dirname, recurse):
url = 'https://lagen.nu/dv/downloaded/%s' % dirname
self.log.debug('Listing contents of %s' % url)
resp = requests.get(url)
iterlinks = lxml.html.document_fromstring(resp.text).iterlinks()
for element, attribute, link, pos in iterlinks:
if link.startswith("/"):
continue
elif link.endswith("/") and recurse:
self.download_www(link, recurse)
elif link.endswith(".zip"):
basefile = os.path.splitext(link)[0]
if dirname:
basefile = dirname + basefile
localpath = self.store.downloaded_path(basefile)
if os.path.exists(localpath) and not self.config.force:
pass # we already got this
else:
absolute_url = urljoin(url, link)
self.log.debug('Fetching %s to %s' % (link, localpath))
resp = requests.get(absolute_url)
with self.store.open_downloaded(basefile, "wb") as fp:
fp.write(resp.content)
self.process_zipfile(localpath)
def _ComputePaths(package, version, discovery_doc):
full_path = urllib_parse.urljoin(
discovery_doc['rootUrl'], discovery_doc['servicePath'])
api_path_component = '/'.join((package, version, ''))
if api_path_component not in full_path:
return full_path, ''
prefix, _, suffix = full_path.rpartition(api_path_component)
return prefix + api_path_component, suffix
def build_absolute_uri(self, location=None):
if location is None:
return None
bits = urlsplit(location)
if not (bits.scheme and bits.netloc):
location = urljoin(self.prod_url, location)
return iri_to_uri(location)
def get_cp_url(client, project_id, section="dashboard"):
project_data = client.get_project(project_id)
url = project_data["dashboard_url"]
if section != "dashboard":
url = urljoin(url, section)
return url
def checkins(self, ixbug):
'''The API does not provide a call for this.'''
kilnhg_url = self._fburl.replace('.fogbugz.', '.kilnhg.')
base_url = urljoin(kilnhg_url, '/fogbugz/casecheckins/{}?token={}')
url = base_url.format(ixbug, self.current_token)
r = requests.get(url)
r.raise_for_status()
return r.json()
