def json_req(self, url, http_post=False, skip_auth=False, raw=False,
**kwargs):
'''
Performs JSON request.
'''
# Encode params
if len(kwargs) > 0:
params = urlencode(
{key: val.encode('utf-8') for key, val in kwargs.items()}
)
else:
params = ''
# Store for exception handling
self.request_url = url
self.request_params = params
# Append parameters
if len(params) > 0 and not http_post:
url = '?'.join((url, params))
# Create request object with custom headers
request = Request(url)
request.timeout = 0.5
request.add_header('User-Agent', USER_AGENT)
# Optional authentication
if not skip_auth:
self.authenticate(request)
# Fire request
if http_post:
handle = urlopen(request, params.encode('utf-8'))
else:
handle = urlopen(request)
# Read and possibly convert response
text = handle.read()
# Needed for Microsoft
if text[:3] == b'\xef\xbb\xbf':
text = text.decode('UTF-8-sig')
else:
text = text.decode('utf-8')
# Replace literal \t
text = text.strip().replace(
'\t', '\\t'
).replace(
'\r', '\\r'
)
# Needed for Google
while ',,' in text or '[,' in text:
text = text.replace(',,', ',null,').replace('[,', '[')
if raw:
return text
# Parse JSON
response = json.loads(text)
# Return data
return response
def download_from_repository(self, repo_source, target):
"""
Download given source file from the repository and store
it as target file
The repo_source location is used relative to the repository
location and will be part of a mime type source like:
file://repo_path/repo_source
:param string source: source file in the repo
:param string target: file path
"""
try:
request = Request(
os.sep.join([self._get_mime_typed_uri(), repo_source])
)
if self.user and self.secret:
credentials = b64encode(
format(':'.join([self.user, self.secret])).encode()
)
request.add_header(
'Authorization', b'Basic ' + credentials
)
location = urlopen(request)
except Exception as e:
raise KiwiUriOpenError(
'{0}: {1}'.format(type(e).__name__, format(e))
)
with open(target, 'wb') as target_file:
target_file.write(location.read())
def _notify_emby(self, message, host=None, emby_apikey=None):
"""Handles notifying Emby host via HTTP API
Returns:
Returns True for no issue or False if there was an error
"""
# fill in omitted parameters
if not host:
host = sickbeard.EMBY_HOST
if not emby_apikey:
emby_apikey = sickbeard.EMBY_APIKEY
url = 'http://%s/emby/Notifications/Admin' % host
values = {'Name': 'Medusa', 'Description': message, 'ImageUrl': sickbeard.LOGO_URL}
data = json.dumps(values)
try:
req = Request(url, data)
req.add_header('X-MediaBrowser-Token', emby_apikey)
req.add_header('Content-Type', 'application/json')
response = urlopen(req)
result = response.read()
response.close()
logger.log(u'EMBY: HTTP response: ' + result.replace('\n', ''), logger.DEBUG)
return True
except (URLError, IOError) as e:
logger.log(u'EMBY: Warning: Couldn\'t contact Emby at ' + url + ' ' + ex(e), logger.WARNING)
return False
def test_extract_macaroons_from_request(self):
def encode_macaroon(m):
macaroons = '[' + utils.macaroon_to_json_string(m) + ']'
return base64.urlsafe_b64encode(utils.to_bytes(macaroons)).decode('ascii')
req = Request('http://example.com')
m1 = pymacaroons.Macaroon(version=pymacaroons.MACAROON_V2, identifier='one')
req.add_header('Macaroons', encode_macaroon(m1))
m2 = pymacaroons.Macaroon(version=pymacaroons.MACAROON_V2, identifier='two')
jar = requests.cookies.RequestsCookieJar()
jar.set_cookie(utils.cookie(
name='macaroon-auth',
value=encode_macaroon(m2),
url='http://example.com',
))
jar.set_cookie(utils.cookie(
name='macaroon-empty',
value='',
url='http://example.com',
))
jar.add_cookie_header(req)
macaroons = httpbakery.extract_macaroons(req)
self.assertEquals(len(macaroons), 2)
macaroons.sort(key=lambda ms: ms[0].identifier)
self.assertEquals(macaroons[0][0].identifier, m1.identifier)
self.assertEquals(macaroons[1][0].identifier, m2.identifier)
def download(baseurl, parameters={}, headers={}):
"""Download Data from an url and returns it as a String
@param baseurl Url to download from (e.g. http://www.google.com)
@param parameters Parameter dict to be encoded with url
@param headers Headers dict to pass with Request
@returns String of data from URL
"""
url = "?".join([baseurl, urlencode(parameters)])
log.debug("Downloading: " + url)
data = ""
for _ in range(MAX_RETRIES):
try:
req = Request(url, headers=headers)
req.add_header(USER_AGENT, USER_AGENT_STRING)
response = urlopen(req)
if six.PY2:
data = response.read()
else:
data = response.read().decode("utf-8")
response.close()
break
except Exception as err:
if not isinstance(err, URLError):
log.debug("Error %s during HTTP Request, abort", repr(err))
raise # propagate non-URLError
log.debug("Error %s during HTTP Request, retrying", repr(err))
else:
raise
return data
def get_version(self):
"""Get the version of this Master.
:returns: This master's version number ``str``
Example::
>>> j = Jenkins()
>>> info = j.get_version()
>>> print info
>>> 1.541
"""
try:
request = Request(self.server + "/login")
request.add_header('X-Jenkins', '0.0')
response = urlopen(request, timeout=self.timeout)
if response is None:
raise EmptyResponseException(
"Error communicating with server[%s]: "
"empty response" % self.server)
if six.PY2:
return response.info().getheader('X-Jenkins')
if six.PY3:
return response.getheader('X-Jenkins')
except (HTTPError, BadStatusLine):
raise BadHTTPException("Error communicating with server[%s]"
% self.server)
def _request_with_auth(url, username, password):
request = Request(url)
base64string = base64.b64encode(
username.encode('ascii') + b':' + password.encode('ascii')
)
request.add_header(b"Authorization", b"Basic " + base64string)
return urlopen(request)
def execute(cls, uri, http_verb, extra_headers=None, batch=False, _body=None, **kw):
"""
if batch == False, execute a command with the given parameters and
return the response JSON.
If batch == True, return the dictionary that would be used in a batch
command.
"""
if batch:
urlsplitter = urlparse(API_ROOT).netloc
ret = {"method": http_verb, "path": uri.split(urlsplitter, 1)[1]}
if kw:
ret["body"] = kw
return ret
if not ('app_id' in ACCESS_KEYS and 'rest_key' in ACCESS_KEYS):
raise core.ParseError('Missing connection credentials')
app_id = ACCESS_KEYS.get('app_id')
rest_key = ACCESS_KEYS.get('rest_key')
master_key = ACCESS_KEYS.get('master_key')
url = uri if uri.startswith(API_ROOT) else cls.ENDPOINT_ROOT + uri
if _body is None:
data = kw and json.dumps(kw, default=date_handler) or "{}"
else:
data = _body
if http_verb == 'GET' and data:
url += '?%s' % urlencode(kw)
data = None
else:
data = data
headers = {
'Content-type': 'application/json',
'X-Parse-Application-Id': app_id,
'X-Parse-REST-API-Key': rest_key
}
headers.update(extra_headers or {})
request = Request(url.encode('utf-8'), data, headers)
if ACCESS_KEYS.get('session_token'):
request.add_header('X-Parse-Session-Token', ACCESS_KEYS.get('session_token'))
elif master_key:
request.add_header('X-Parse-Master-Key', master_key)
request.get_method = lambda: http_verb
try:
response = urlopen(request, timeout=CONNECTION_TIMEOUT)
except HTTPError as e:
exc = {
400: core.ResourceRequestBadRequest,
401: core.ResourceRequestLoginRequired,
403: core.ResourceRequestForbidden,
404: core.ResourceRequestNotFound
}.get(e.code, core.ParseError)
raise exc(e.read())
return json.loads(response.read().decode('utf-8'))
def _query_api(self, clip):
# Adapted from SpeechRecognition source code, modified to get text onsets
flac_data = clip.get_flac_data(
convert_rate = None if clip.sample_rate >= 16000 else 16000,
convert_width = None if clip.sample_width >= 2 else 2
)
model = "{0}_BroadbandModel".format("en-US")
url = "https://stream.watsonplatform.net/speech-to-text/api/v1/recognize?{0}".format(urlencode({
"profanity_filter": "false",
"continuous": "true",
"model": model,
"timestamps": "true",
}))
request = Request(url, data = flac_data, headers = {
"Content-Type": "audio/x-flac",
"X-Watson-Learning-Opt-Out": "true",
})
if hasattr("", "encode"): # Python 2.6 compatibility
authorization_value = base64.standard_b64encode("{0}:{1}".format(self.username, self.password).encode("utf-8")).decode("utf-8")
else:
authorization_value = base64.standard_b64encode("{0}:{1}".format(self.username, self.password))
request.add_header("Authorization", "Basic {0}".format(authorization_value))
try:
response = urlopen(request, timeout=None)
except HTTPError as e:
raise Exception("recognition request failed: {0}".format(getattr(e, "reason", "status {0}".format(e.code))))
except URLError as e:
raise Exception("recognition connection failed: {0}".format(e.reason))
response_text = response.read().decode("utf-8")
result = json.loads(response_text)
return result
def download_translations(self, source, language, text, unit, user):
"""Download list of possible translations from a service."""
# should the machine translation service be used?
# (rather than only the term database)
enable_mt = False
if isinstance(settings.MT_SAP_USE_MT, bool):
enable_mt = settings.MT_SAP_USE_MT
# build the json body
request_data_as_bytes = json.dumps(
{
'targetLanguages': [language],
'sourceLanguage': source,
'enableMT': enable_mt,
'enableTranslationQualityEstimation': enable_mt,
'units': [{'value': text}]
},
ensure_ascii=False
).encode('utf-8')
# create the request
translation_url = settings.MT_SAP_BASE_URL + 'translate'
request = Request(
translation_url if six.PY3 else translation_url.encode("utf-8")
)
request.timeout = 0.5
request.add_header('User-Agent', USER_AGENT.encode('utf-8'))
request.add_header('Referer', get_site_url().encode('utf-8'))
request.add_header('Content-Type', 'application/json; charset=utf-8')
request.add_header('Content-Length', len(request_data_as_bytes))
request.add_header('Accept', 'application/json; charset=utf-8')
self.authenticate(request)
# Read and possibly convert response
content = urlopen(
request, request_data_as_bytes
).read().decode('utf-8')
# Replace literal \t
content = content.strip().replace(
'\t', '\\t'
).replace(
'\r', '\\r'
)
response = json.loads(content)
translations = []
# prepare the translations for weblate
for item in response['units']:
for translation in item['translations']:
translations.append((
translation['value'],
translation.get('qualityIndex', 100),
self.name,
text
))
return translations
def _send_to_kodi(command, host=None, username=None, password=None, dest_app="KODI"): # pylint: disable=too-many-arguments
"""Handles communication to KODI servers via HTTP API
Args:
command: Dictionary of field/data pairs, encoded via urllib and passed to the KODI API via HTTP
host: KODI webserver host:port
username: KODI webserver username
password: KODI webserver password
Returns:
Returns response.result for successful commands or False if there was an error
"""
# fill in omitted parameters
if not username:
username = sickbeard.KODI_USERNAME
if not password:
password = sickbeard.KODI_PASSWORD
if not host:
logger.log(u'No %s host passed, aborting update' % dest_app, logger.WARNING)
return False
for key in command:
if isinstance(command[key], text_type):
command[key] = command[key].encode('utf-8')
enc_command = urlencode(command)
logger.log(u"%s encoded API command: %r" % (dest_app, enc_command), logger.DEBUG)
# url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, enc_command) # maybe need for old plex?
url = 'http://%s/kodiCmds/kodiHttp/?%s' % (host, enc_command)
try:
req = Request(url)
# if we have a password, use authentication
if password:
base64string = base64.encodestring('%s:%s' % (username, password))[:-1]
authheader = "Basic %s" % base64string
req.add_header("Authorization", authheader)
logger.log(u"Contacting %s (with auth header) via url: %s" % (dest_app, ss(url)), logger.DEBUG)
else:
logger.log(u"Contacting %s via url: %s" % (dest_app, ss(url)), logger.DEBUG)
try:
response = urlopen(req)
except (BadStatusLine, URLError) as e:
logger.log(u"Couldn't contact %s HTTP at %r : %r" % (dest_app, url, ex(e)), logger.DEBUG)
return False
result = response.read().decode(sickbeard.SYS_ENCODING)
response.close()
logger.log(u"%s HTTP response: %s" % (dest_app, result.replace('\n', '')), logger.DEBUG)
return result
except Exception as e:
logger.log(u"Couldn't contact %s HTTP at %r : %r" % (dest_app, url, ex(e)), logger.DEBUG)
return False
def http_request(method, url, request=None, timeout=30):
"""Perform HTTP request"""
if method == 'POST':
return http_post(url, request, timeout=timeout)
else: # GET
request = Request(url)
request.add_header('User-Agent', 'pycsw (http://pycsw.org/)')
return urlopen(request, timeout=timeout).read()
def post_soap(self, url, xml, soapaction=None):
url = self.opener.relative(url)
request = Request(url, etree.tostring(soap_body(xml)))
request.add_header('Content-type', 'text/xml; charset=utf-8')
if soapaction:
request.add_header('Soapaction', soapaction)
response = self.opener.open(request, timeout=self.timeout)
return etree.parse(response).xpath('/soap:Envelope/soap:Body/*', namespaces=namespaces)[0]
def test_filter_json_post_data(tmpdir):
data = json.dumps({"id": "secret", "foo": "bar"}).encode("utf-8")
request = Request("http://httpbin.org/post", data=data)
request.add_header("Content-Type", "application/json")
cass_file = str(tmpdir.join("filter_jpd.yaml"))
with vcr.use_cassette(cass_file, filter_post_data_parameters=["id"]):
urlopen(request)
with vcr.use_cassette(cass_file, filter_post_data_parameters=["id"]) as cass:
assert b'"id": "secret"' not in cass.requests[0].body
def test_filter_json_post_data(tmpdir):
data = json.dumps({'id': 'secret', 'foo': 'bar'}).encode('utf-8')
request = Request('http://httpbin.org/post', data=data)
request.add_header('Content-Type', 'application/json')
cass_file = str(tmpdir.join('filter_jpd.yaml'))
with vcr.use_cassette(cass_file, filter_post_data_parameters=['id']):
urlopen(request)
with vcr.use_cassette(cass_file, filter_post_data_parameters=['id']) as cass:
assert b'"id": "secret"' not in cass.requests[0].body
def download_avatar_image(user, size):
"""Download avatar image from remote server."""
url = avatar_for_email(user.email, size)
request = Request(url)
request.add_header('User-Agent', USER_AGENT)
# Fire request
handle = urlopen(request, timeout=1.0)
# Read and possibly convert response
return bytes(handle.read())
def submit_request(self, query):
opener = build_opener(HTTPHandler)
data = json.dumps(query.fetch_json()).encode('utf8')
request = Request("http://{}/{}{}".format(
self.admin_addr, query.path, query.fetch_url_params()), data=data)
request.get_method = lambda: 'POST'
request.add_header("Content-Type", 'application/json')
request.add_header("Content-Length", len(data))
connection = opener.open(request)
return connection.read().decode('utf-8')
def soap_req(self, action, **kwargs):
template = get_template(
'machine/microsoft_terminology_{}.xml'.format(action.lower())
)
payload = template.render(kwargs)
request = Request(self.MS_TM_API_URL, payload.encode('utf-8'))
request.timeout = 0.5
request.add_header(
'SOAPAction', '"{}"'.format(self.MS_TM_SOAP_HEADER + action)
)
request.add_header('Content-Type', 'text/xml; charset=utf-8')
return urlopen(request)
def _action(self, params, body=None, content_type=None):
# about token, see https://github.com/bittorrent/webui/wiki/TokenSystem
url = self.base_url + '?token=' + self.token + '&' + urlencode(params)
request = Request(url)
if body:
request.data = body
request.add_header('Content-length', len(body))
if content_type:
request.add_header('Content-type', content_type)
response = self.opener.open(request)
return response.code, json.loads(response.read())
def get_access_token(self):
data = dict(
client_id=self.client_id,
client_secret=self.client_secret,
scope="http://api.microsofttranslator.com",
grant_type="client_credentials",
)
data_bytes = urlencode(data).encode("utf-8")
request = Request(self.AUTH_URL, data_bytes, method="POST")
request.add_header("Content-Type", "application/x-www-form-urlencoded;charset=utf-8")
response = urlopen(request, data_bytes)
result = response.read().decode("utf-8")
result_dict = loads(result)
return result_dict
def send_request(self, openid, attributes, cafile=None):
post_body = self.build_request(openid, attributes)
req = Request(self.url, post_body.encode('ascii'))
req.add_header('Content-Type', 'text/xml')
if six.PY3:
data = req.data
else:
data = req.get_data()
req.add_header('Content-Length', str(len(data)))
log.debug(req.headers)
log.debug(data)
resp = urlopen(req, cafile=cafile)
return AttributeServiceResponse(resp)
def upload(self, filename):
if not self.cdash_upload_url:
return
# Compute md5 checksum for the contents of this file.
md5sum = checksum(hashlib.md5, filename, block_size=8192)
opener = build_opener(HTTPHandler)
with open(filename, 'rb') as f:
url = "{0}&MD5={1}".format(self.cdash_upload_url, md5sum)
request = Request(url, data=f)
request.add_header('Content-Type', 'text/xml')
request.add_header('Content-Length', os.path.getsize(filename))
# By default, urllib2 only support GET and POST.
# CDash needs expects this file to be uploaded via PUT.
request.get_method = lambda: 'PUT'
url = opener.open(request)
def __download(url, filename=None, filename_only=False):
# http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way
request = Request(url)
request.add_header('Accept-encoding', 'gzip,deflate')
r = urlopen(request)
try:
filename = filename or getFileName(url, r)
if not filename_only:
with open(filename, 'wb') as f:
if r.info().get('Content-Encoding') == 'gzip':
buf = StringIO(r.read())
src = gzip.GzipFile(fileobj=buf)
else:
src = r
shutil.copyfileobj(src, f)
finally:
r.close()
return filename
def http_call(method, url, data=None):
"""Utility method for making HTTP requests."""
LOG.debug("http_call(): Calling %s %s" % (method, url))
opener = build_opener(HTTPHandler)
if data:
data = simplejson.dumps(data)
LOG.debug("http_call(): With body: %s" % data)
request = Request(url, data)
request.add_header('Accept', 'application/json')
if data:
request.add_header('Content-Type', 'application/json')
request.get_method = lambda: method
resp = opener.open(request)
if resp.getcode() >= 400:
raise exceptions.RomanaException("Error in %s %s with payload %s: %s", method, url, data, resp)
body = resp.read()
data = simplejson.loads(body)
return data
def update_library(self, show=None):
"""Handles updating the Emby Media Server host via HTTP API
Returns:
Returns True for no issue or False if there was an error
"""
if sickbeard.USE_EMBY:
if not sickbeard.EMBY_HOST:
logger.log(u'EMBY: No host specified, check your settings', logger.DEBUG)
return False
if show:
if show.indexer == 1:
provider = 'tvdb'
elif show.indexer == 2:
logger.log(u'EMBY: TVRage Provider no longer valid', logger.WARNING)
return False
else:
logger.log(u'EMBY: Provider unknown', logger.WARNING)
return False
query = '?%sid=%s' % (provider, show.indexerid)
else:
query = ''
url = 'http://%s/emby/Library/Series/Updated%s' % (sickbeard.EMBY_HOST, query)
values = {}
data = urlencode(values)
try:
req = Request(url, data)
req.add_header('X-MediaBrowser-Token', sickbeard.EMBY_APIKEY)
response = urlopen(req)
result = response.read()
response.close()
logger.log(u'EMBY: HTTP response: ' + result.replace('\n', ''), logger.DEBUG)
return True
except (URLError, IOError) as e:
logger.log(u'EMBY: Warning: Couldn\'t contact Emby at ' + url + ' ' + ex(e), logger.WARNING)
return False
def get_github_email(access_token):
"""Get real email from GitHub"""
request = Request('https://api.github.com/user/emails')
request.add_header('User-Agent', USER_AGENT)
request.add_header(
'Authorization',
'token {0}'.format(access_token)
)
handle = urlopen(request, timeout=1.0)
data = json.loads(handle.read().decode('utf-8'))
email = None
for entry in data:
# Skip not verified ones
if not entry['verified']:
continue
email = entry['email']
if entry['primary']:
break
return email
def make_image_object_from_url(image_url):
# parse url
parsed_url = urlparse(image_url)
# handle absolute and relative urls, Assuming http for now.
if not parsed_url.scheme:
image_url = '%s%s' % (get_setting('site', 'global', 'siteurl'), image_url)
request = Request(image_url)
request.add_header('User-Agent', settings.TENDENCI_USER_AGENT)
opener = build_opener()
# make image object
try:
socket.setdefaulttimeout(1.5)
data = opener.open(request).read() # get data
im = Image.open(BytesIO(data))
except:
im = None
return im
def translate_strings(self, str_array, from_lang=None, to_lang=None):
if len(str_array) > self.MAX_API_ARRAY:
raise ValueError(
"List is too big to translate, %s is greater than %s" % (len(str_array), self.MAX_API_ARRAY))
if from_lang is None:
from_lang = self.from_lang
if to_lang is None:
to_lang = self.to_langs[0]
url = self.BASE_API + "TranslateArray?"
# Must be in this order!
data = [("from", from_lang)]
data.extend([("texts", thing) for thing in str_array])
data.extend([("to", to_lang)])
strings_enc = "\n".join([self.serialize(thing) for thing in str_array])
category = self.api_category
content_type = "text/plain"
data = """\
<TranslateArrayRequest>
<AppId />
<From>%(from_lang)s</From>
<Options>
<Category xmlns="http://schemas.datacontract.org/2004/07/Microsoft.MT.Web.Service.V2" >%(category)s</Category>
<ContentType xmlns="http://schemas.datacontract.org/2004/07/Microsoft.MT.Web.Service.V2">%(content_type)s</ContentType>
<ReservedFlags xmlns="http://schemas.datacontract.org/2004/07/Microsoft.MT.Web.Service.V2" />
<State xmlns="http://schemas.datacontract.org/2004/07/Microsoft.MT.Web.Service.V2" />
<Uri xmlns="http://schemas.datacontract.org/2004/07/Microsoft.MT.Web.Service.V2" />
<User xmlns="http://schemas.datacontract.org/2004/07/Microsoft.MT.Web.Service.V2" />
</Options>
<Texts>
%(strings_enc)s
</Texts>
<To>%(to_lang)s</To>
</TranslateArrayRequest>""" % locals()
data_bytes = bytes(data.encode("utf-8"))
request = Request(url, data_bytes, method="POST")
request.add_header("Content-Type", "text/xml")
result = self.run_request(request)
return self.simplify_array_result(str_array, self.deserialize_array(result))
def make_image_object_from_url(image_url):
# parse url
parsed_url = urlparse(image_url)
# handle absolute and relative urls, Assuming http for now.
if not parsed_url.scheme:
image_url = '%s%s' % (get_setting('site', 'global',
'siteurl'), image_url)
request = Request(image_url)
request.add_header('User-Agent', settings.TENDENCI_USER_AGENT)
opener = build_opener()
# make image object
try:
socket.setdefaulttimeout(1.5)
data = opener.open(request).read() # get data
im = Image.open(BytesIO(data))
except:
im = None
return im
def render(self):
client = self.context.client(self.request)
uri, headers, body = client.sign(self.context.request_token_uri)
req = Request(uri)
for h, v in headers:
req.add_header(h, v)
req.add_data(body)
try:
res = urlopen(req).read() # should be doing a post
access = self.context.process(res)
# Redirect with parameter: oauth_token (optional)
data = dict(oauth_token=access.parms['oauth_token'])
url = "{}?{}".format(self.context.auth_uri, urlencode(data))
self.redirect(url)
return '<a class="autolink" href="%s">Please visit: %s</a>' % (
str(url), str(url))
except HTTPError as e:
print "Error while opening {}: {}".format(str(req), str(e))
self.error = str(e)
def get_github_email(access_token):
"""Get real email from GitHub"""
request = Request('https://api.github.com/user/emails')
request.timeout = 1.0
request.add_header('User-Agent', USER_AGENT)
request.add_header(
'Authorization',
'token {0}'.format(access_token)
)
handle = urlopen(request)
data = json.loads(handle.read().decode('utf-8'))
email = None
for entry in data:
# Skip not verified ones
if not entry['verified']:
continue
email = entry['email']
if entry['primary']:
break
return email
def __new__(self, token):
app = grok.getApplication()
users = IOAuthPrincipalSource(app)
uri = "https://api.twitter.com/1.1/account/verify_credentials.json"
req = Request(url)
req.add_header('User-Agent', 'Python urlib2 (grok.zopefoundation.org, Python 2.7)')
req.add_header("Content-Type", "application/json")
req.add_header("Authorization", "{} {}".format(token.info['token_type'],
token.info['access_token']))
res = urlopen(req).read()
if res: res = json.loads(res)
if res is None:
return None
else:
print "result=%s" % res
uid = u"Twitter.{}".format(res['id'])
found = users.find(id=uid)
if len(found)==0:
user = users.new(id=uid)
else:
user = list(found)[0]
user.authInfo = token.info
user.domain = u'Twitter'
user.login = uid
user.secret = token.info['access_token']
return user
def __new__(self, token):
app = grok.getApplication()
users = IOAuthPrincipalSource(app)
url = u"https://graph.facebook.com/me"
print "User token info found: %s" % token.info
req = Request(url)
req.add_header('User-Agent', 'Python urlib2 (grok.zopefoundation.org, Python 2.7)')
req.add_header("Content-Type", "application/json")
req.add_header("Authorization", "{} {}".format(token.info['token_type'],
token.info['access_token']))
req.add_data(urlencode(dict(access_token=token.info['access_token'],
fields='id,name,first_name,last_name')))
res = urlopen(req).read()
if res: res = json.loads(res)
if res is None:
return None
else:
print "Personal info returned: %s" % res
uid = u"Facebook.{}".format(res['id'])
found = users.find(id=uid)
if len(found)==0:
user = users.new(id=uid)
else:
user = list(found)[0]
user.authInfo = token.info
user.title = unicode(res['name'])
user.description = u'{} {}'.format(res['first_name'], res['last_name'])
user.domain = u'Facebook'
user.login = unicode(res['id'])
user.secret = unicode(token.info['access_token'])
return user
def __new__(self, token):
app = grok.getApplication()
users = IOAuthPrincipalSource(app)
url = u"https://discordapp.com/api/users/@me"
print "User token info found: %s" % token.info
req = Request(url)
req.add_header('User-Agent', 'Python urlib2 (grok.zopefoundation.org, Python 2.7)')
req.add_header("Content-Type", "application/json")
req.add_header("Authorization", "{} {}".format(token.info['token_type'],
token.info['access_token']))
res = urlopen(req).read()
if res: res = json.loads(res)
if res is None:
return None
else:
print "Personal info returned: %s" % res
uid = u"Discord.{}".format(res['id'])
found = users.find(id=uid)
if len(found)==0:
user = users.new(id=uid)
else:
user = list(found)[0]
user.authInfo = token.info
user.title = unicode(res['username'])
user.description = user.title
user.domain = u'Discord'
user.login = user.disco_id = unicode(res['id'])
user.secret = unicode(token.info['access_token'])
return user
def _notify_emby(self, message, host=None, emby_apikey=None):
"""Handles notifying Emby host via HTTP API
Returns:
Returns True for no issue or False if there was an error
"""
# fill in omitted parameters
if not host:
host = app.EMBY_HOST
if not emby_apikey:
emby_apikey = app.EMBY_APIKEY
url = 'http://%s/emby/Notifications/Admin' % host
values = {
'Name': 'Medusa',
'Description': message,
'ImageUrl': app.LOGO_URL
}
data = json.dumps(values)
try:
req = Request(url, data)
req.add_header('X-MediaBrowser-Token', emby_apikey)
req.add_header('Content-Type', 'application/json')
response = urlopen(req)
result = response.read()
response.close()
log.debug(u'EMBY: HTTP response: {0}', result.replace('\n', ''))
return True
except (URLError, IOError) as error:
log.warning(
u'EMBY: Warning: Unable to contact Emby at {url}: {error}', {
'url': url,
'error': ex(error)
})
return False
def __new__(self, token):
app = grok.getApplication()
users = IOAuthPrincipalSource(app)
url = u"https://www.googleapis.com/userinfo/v2/me"
req = Request(url)
req.add_header('User-Agent', 'Python urlib2 (grok.zopefoundation.org, Python 2.7)')
req.add_header("Content-Type", "application/json")
req.add_header("Authorization", "{} {}".format(token.info['token_type'],
token.info['access_token']))
res = urlopen(req).read()
if res: res = json.loads(res)
if res is None:
return None
else:
uid = u"Google.{}".format(res['id'])
found = users.find(id=uid)
if len(found)==0:
user = users.new(id=uid)
else:
user = list(found)[0]
user.authInfo = token.info
user.title = unicode(res['name'])
user.description = u'{} {}'.format(res['given_name'], res['family_name'])
user.domain = u'Google'
user.login = unicode(res['id'])
user.secret = unicode(token.info['access_token'])
return user
def upload(self, filename):
if not self.cdash_upload_url:
return
# Compute md5 checksum for the contents of this file.
md5sum = checksum(hashlib.md5, filename, block_size=8192)
opener = build_opener(HTTPHandler)
with open(filename, 'rb') as f:
params_dict = {
'build': self.buildname,
'site': self.site,
'stamp': self.buildstamp,
'MD5': md5sum,
}
encoded_params = urlencode(params_dict)
url = "{0}&{1}".format(self.cdash_upload_url, encoded_params)
request = Request(url, data=f)
request.add_header('Content-Type', 'text/xml')
request.add_header('Content-Length', os.path.getsize(filename))
if self.authtoken:
request.add_header('Authorization',
'Bearer {0}'.format(self.authtoken))
# By default, urllib2 only support GET and POST.
# CDash needs expects this file to be uploaded via PUT.
request.get_method = lambda: 'PUT'
response = opener.open(request)
if self.current_package_name not in self.buildIds:
resp_value = response.read()
if isinstance(resp_value, bytes):
resp_value = resp_value.decode('utf-8')
match = self.buildid_regexp.search(resp_value)
if match:
buildid = match.group(1)
self.buildIds[self.current_package_name] = buildid
def home_assistant(items, homassistant_url, homeassistant_password):
data = urlencode({
'title': 'RSScrawler:',
'body': "\n\n".join(items)
})
if six.PY3:
data = data.encode("utf-8")
try:
req = Request(homassistant_url, data)
req.add_header('X-HA-Access', homeassistant_password)
req.add_header('Content-Type', 'application/json')
response = urlopen(req)
except HTTPError:
log_debug('FEHLER - Konnte Home Assistant API nicht erreichen')
return False
res = json.load(response)
if res['sender_name']:
log_debug('Home Assistant Erfolgreich versendet')
else:
log_debug('FEHLER - Konnte nicht an Home Assistant Senden')
def pushbullet(items, token):
data = urlencode({
'type': 'note',
'title': 'RSScrawler:',
'body': "\n\n".join(items)
})
if six.PY3:
data = data.encode("utf-8")
try:
req = Request('https://api.pushbullet.com/v2/pushes', data)
req.add_header('Access-Token', token)
response = urlopen(req)
except HTTPError:
log_debug('FEHLER - Konnte Pushbullet API nicht erreichen')
return False
res = json.load(response)
if res['sender_name']:
log_debug('Pushbullet Erfolgreich versendet')
else:
log_debug('FEHLER - Konnte nicht an Pushbullet Senden')
def oembed(url, max_width=None):
# Find provider
provider = get_oembed_provider(url)
if provider is None:
raise EmbedNotFoundException
# Work out params
params = {'url': url, 'format': 'json'}
if max_width:
params['maxwidth'] = max_width
# Perform request
request = Request(provider + '?' + urlencode(params))
request.add_header('User-agent', 'Mozilla/5.0')
try:
r = urllib_request.urlopen(request)
except URLError:
raise EmbedNotFoundException
oembed = json.loads(r.read().decode('utf-8'))
# Convert photos into HTML
if oembed['type'] == 'photo':
html = '<img src="%s" />' % (oembed['url'], )
else:
html = oembed.get('html')
# Return embed as a dict
return {
'title': oembed['title'] if 'title' in oembed else '',
'author_name':
oembed['author_name'] if 'author_name' in oembed else '',
'provider_name':
oembed['provider_name'] if 'provider_name' in oembed else '',
'type': oembed['type'],
'thumbnail_url': oembed.get('thumbnail_url'),
'width': oembed.get('width'),
'height': oembed.get('height'),
'html': html,
}
def get_feed_html(feed_url):
try:
req = Request(feed_url)
req.add_header(
"User-Agent",
("Webring Pelican plugin/{} " +
"+https://github.com/pelican/pelican-plugins"
).format(WEBRING_VERSION),
)
return urlopen(req).read().decode("utf-8")
except URLError as e:
if hasattr(e, "reason"):
warning(
"webring plugin: failed to connect to feed url (%s).",
feed_url,
e.reason,
)
if hasattr(e, "code"):
warning("webring plugin: server returned %s error (%s).", e.code,
feed_url)
except ValueError as e:
warning("webring plugin: wrong url provided (%s).", e)
def DownloadSetting(link):
req = Request(link)
req.add_header('User-Agent', 'VAS')
response = urlopen(req)
newlink = response.read()
response.close()
Setting = open(Directory + '/Settings/tmp/listE2.zip', 'wb')
Setting.write(newlink)
Setting.close()
if os.path.exists(Directory + '/Settings/tmp/listE2.zip'):
os.system('mkdir ' + Directory + '/Settings/tmp/listE2_unzip')
try:
os.system('unzip -q ' + Directory + '/Settings/tmp/listE2.zip -d ' + Directory + '/Settings/tmp/listE2_unzip')
except:
print("ERROR unzip listE2.zip")
if not os.path.exists(Directory + '/Settings/tmp/setting'):
os.system('mkdir ' + Directory + '/Settings/tmp/setting')
try:
os.system('unzip -q ' + Directory + '/Settings/tmp/listE2_unzip/*.zip -d ' + Directory + '/Settings/tmp/setting')
except:
print("ERROR unzip %s.zip", name)
return False
def update(self, code=None, state=None, **args):
''' Either code or error is defined here if this is in response to Authorization '''
super(V2TokenView, self).update(**args)
oauth = self.context.__parent__.__parent__
self.context.__parent__.__parent__.error_msg = None
if self.error is None:
if code is not None:
token = self.context.__parent__
if token.state == state:
self.context.code = code
data = urlencode(self.context.parms)
print "----------------------------------------------------"
print "url=[%s]; data=[%s]" % (self.context.uri, data)
req = Request(self.context.uri)
req.add_header(
'User-Agent',
'Python urlib2 (grok.zopefoundation.org, Python 2.7)')
req.add_header('Content-Type',
'application/x-www-form-urlencoded')
req.add_header('Accept', 'application/json')
req.add_data(data)
try:
res = urlopen(req).read() # should be doing a post
self.context.info = json.loads(res)
try:
# Update session information with auth info
session = ISession(self.request)['OAuth2']
session['info'] = self.context.info
service = self.context.__parent__.service
principal = component.queryAdapter(self.context,
IOAuthPrincipal,
name=service)
session[
'principal'] = principal if principal else None
# If we get here, we can notify subscribers.
grok.notify(
OAuthenticatedEvent(session['principal']))
self.redirect(self.url(grok.getApplication()))
except Exception as e:
print "Problem in adapter for service: {}".format(
str(e))
self.error = str(e)
except HTTPError as e:
print "Error while exchanging tokens: {}".format(
str(e))
self.error = str(e)
else:
self.error = 'State Mismatch'
else:
self.error = 'Invalid code'
if type(self.error) is str and len(self.error):
print "-------------------------------------------------"
print "Error [%s] in token exchange" % self.error
oauth.error_msg = self._render_template()
self.redirect(self.url(grok.getApplication()))
def _download_all_metadata(url):
"""
Downloads the json file that contains all of the metadata for a specific file type (read:
audio files, benchmark files, or trained models) that is on the EFZ server. This is retrieved
from one of following three URLs (which are stored in nussl.constants):
NUSSL_EFZ_AUDIO_METADATA_URL, NUSSL_EFZ_BENCHMARK_METADATA_URL, or NUSSL_EFZ_MODEL_METADATA_URL.
Args:
url (str): URL for the EFZ server that has metadata. One of these three:
NUSSL_EFZ_AUDIO_METADATA_URL, NUSSL_EFZ_BENCHMARK_METADATA_URL, or
NUSSL_EFZ_MODEL_METADATA_URL.
Returns:
(list): List of dicts with metadata for the desired file type.
"""
request = Request(url)
# Make sure to get the newest data
request.add_header('Pragma', 'no-cache')
request.add_header('Cache-Control', 'max-age=0')
response = urlopen(request)
return json.loads(response.read())
def test_upload(self):
self.assertFalse(self.test_fs.exists('top/middle/upload.txt'))
data = textwrap.dedent("""
-DATA
Content-Disposition: form-data; name="file"; filename="upload.txt"
Content-Type: text/plain
This is an upload test.
-DATA--
""").lstrip().encode('utf-8')
request = Request(self._url('top/middle/'), data=data)
request.add_header("Content-Type",
"multipart/form-data; boundary=-DATA")
request.add_header("Referer", 'top/middle/')
with closing(urlopen(request)) as res:
self.assertEqual(res.code, 200)
self.assertTrue(self.test_fs.exists('top/middle/upload.txt'))
self.assertEqual(self.test_fs.gettext('top/middle/upload.txt'),
'This is an upload test.\n')
def get_version(self):
"""Get the version of this Master.
:returns: This master's version number ``str``
Example::
>>> j = Jenkins()
>>> info = j.get_version()
>>> print info
>>> 1.541
"""
try:
request = Request(self.server)
request.add_header('X-Jenkins', '0.0')
response = urlopen(request, timeout=self.timeout)
return response.info().getheader('X-Jenkins')
except HTTPError:
raise JenkinsException("Error communicating with server[%s]"
% self.server)
except BadStatusLine:
raise JenkinsException("Error communicating with server[%s]"
% self.server)
def __send_request(self, method, uri, data):
retry_codes = {429: 3}
@request_retry(codes=retry_codes)
def __get_response(_request):
return urlopen(_request).read()
url = self.__url + uri
request = Request(url)
if method == 'POST':
request.add_data(json.dumps(data))
auth = base64.encodestring(
'{0}:{1}'.format(self.user, self.password)).strip()
request.add_header('Authorization', 'Basic {}'.format(auth))
request.add_header('Content-Type', 'application/json')
e = None
try:
response = __get_response(request)
except HTTPError as e:
response = e.read()
if response:
result = json.loads(response)
else:
result = {}
if e is not None:
if result and 'error' in result:
error = '"' + result['error'] + '"'
else:
error = 'No additional error message received'
raise APIError('TestRail API returned HTTP %s (%s)' %
(e.code, error))
return result
def get_system(token,
method,
api_cmd,
api_cmd_headers=None,
api_cmd_payload=None,
timeout=10):
"""
Make a rest-api request
Returns: response as a dictionary
"""
LOG.debug("%s cmd:%s hdr:%s payload:%s" %
(method, api_cmd, api_cmd_headers, api_cmd_payload))
response = None
try:
request_info = Request(api_cmd)
request_info.get_method = lambda: method
if token:
request_info.add_header("X-Auth-Token", token.get_id())
request_info.add_header("Accept", "application/json")
if api_cmd_headers is not None:
for header_type, header_value in api_cmd_headers.items():
request_info.add_header(header_type, header_value)
if api_cmd_payload is not None:
request_info.add_data(api_cmd_payload)
request = urlopen(request_info, timeout=timeout)
response = request.read()
if response == "":
response = json.loads("{}")
else:
response = json.loads(response)
request.close()
except HTTPError as e:
if 401 == e.code:
if token:
token.set_expired()
LOG.warn("HTTP Error e.code=%s e=%s" % (e.code, e))
if hasattr(e, 'msg') and e.msg:
response = json.loads(e.msg)
else:
response = json.loads("{}")
raise
except URLError:
LOG.error("Cannot access %s" % api_cmd)
raise
finally:
return response
def rest_api_request(token, method, api_cmd,
api_cmd_payload=None, timeout=10):
"""
Make a rest-api request
Returns: response as a dictionary
"""
api_cmd_headers = dict()
api_cmd_headers['Content-type'] = "application/json"
api_cmd_headers['User-Agent'] = "cert-mon/1.0"
try:
request_info = Request(api_cmd)
request_info.get_method = lambda: method
if token:
request_info.add_header("X-Auth-Token", token.get_id())
request_info.add_header("Accept", "application/json")
if api_cmd_headers is not None:
for header_type, header_value in api_cmd_headers.items():
request_info.add_header(header_type, header_value)
if api_cmd_payload is not None:
request_info.add_data(api_cmd_payload)
request = None
try:
request = urlopen(request_info, timeout=timeout)
response = request.read()
finally:
if request:
request.close()
if response == "":
response = json.loads("{}")
else:
response = json.loads(response)
except HTTPError as e:
if 401 == e.code:
if token:
token.set_expired()
raise
except URLError:
LOG.error("Cannot access %s" % api_cmd)
raise
return response
def rest_api_request(token,
method,
api_cmd,
api_cmd_headers=None,
api_cmd_payload=None,
timeout=10):
"""
Make a rest-api request
Returns: response as a dictionary
"""
# signal.signal(signal.SIGALRM, _timeout_handler)
# if hasattr(signal, 'SIGALRM'):
# signal.alarm(timeout)
LOG.info("%s cmd:%s hdr:%s payload:%s" %
(method, api_cmd, api_cmd_headers, api_cmd_payload))
response = None
try:
request_info = Request(api_cmd)
request_info.get_method = lambda: method
if token:
request_info.add_header("X-Auth-Token", token.get_id())
request_info.add_header("Accept", "application/json")
if api_cmd_headers is not None:
for header_type, header_value in api_cmd_headers.items():
request_info.add_header(header_type, header_value)
if api_cmd_payload is not None:
request_info.add_data(api_cmd_payload)
request = urlopen(request_info, timeout=timeout)
response = request.read()
if response == "":
response = json.loads("{}")
else:
response = json.loads(response)
request.close()
LOG.info("Response=%s" % response)
except HTTPError as e:
if 401 == e.code:
if token:
token.set_expired()
LOG.warn("HTTP Error e.code=%s e=%s" % (e.code, e))
if hasattr(e, 'msg') and e.msg:
response = json.loads(e.msg)
else:
response = json.loads("{}")
LOG.info("HTTPError response=%s" % (response))
raise OpenStackRestAPIException(e.message, e.code, "%s" % e)
except URLError as e:
LOG.warn("URLError Error e=%s" % (e))
raise OpenStackException(e.message, "%s" % e)
except si_exception.SysInvSignalTimeout as e:
LOG.warn("Timeout Error e=%s" % (e))
raise OpenStackException(e.message, "%s" % e)
finally:
signal.alarm(0)
return response
def _request_with_auth(url, username, password):
request = Request(url)
base64string = base64.b64encode(
username.encode('ascii') + b':' + password.encode('ascii'))
request.add_header(b"Authorization", b"Basic " + base64string)
return urlopen(request)
def json_req(self,
url,
http_post=False,
skip_auth=False,
raw=False,
json_body=False,
**kwargs):
"""Perform JSON request."""
# JSON body requires using POST
if json_body:
http_post = True
# Encode params
if kwargs:
if json_body:
params = json.dumps(kwargs)
else:
params = urlencode(kwargs)
else:
if json_body:
params = '{}'
else:
params = ''
# Store for exception handling
self.request_url = url
self.request_params = params
# Append parameters
if params and not http_post:
url = '?'.join((url, params))
# Create request object with custom headers
request = Request(url)
request.add_header('User-Agent', USER_AGENT)
request.add_header('Referer', get_site_url())
# Optional authentication
if not skip_auth:
self.authenticate(request)
# Fire request
if http_post:
handle = urlopen(request, params.encode('utf-8'), timeout=5.0)
else:
handle = urlopen(request, timeout=5.0)
# Read and possibly convert response
text = handle.read()
# Needed for Microsoft
if text[:3] == b'\xef\xbb\xbf':
text = text.decode('UTF-8-sig')
else:
text = text.decode('utf-8')
# Replace literal \t
text = text.strip().replace('\t', '\\t').replace('\r', '\\r')
# Needed for Google
while ',,' in text or '[,' in text:
text = text.replace(',,', ',null,').replace('[,', '[')
if raw:
return text
# Parse and return JSON
return json.loads(text)
def request_wolfram_alpha(input, verbose=False):
r"""
Request Wolfram Alpha website.
INPUT:
- ``input`` -- string
- ``verbose`` -- bool (default: ``False``)
OUTPUT:
json
EXAMPLES::
sage: from sage.symbolic.integration.external import request_wolfram_alpha
sage: page_data = request_wolfram_alpha('integrate Sin[x]') # optional internet
sage: [str(a) for a in sorted(page_data.keys())] # optional internet
['queryresult']
sage: [str(a) for a in sorted(page_data['queryresult'].keys())] # optional internet
['datatypes',
'encryptedEvaluatedExpression',
'encryptedParsedExpression',
'error',
'host',
'id',
'numpods',
'parsetimedout',
'parsetiming',
'pods',
'recalculate',
'related',
'server',
'sponsorCategories',
'success',
'timedout',
'timedoutpods',
'timing',
'version']
"""
# import compatible with py2 and py3
from six.moves.urllib.parse import urlencode
from six.moves.urllib.request import Request, build_opener, HTTPCookieProcessor
import json
from http.cookiejar import CookieJar
# we need cookies for this...
cj = CookieJar()
opener = build_opener(HTTPCookieProcessor(cj))
# build initial query for code
req = Request("http://www.wolframalpha.com/input/api/v1/code")
resp = opener.open(req)
# the website returns JSON containing the code
page_data = json.loads(resp.read().decode("utf-8"))
if not ("code" in page_data.keys()):
raise ValueError("Wolfram did not return a code")
proxy_code = page_data['code']
if verbose:
print("Code: {}".format(proxy_code))
print("Cookies: {}".format(cj))
# now we can make a request
# some parameters documented here:
# https://products.wolframalpha.com/api/documentation/#parameter-reference
# the following are the parameters used by the website
params = {
'assumptionsversion': '2',
'async': 'true',
'banners': 'raw',
'debuggingdata': 'false',
'format': 'image,plaintext,imagemap,sound,minput,moutput',
'formattimeout': '8',
'input': input,
'output': 'JSON',
'parsetimeout': '5',
'podinfosasync': 'true',
'proxycode': proxy_code,
'recalcscheme': 'parallel',
'sbsdetails': 'true',
'scantimeout': '0.5',
'sponsorcategories': 'true',
'statemethod': 'deploybutton',
'storesubpodexprs': 'true'
}
# # we can also change some parameters
# params = {
# 'assumptionsversion': '2',
# 'banners': 'raw',
# 'format': 'minput,moutput',
# 'formattimeout': '8',
# 'input': input,
# 'output': 'JSON',
# 'parsetimeout': '5',
# 'proxycode': proxy_code,
# 'scantimeout': '0.5',
# 'storesubpodexprs': 'true'
# }
params = urlencode(params)
url = "https://www.wolframalpha.com/input/json.jsp?%s" % params
req = Request(url)
req.add_header('Referer',
"https://www.wolframalpha.com/input/") # seems important
resp = opener.open(req)
# the website returns JSON containing the code
return json.loads(resp.read().decode("utf-8"))
def test_request_cookies(server):
req = Request(url=server.get_url())
req.add_header('Cookie', 'foo=bar')
urlopen(req)
assert server.request['cookies']['foo']['value'] == 'bar'
def execute(cls, uri, http_verb, extra_headers=None, batch=False, _body=None, **kw):
"""
if batch == False, execute a command with the given parameters and
return the response JSON.
If batch == True, return the dictionary that would be used in a batch
command.
"""
if batch:
urlsplitter = urlparse(API_ROOT).netloc
ret = {"method": http_verb, "path": uri.split(urlsplitter, 1)[1]}
if kw:
ret["body"] = kw
return ret
if not ('app_id' in ACCESS_KEYS and 'rest_key' in ACCESS_KEYS):
raise core.ParseError('Missing connection credentials')
app_id = ACCESS_KEYS.get('app_id')
rest_key = ACCESS_KEYS.get('rest_key')
master_key = ACCESS_KEYS.get('master_key')
url = uri if uri.startswith(API_ROOT) else cls.ENDPOINT_ROOT + uri
if _body is None:
data = kw and json.dumps(kw, default=date_handler) or "{}"
else:
data = _body
if http_verb == 'GET' and data:
url += '?%s' % urlencode(kw)
data = None
else:
if cls.__name__ == 'File':
data = data
else:
data = data.encode('utf-8')
headers = {
'Content-type': 'application/json',
'X-Parse-Application-Id': app_id,
'X-Parse-REST-API-Key': rest_key
}
headers.update(extra_headers or {})
if cls.__name__ == 'File':
#request = Request(url.encode('utf-8'), data, headers)
request = Request(url, data, headers)
else:
request = Request(url, data, headers)
if ACCESS_KEYS.get('session_token'):
request.add_header('X-Parse-Session-Token', ACCESS_KEYS.get('session_token'))
elif master_key:
request.add_header('X-Parse-Master-Key', master_key)
request.get_method = lambda: http_verb
try:
response = urlopen(request, timeout=CONNECTION_TIMEOUT)
except HTTPError as e:
exc = {
400: core.ResourceRequestBadRequest,
401: core.ResourceRequestLoginRequired,
403: core.ResourceRequestForbidden,
404: core.ResourceRequestNotFound
}.get(e.code, core.ParseError)
raise exc(e.read())
return json.loads(response.read().decode('utf-8'))
import json
import os
import time
from six.moves.urllib.request import Request, urlopen
path = os.path.dirname(os.path.realpath(__file__))
host = "localhost"
data = {"offer_sdp": "asdf"}
req = Request("http://%s:5502/v1/rtc-offer" % host)
req.add_header("Content-Type", "application/json")
response = urlopen(req, json.dumps(data).encode("utf-8"))
response_body = response.read()
response_json = json.loads(response_body)
print("RTC offer %s" % response_json)
def _send_to_kodi(command,
host=None,
username=None,
password=None,
dest_app="KODI"): # pylint: disable=too-many-arguments
"""Handles communication to KODI servers via HTTP API
Args:
command: Dictionary of field/data pairs, encoded via urllib and passed to the KODI API via HTTP
host: KODI webserver host:port
username: KODI webserver username
password: KODI webserver password
Returns:
Returns response.result for successful commands or False if there was an error
"""
# fill in omitted parameters
if not username:
username = app.KODI_USERNAME
if not password:
password = app.KODI_PASSWORD
if not host:
logger.log(u'No %s host passed, aborting update' % dest_app,
logger.WARNING)
return False
for key in command:
if isinstance(command[key], text_type):
command[key] = command[key].encode('utf-8')
enc_command = urlencode(command)
logger.log(u"%s encoded API command: %r" % (dest_app, enc_command),
logger.DEBUG)
# url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, enc_command) # maybe need for old plex?
url = 'http://%s/kodiCmds/kodiHttp/?%s' % (host, enc_command)
try:
req = Request(url)
# if we have a password, use authentication
if password:
base64string = base64.encodestring('%s:%s' %
(username, password))[:-1]
authheader = "Basic %s" % base64string
req.add_header("Authorization", authheader)
logger.log(
u"Contacting %s (with auth header) via url: %s" %
(dest_app, ss(url)), logger.DEBUG)
else:
logger.log(u"Contacting %s via url: %s" % (dest_app, ss(url)),
logger.DEBUG)
try:
response = urlopen(req)
except (BadStatusLine, URLError) as e:
logger.log(
u"Couldn't contact %s HTTP at %r : %r" %
(dest_app, url, ex(e)), logger.DEBUG)
return False
result = response.read().decode(app.SYS_ENCODING)
response.close()
logger.log(
u"%s HTTP response: %s" % (dest_app, result.replace('\n', '')),
logger.DEBUG)
return result
except Exception as e:
logger.log(
u"Couldn't contact %s HTTP at %r : %r" %
(dest_app, url, ex(e)), logger.DEBUG)
return False
def url_downloader(url, data=None, path=None, cookie=None,
timeout=5, retry=1, retry_ivl=5, agent=None, proxy=None):
"""Download URL link
url: url to download
data: post data
path: download to local file
timeout: socket timeout
retry: retry times to download url
retry_ivl: interval time when retry
agent: http user agent
proxy: socks5://127.0.0.1:1080
"""
while True:
try:
if isinstance(data, dict):
data = urlencode(data)
request = Request(url, data=data)
request.add_header('User-Agent', agent or get_user_agent())
if data:
request.add_header(
'Content-Type',
'application/x-www-form-urlencoded;charset=utf-8')
response = None
handlers = []
if proxy:
scheme, host, port = proxy.split(':')
host = host.strip('/')
proxy_handler = SocksiPyHandler(
socks.PROXY_TYPES[scheme.upper()], host, int(port)
)
handlers.append(proxy_handler)
if cookie is None:
cookie = CookieJar()
cookie_handler = HTTPCookieProcessor(cookie)
handlers.append(cookie_handler)
opener = build_opener(*handlers)
response = opener.open(request, timeout=timeout)
content_encoding = response.info().get('content-encoding')
if content_encoding:
r_data = gzip.decompress(response.read())
else:
r_data = response.read()
if path:
with open(path, 'wb') as f:
f.write(r_data)
r_data = None
response.close()
mime = response.info().get('content-type')
real_url = response.geturl()
err_msg = 'Ok'
break
except (URLError, socket.error, Exception) as err:
response and response.close()
retry -= 1
err_msg = str(err)
if retry > 0:
time.sleep(retry_ivl)
retry_ivl += retry_ivl
timeout += timeout
else:
mime = r_data = real_url = None
break
return {
'mime': mime,
'path': path,
'data': r_data,
'url': real_url,
'cookie': cookie,
'error': err_msg,
}
def __init__(self, name, additional_headers={}):
"""
@param name: URL to be opened
@keyword additional_headers: additional HTTP request headers to be added to the call
"""
try:
# Note the removal of the fragment ID. This is necessary, per the HTTP spec
req = Request(url=name.split('#')[0])
for key in additional_headers:
req.add_header(key, additional_headers[key])
if 'Accept' not in additional_headers:
req.add_header('Accept', 'text/html, application/xhtml+xml')
self.data = urlopen(req)
self.headers = self.data.info()
if URIOpener.CONTENT_TYPE in self.headers:
# The call below will remove the possible media type parameters, like charset settings
ct = content_type(self.headers[URIOpener.CONTENT_TYPE])
self.content_type = ct.media_type
if 'charset' in ct.parmdict:
self.charset = ct.parmdict['charset']
else:
self.charset = None
# print
else:
# check if the suffix can be used for the content type; this may be important
# for file:// type URI or if the server is not properly set up to return the right
# mime type
self.charset = None
self.content_type = ""
for suffix in preferred_suffixes.keys():
if name.endswith(suffix):
self.content_type = preferred_suffixes[suffix]
break
if URIOpener.CONTENT_LOCATION in self.headers:
self.location = urljoin(
self.data.geturl(),
self.headers[URIOpener.CONTENT_LOCATION])
else:
self.location = name
self.expiration_date = datetime.datetime.utcnow(
) + datetime.timedelta(days=1)
if URIOpener.EXPIRES in self.headers:
try:
# Thanks to Deron Meranda for the HTTP date conversion method...
self.expiration_date = parse_http_datetime(
self.headers[URIOpener.EXPIRES])
except:
# The Expires date format was wrong, sorry, forget it...
pass
self.last_modified_date = None
if URIOpener.LAST_MODIFIED in self.headers:
try:
# Thanks to Deron Meranda for the HTTP date conversion method...
self.last_modified_date = parse_http_datetime(
self.headers[URIOpener.LAST_MODIFIED])
except:
# The last modified date format was wrong, sorry, forget it...
pass
except HTTPError:
e = sys.exc_info()[1]
from . import HTTPError
msg = BaseHTTPRequestHandler.responses[e.code]
raise HTTPError('%s' % msg[1], e.code)
except Exception:
e = sys.exc_info()[1]
from . import RDFaError
raise RDFaError('%s' % e)
def _send_to_kodi_json(command, host=None, username=None, password=None, dest_app="KODI"):
"""Handles communication to KODI servers via JSONRPC
Args:
command: Dictionary of field/data pairs, encoded via urllib and passed to the KODI JSON-RPC via HTTP
host: KODI webserver host:port
username: KODI webserver username
password: KODI webserver password
Returns:
Returns response.result for successful commands or False if there was an error
"""
# fill in omitted parameters
if not username:
username = app.KODI_USERNAME
if not password:
password = app.KODI_PASSWORD
if not host:
logger.log(u'No %s host passed, aborting update' % dest_app, logger.WARNING)
return False
command = command.encode('utf-8')
logger.log(u"%s JSON command: %s" % (dest_app, command), logger.DEBUG)
url = 'http://%s/jsonrpc' % host
try:
req = Request(url, command)
req.add_header("Content-type", "application/json")
# if we have a password, use authentication
if password:
base64string = base64.encodestring('%s:%s' % (username, password))[:-1]
authheader = "Basic %s" % base64string
req.add_header("Authorization", authheader)
logger.log(u"Contacting %s (with auth header) via url: %s" % (dest_app, ss(url)), logger.DEBUG)
else:
logger.log(u"Contacting %s via url: %s" % (dest_app, ss(url)), logger.DEBUG)
try:
response = urlopen(req)
except (BadStatusLine, URLError) as e:
if app.KODI_ALWAYS_ON:
logger.log(u"Error while trying to retrieve %s API version for %s: %r" % (dest_app, host, ex(e)), logger.WARNING)
return False
# parse the json result
try:
result = json.load(response)
response.close()
logger.log(u"%s JSON response: %s" % (dest_app, result), logger.DEBUG)
return result # need to return response for parsing
except ValueError as e:
logger.log(u"Unable to decode JSON: " + str(response.read()), logger.WARNING)
return False
except IOError as e:
if app.KODI_ALWAYS_ON:
logger.log(u"Warning: Couldn't contact %s JSON API at %s: %r" % (dest_app, ss(url), ex(e)), logger.WARNING)
return False