def marketTradeHist(self, pair, start=False, end=False):
"""
Returns public trade history for <pair>
starting at <start> and ending at [end=time()]
"""
if self.coach:
self.coach.wait()
args = {
'command': 'returnTradeHistory',
'currencyPair': str(pair).upper()
}
if start:
args['start'] = start
if end:
args['end'] = end
try:
ret = _get('https://poloniex.com/public?' + _urlencode(args),
timeout=self.timeout)
self.logger.debug(ret.url)
except Exception as e:
raise e
if not self.jsonNums:
try:
return _loads(ret.text, parse_float=unicode)
except NameError:
return _loads(ret.text, parse_float=str)
return _loads(ret.text,
parse_float=self.jsonNums,
parse_int=self.jsonNums)
def geocode(self, address):
if self.status.remainingQuota == 0:
raise GeocoderFatalException("Today's quota has been depleted.")
url = '%s?%s' % (self.url, _urlencode({'sensor': 'false', 'address': address}))
if self.logger: self.logger.info('GET: %s' % (url,))
responseFile = _urlopen(url)
self.status.decrementQuota()
responseJson = responseFile.read().decode('utf-8')
response = _json.loads(responseJson)
if response['status'] == 'ZERO_RESULTS': return (None, None)
if self.logger: self.logger.info('Status: %s' % (response['status'],))
assert(response['status'] == 'OK')
assert(len(response['results']) > 0)
result = response['results'][0]
geometry = result['geometry']
location = geometry['location']
latitude = location['lat']
longitude = location['lng']
if self.logger: self.logger.info('Geocoded "%s" to (%f,%f)' % (address, latitude, longitude))
return (latitude, longitude)
def marketTradeHist(self, pair, start=False, end=False):
"""
Returns public trade history for <pair>
starting at <start> and ending at [end=time()]
"""
if self._coaching:
self.apicoach.wait()
if not start:
start = time() - self.HOUR
if not end:
end = time()
try:
ret = _get(
'https://poloniex.com/public?' + _urlencode({
'command': 'returnTradeHistory',
'currencyPair': str(pair).upper(),
'start': str(start),
'end': str(end)
}),
timeout=self.timeout)
except Exception as e:
raise e
try:
return _loads(ret.text, parse_float=unicode)
except NameError:
return _loads(ret.text, parse_float=str)
def __call__(self):
succeedFlag = True
try:
self._driver = webdriver.Chrome()
self._driver.implicitly_wait(4)
try:
lastPageIndex = self.get_last_page_index()
if lastPageIndex == -1:
succeedFlag = False
else:
for i in range(1, min(self._numPages, lastPageIndex) + 1):
url = STRING_URL_BASE + _urlencode(
{
'keyword': self._keywordQuoted,
'page': str(i)
},
quote_via=_quote)
__log__.info('[+]Page %d downloading...' % i)
flag = self.collect_url_on_page(url)
if flag:
__log__.info('[+]Page %d download completed' % i)
else:
__log__.error('[-]Page %d download failed' % i)
finally:
self._driver.quit()
except common.exceptions.WebDriverException as err:
__log__.error('[-]Error: %s' % err)
finally:
EVENT_ALL_DONE.set()
with self.__cntLock:
self.__instanceList.remove(self)
return succeedFlag
def _request_authorization_code(self):
scheme, netloc, path, _, _, _ = _urlparse.urlparse(self._auth_endpoint)
query = _urlencode(self._params)
endpoint = _urlparse.urlunparse(
(scheme, netloc, path, None, query, None))
auth_logger.debug(f"Requesting authorization code through {endpoint}")
_webbrowser.open_new_tab(endpoint)
def _submit(self, operationUrl, data):
# type: (str, dict) -> dict
orderedData = _OrderedDict()
isBatch = "batch" in operationUrl
if not self.submitRequests and "format" in data.keys():
data.pop("format")
for key in sorted(data.keys()):
orderedData[key] = data[key]
data = orderedData
requestUrls = data.pop("requests") if isBatch else []
requestAsParams = "&".join(["requests[]=" + url for url in requestUrls]) if isBatch else ""
urlParams = _urlencode(data)
urlParams += "&" + requestAsParams if isBatch else ""
urlToSignature = operationUrl + urlParams + self.privateKey
signature = _md5(urlToSignature.encode()).hexdigest()
finalUrl = operationUrl + urlParams + "&signature=" + signature
if self.submitRequests:
if _DEBUG: print("Requesting URL:", finalUrl)
response = _urlopen(finalUrl).read().decode()
if self.responseFormat == "json":
return _literal_eval(response)["response"]
else:
return response
else:
if _DEBUG: print("Generated URL:", finalUrl)
return finalUrl
def list_submissions(self, id=None, student=None, grader=None):
"""
Returns the list of submissions associated with an assignment, which
optionally can be filtered according to a specific submitting `student`
or a `grader`.
"""
_class_type = type(self)
id = self._get_id(id=id)
endpoint = "{}/submissions".format(self.instance_endpoint_by_id(id=id))
endpoint_params = {}
if student != None:
# Filter according to a specific student (will be URL-quoted later)
endpoint_params["student"] = student
if grader != None:
# Filter according to a specific grader (will be URL-quoted later)
endpoint_params["grader"] = grader
if len(endpoint_params) > 0:
endpoint += "?{}".format(_urlencode(endpoint_params))
ret = self._requestor._request(
endpoint=endpoint,
method="GET",
)
if ret.status_code == 200:
# Returns a list of all submissions
return list(
map(lambda kwargs: _submissions.Submissions(**kwargs),
ret.json))
def urlencode(params):
utf8_params = encode_params_utf8(params)
urlencoded = _urlencode(utf8_params)
if isinstance(urlencoded, str):
return urlencoded
else:
return urlencoded.decode("utf-8")
def _encode_signed_url(self, url):
"""Add a signature parameter to a url.
:param url: A fully formed url, without a signature.
:return: A str of the signed url, ready to use for an API call.
"""
url_bytes = bytearray(url + self._private_key, encoding='ascii')
signature = _hashlib.sha1(url_bytes).hexdigest()
parameter = _urlencode({"signature": signature})
return "{}&{}".format(url, parameter)
def _submit(self, endpoint, values):
"""Submit an API call with url and optional parameters.
:param endpoint: A fully formed url including endpoint.
:param values: A dictionary of parameters. Non-signed.
:return: A `Future` containing the API response.
"""
url = "{}?{}".format(endpoint, _urlencode(values))
signed_url = self._encode_signed_url(url)
return self._executor.submit(self._get_response, signed_url)
def private_command(self, args, timeout=30):
date = None
for _ in range(timeout):
args['nonce'] = int(time.time() * 10)
sign = _new(self.api_secret.encode('utf-8'),
_urlencode(args).encode('utf-8'), _sha512)
headers = {'Key': self.api_key, 'Sign': sign.hexdigest()}
payload = {
'url': 'https://poloniex.com/tradingApi',
'headers': headers,
'data': args
}
data = self.post_with_retry(payload)
if 'error' in data:
print(data)
data = None
time.sleep(1)
continue
else:
break
return data
def marketTradeHist(self, pair, start=False, end=time()):
"""
Returns public trade history for <pair>
starting at <start> and ending at [end=time()]
"""
if self._coaching:
self.apicoach.wait()
if not start:
start = time()-self.HOUR
try:
ret = _get(
'https://poloniex.com/public?'+_urlencode({
'command': 'returnTradeHistory',
'currencyPair': str(pair).upper(),
'start': str(start),
'end': str(end)
}),
timeout=self.timeout)
except Exception as e:
raise e
try:
return _loads(ret.text, parse_float=unicode)
except NameError:
return _loads(ret.text, parse_float=str)
def _request_authorization_code(self):
scheme, netloc, path, _, _, _ = _urlparse.urlparse(self._auth_endpoint)
query = _urlencode(self._params)
endpoint = _urlparse.urlunparse((scheme, netloc, path, None, query, None))
_webbrowser.open_new_tab(endpoint)
def urlencode(query, doseq):
return _urlencode(query, doseq).encode('ascii')
def __call__(self, command, args={}):
"""
Main Api Function
- encodes and sends <command> with optional [args] to Poloniex api
- raises 'ValueError' if an api key or secret is missing
(and the command is 'private'), or if the <command> is not valid
- returns decoded json api message
"""
global PUBLIC_COMMANDS, PRIVATE_COMMANDS
# check in with the coach
if self._coaching:
self.apicoach.wait()
# pass the command
args['command'] = command
# private?
if command in PRIVATE_COMMANDS:
# check for keys
if not self.Key or not self.Secret:
raise ValueError("A Key and Secret needed!")
# set nonce
args['nonce'] = self.nonce
try:
# encode arguments for url
postData = _urlencode(args)
# sign postData with our Secret
sign = _new(
self.Secret.encode('utf-8'),
postData.encode('utf-8'),
_sha512)
# post request
ret = _post(
'https://poloniex.com/tradingApi',
data=args,
headers={
'Sign': sign.hexdigest(),
'Key': self.Key
},
timeout=self.timeout)
except Exception as e:
raise e
finally:
# increment nonce(no matter what)
self.nonce += 1
# return decoded json
try:
return _loads(ret.text, parse_float=unicode)
except NameError:
return _loads(ret.text, parse_float=str)
# public?
elif command in PUBLIC_COMMANDS:
try:
ret = _get(
'https://poloniex.com/public?' + _urlencode(args),
timeout=self.timeout)
except Exception as e:
raise e
try:
return _loads(ret.text, parse_float=unicode)
except NameError:
return _loads(ret.text, parse_float=str)
else:
raise ValueError("Invalid Command!")
def __call__(self, command, args={}):
"""
Main Api Function
- encodes and sends <command> with optional [args] to Poloniex api
- raises 'ValueError' if an api key or secret is missing
(and the command is 'private'), or if the <command> is not valid
- returns decoded json api message
"""
global PUBLIC_COMMANDS, PRIVATE_COMMANDS
# check in with the coach
self.apicoach.wait()
# pass the command
args['command'] = command
# private?
if command in PRIVATE_COMMANDS:
# check for keys
if not self.Key or not self.Secret:
raise ValueError("A Key and Secret needed!")
# set nonce
args['nonce'] = self.nonce
try:
# encode arguments for url
postData = _urlencode(args)
# sign postData with our Secret
sign = _new(self.Secret.encode('utf-8'),
postData.encode('utf-8'), _sha512)
# post request
ret = _post('https://poloniex.com/tradingApi',
data=args,
headers={
'Sign': sign.hexdigest(),
'Key': self.Key
},
timeout=self.timeout)
except Exception as e:
raise e
finally:
pass
# return decoded json
try:
text = ret.text
if command not in self.nolog:
self.logger.debug("""
<{0}>
<args>{1}</args>
<RESULT>{2}</RESULT>
</{0}>
""".format(command, args, text, command))
struct = _loads(text, parse_float=unicode)
struct = self.retval_wrapper(struct)
return struct
except NameError:
return _loads(text, parse_float=str)
except:
self.logger.debug("Unexpected error:", sys.exc_info()[0])
self.logger.debug("<error>%s</error>", text)
raise
# public?
elif command in PUBLIC_COMMANDS:
try:
args['nonce'] = self.nonce
ret = _get('https://poloniex.com/public?' + _urlencode(args),
timeout=self.timeout)
except Exception as e:
raise e
try:
text = ret.text
if command not in self.nolog:
self.logger.debug("""
<{0}>
<args>{1}</args>
<result>{2}</result>
</{0}>
""".format(command, args, text, command))
struct = _loads(text, parse_float=unicode)
struct = self.retval_wrapper(struct)
return struct
except NameError:
return _loads(text, parse_float=str)
except:
self.logger.debug("Unexpected error:", sys.exc_info()[0])
self.logger.debug("<error>%s</error>", text)
raise
else:
raise ValueError("Invalid Command!")
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
backend=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
header_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
raise_error=True,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = hubblestack.config.master_config(
os.path.join(hubblestack.syspaths.CONFIG_DIR, 'master'))
elif node == 'minion':
opts = hubblestack.config.get_config(
os.path.join(hubblestack.syspaths.CONFIG_DIR, 'minion'))
else:
opts = {}
if not backend:
backend = opts.get('backend', 'tornado')
match = re.match(
r'https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)',
url)
if not match:
hubblestack.utils.network.refresh_dns()
if backend == 'requests':
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = tornado.httputil.url_concat(url, params) if params else url
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(data_file, data_render, data_renderer, template_dict,
opts)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug('Requesting URL %s using %s method', log_url, method)
log.debug("Using backend: %s", backend)
if method == 'POST' and log.isEnabledFor(logging.TRACE):
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = 'XXXXXXXXXX'
log.trace('Request POST Data: %s', pprint.pformat(log_data))
else:
log.trace('Request POST Data: %s', pprint.pformat(data))
if header_file is not None:
header_tpl = _render(header_file, header_render, header_renderer,
template_dict, opts)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(
opts.get('cachedir', hubblestack.syspaths.CACHE_DIR),
'cookies.txt')
if session_cookie_jar is None:
session_cookie_jar = os.path.join(
opts.get('cachedir', hubblestack.syspaths.CACHE_DIR),
'cookies.session.p')
if persist_session is True and HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with hubblestack.utils.files.fopen(session_cookie_jar,
'rb') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with hubblestack.utils.files.fopen(session_cookie_jar,
'wb') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if backend == 'requests':
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: %s', sess.headers)
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == 'urllib2':
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = http.cookiejar.MozillaCookieJar(cookie_jar)
else:
sess_cookies = http.cookiejar.LWPCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if backend == 'requests':
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, str):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error(
'The client-side certificate path that'
' was passed is not valid: %s', cert)
result = sess.request(method,
url,
params=params,
data=data,
**req_kwargs)
result.raise_for_status()
if stream is True:
# fake a HTTP response header
header_callback('HTTP/1.0 {0} MESSAGE'.format(result.status_code))
# fake streaming the content
streaming_callback(result.content)
return {
'handle': result,
}
if handle is True:
return {
'handle': result,
'body': result.content,
}
log.debug('Final URL location of Response: %s',
sanitize_url(result.url, hide_fields))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
body = result.content
if not isinstance(body, str):
body = body.decode(result.encoding or 'utf-8')
ret['body'] = body
elif backend == 'urllib2':
request = urllib_request.Request(url_full, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https'):
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warning(
'match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!')
elif verify_ssl is False:
log.warning('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!')
else:
if ':' in hostname:
hostname, port = hostname.split(':')
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = ('The certificate was invalid. '
'Error returned was: %s',
pprint.pformat(exc))
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, str):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error(
'The client-side certificate path that was '
'passed is not valid: %s', cert)
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(
urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[0] = http.client.HTTPSConnection(
**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {'Error': str(exc)}
if stream is True or handle is True:
return {
'handle': result,
'body': result.content,
}
result_status_code = result.code
result_headers = dict(result.info())
result_text = result.read()
if 'Content-Type' in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers['Content-Type'])
if res_content_type.startswith('text/') and \
'charset' in res_params and \
not isinstance(result_text, str):
result_text = result_text.decode(res_params['charset'])
if isinstance(result_text, bytes):
result_text = result_text.decode('utf-8')
ret['body'] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, str):
if os.path.exists(cert):
req_kwargs['client_cert'] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['client_cert'] = cert[0]
req_kwargs['client_key'] = cert[1]
else:
log.error(
'The client-side certificate path that '
'was passed is not valid: %s', cert)
if isinstance(data, dict):
data = _urlencode(data)
if verify_ssl:
req_kwargs['ca_certs'] = ca_bundle
max_body = opts.get('http_max_body',
hubblestack.config.DEFAULT_OPTS['http_max_body'])
connect_timeout = opts.get(
'http_connect_timeout',
hubblestack.config.DEFAULT_OPTS['http_connect_timeout'])
timeout = opts.get(
'http_request_timeout',
hubblestack.config.DEFAULT_OPTS['http_request_timeout'])
client_argspec = None
proxy_host = opts.get('proxy_host', None)
if proxy_host:
# tornado requires a str for proxy_host, cannot be a unicode str in py2
proxy_host = hubblestack.utils.stringutils.to_str(proxy_host)
proxy_port = opts.get('proxy_port', None)
proxy_username = opts.get('proxy_username', None)
if proxy_username:
# tornado requires a str, cannot be unicode str in py2
proxy_username = hubblestack.utils.stringutils.to_str(
proxy_username)
proxy_password = opts.get('proxy_password', None)
if proxy_password:
# tornado requires a str, cannot be unicode str in py2
proxy_password = hubblestack.utils.stringutils.to_str(
proxy_password)
no_proxy = opts.get('no_proxy', [])
# Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones
# except we remove the valid ones if a url has a no_proxy hostname in it
if urlparse(url_full).hostname in no_proxy:
proxy_host = None
proxy_port = None
# We want to use curl_http if we have a proxy defined
if proxy_host and proxy_port:
if HAS_CURL_HTTPCLIENT is False:
ret['error'] = (
'proxy_host and proxy_port has been set. This requires pycurl and tornado, '
'but the libraries does not seem to be installed')
log.error(ret['error'])
return ret
tornado.httpclient.AsyncHTTPClient.configure(
'tornado.curl_httpclient.CurlAsyncHTTPClient')
client_argspec = hubblestack.utils.args.get_function_argspec(
tornado.curl_httpclient.CurlAsyncHTTPClient.initialize)
else:
client_argspec = hubblestack.utils.args.get_function_argspec(
tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize)
supports_max_body_size = 'max_body_size' in client_argspec.args
req_kwargs.update({
'method': method,
'headers': header_dict,
'auth_username': username,
'auth_password': password,
'body': data,
'validate_cert': verify_ssl,
'allow_nonstandard_methods': True,
'streaming_callback': streaming_callback,
'header_callback': header_callback,
'connect_timeout': connect_timeout,
'request_timeout': timeout,
'proxy_host': proxy_host,
'proxy_port': proxy_port,
'proxy_username': proxy_username,
'proxy_password': proxy_password,
'raise_error': raise_error,
'decompress_response': False,
})
# Unicode types will cause a TypeError when Tornado's curl HTTPClient
# invokes setopt. Therefore, make sure all arguments we pass which
# contain strings are str types.
req_kwargs = hubblestack.utils.data.decode(req_kwargs, to_str=True)
try:
download_client = HTTPClient(max_body_size=max_body) \
if supports_max_body_size \
else HTTPClient()
result = download_client.fetch(url_full, **req_kwargs)
except tornado.httpclient.HTTPError as exc:
ret['status'] = exc.code
ret['error'] = str(exc)
return ret
except socket.gaierror as exc:
if status is True:
ret['status'] = 0
ret['error'] = str(exc)
return ret
if stream is True or handle is True:
return {
'handle': result,
'body': result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
if 'Content-Type' in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers['Content-Type'])
if res_content_type.startswith('text/') and \
'charset' in res_params and \
not isinstance(result_text, str):
result_text = result_text.decode(res_params['charset'])
if isinstance(result_text, bytes):
result_text = result_text.decode('utf-8')
ret['body'] = result_text
if 'Set-Cookie' in result_headers and cookies is not None:
result_cookies = parse_cookie_header(result_headers['Set-Cookie'])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: %s', result_status_code)
log.trace('Response Headers: %s', result_headers)
log.trace('Response Cookies: %s', sess_cookies)
# log.trace("Content: %s", result_text)
coding = result_headers.get('Content-Encoding', "identity")
# Requests will always decompress the content, and working around that is annoying.
if backend != 'requests':
result_text = __decompressContent(coding, result_text)
try:
log.trace('Response Text: %s', result_text)
except UnicodeEncodeError as exc:
log.trace(
'Cannot Trace Log Response Text: %s. This may be due to '
'incompatibilities between requests and logging.', exc)
if text_out is not None:
with hubblestack.utils.files.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with hubblestack.utils.files.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with hubblestack.utils.files.fopen(session_cookie_jar,
'wb') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get('content-type',
'application/json')
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
elif 'yaml' in content_type:
decode_type = 'yaml'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'yaml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = ('Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)))
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = hubblestack.utils.json.loads(result_text)
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
elif decode_type == 'yaml':
ret['dict'] = hubblestack.utils.data.decode(
hubblestack.utils.yaml.safe_load(result_text))
else:
text = True
if decode_out:
with hubblestack.utils.files.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
def url_encode(params):
encoded = []
for k, v in params:
encoded.append((to_bytes(k), to_bytes(v)))
return to_unicode(_urlencode(encoded))
def __call__(self, group, command, args={}):
"""
Queries Bittrex with given method and args
- encodes and sends <command> with optional [args] to Poloniex api
- raises 'bittrex.BittrexError' if an api key or secret is missing
(and the command is 'private') or if the <command> is not valid
- returns decoded json api message
:param group: Param for queries classification in API
:type command: str
:param command: Query method for getting info
:type command: str
:param args: Extra options for query
:type args: dict
:return: JSON response from Bittrex
:rtype : dict
"""
from requests.exceptions import ReadTimeout, ConnectionError
base_url = 'https://bittrex.com/Api/v2.0/'
if command in PRIVATE_COMMANDS:
if not self.api_key or not self.api_secret:
raise BittrexError("Key and Secret needed!")
url = base_url + 'key/{}/{}?'.format(group, command)
args['nonce'] = self.nonce
args['apikey'] = self.api_key
url += _urlencode(args)
if self.debug_endpoint == True:
print(url)
sign = _new(self.api_secret.encode('utf-8'), url.encode('utf-8'),
_sha512).hexdigest()
# post request
ret = _get(url, headers={'apisign': sign}, timeout=self.timeout)
if ret.status_code != 200:
raise BittrexError("Status Code: %s" % ret.status_code)
jsonout = _loads(ret.text,
parse_float=self.parse_float,
parse_int=self.parse_int)
return jsonout
elif command in PUBLIC_COMMANDS:
base_url += 'pub/{}/'.format(group)
url = base_url + command + '?' + _urlencode(args)
if self.debug_endpoint == True:
print(url)
ret = _get(url, timeout=self.timeout)
if ret.status_code != 200:
raise BittrexError("Status Code: %s" % ret.status_code)
jsonout = _loads(ret.text,
parse_float=self.parse_float,
parse_int=self.parse_int)
return jsonout
else:
raise BittrexError("Invalid Command: %s" % command)
def urlencode(parameters, doseq=False):
return _urlencode(parameters, doseq).replace('&', '&')
def facetDict(self, path, fieldname, term, displayValue):
termDict = dict(self._next_request)
facetFilter = set(termDict.get('facet-filter', []))
facetFilter.add("{0}={1}".format(fieldname, term['term']))
termDict['facet-filter'] = list(facetFilter)
result = dict(count=term['count'],
value=term['term'],
link="{0}?{1}".format(
path, urlencode(sorted(termDict.items()))))
if displayValue is not None:
result['displayValue'] = displayValue
return result
urlencode = lambda arg: _urlencode(arg, doseq=True)
MILLIS = Decimal('0.001')
class MissingArgument(ValueError):
def __init__(self, argument):
ValueError.__init__(
self, "Missing required argument: {}".format(repr(argument)))
class InvalidArgument(ValueError):
def __init__(self, argument, message):
ValueError.__init__(
self, "Invalid argument: {}, {}".format(repr(argument), message))
def urlencode(**kwargs):
return _urlencode(kwargs)
def __call__(self, command, args={}):
"""
Main Api Function
- encodes and sends <command> with optional [args] to Poloniex api
- raises 'ValueError' if an api key or secret is missing
(and the command is 'private'), or if the <command> is not valid
- returns decoded json api message
"""
global PUBLIC_COMMANDS, PRIVATE_COMMANDS
# check in with the coach
if self._coaching:
self.apicoach.wait()
# pass the command
args['command'] = command
# private?
if command in PRIVATE_COMMANDS:
# check for keys
if not self.Key or not self.Secret:
raise ValueError("A Key and Secret needed!")
# set nonce
args['nonce'] = self.nonce
try:
# encode arguments for url
postData = _urlencode(args)
# sign postData with our Secret
sign = _new(
self.Secret.encode('utf-8'),
postData.encode('utf-8'),
_sha512)
# post request
ret = _post(
'https://poloniex.com/tradingApi',
data=args,
headers={
'Sign': sign.hexdigest(),
'Key': self.Key
},
timeout=self.timeout)
except Exception as e:
raise e
finally:
# increment nonce(no matter what)
self.nonce += 1
# return decoded json
try:
return _loads(ret.text, parse_float=unicode)
except NameError:
return _loads(ret.text, parse_float=str)
# public?
elif command in PUBLIC_COMMANDS:
try:
ret = _post(
'https://poloniex.com/public?' + _urlencode(args),
timeout=self.timeout)
except Exception as e:
raise e
try:
return _loads(ret.text, parse_float=unicode)
except NameError:
return _loads(ret.text, parse_float=str)
else:
raise ValueError("Invalid Command!")
def urlencode(**kwargs):
return _urlencode(kwargs)
def urlencode(parameters, doseq=False):
return _urlencode(parameters, doseq)
def urlencode(query, doseq):
return _urlencode(query, doseq).encode('ascii')
def urlencode(query, doseq):
s = _urlencode(query, doseq)
if not isinstance(s, bytes):
s = s.encode("ascii")
return s
def __call__(self, method, command, args={}):
if not self.key or not self.secret:
raise Exception("A Key and Secret needed!")
# Signature generation
method_name = method
if method == 'Get':
content_type = ''
else:
content_type = 'application/json'
now = self.nonce
message_to_encrypt = method_name + self.url + command + content_type + now
message_to_encrypt = message_to_encrypt.lower()
signature = hmac.new(self.secret.encode(),
msg=message_to_encrypt.encode(),
digestmod=hashlib.sha256).digest()
signature_base64 = base64.b64encode(signature, altchars=None)
headers = {
'API_PUBLIC_KEY': self.key,
'API_REQUEST_SIGNATURE': signature_base64,
'API_REQUEST_DATE': now,
'Content-Type': content_type
}
successful = False
if method == 'Post':
data = json.dumps(args)
logger.debug("Gatecoin - Request Post: %s with %s",
self.url + command, data)
response = requests.post(self.url + command,
data=data,
headers=headers,
timeout=self.timeout)
successful = True
# put
elif method == 'Put':
data = json.dumps(args)
logger.debug("Gatecoin - Request Put: %s with %s",
self.url + command, data)
response = requests.put(self.url + command,
data=data,
headers=headers,
timeout=self.timeout)
successful = True
# get
elif method == 'Get':
logger.debug("Gatecoin - Request Get: %s",
self.url + command + _urlencode(args))
response = requests.get(self.url + command + _urlencode(args),
headers=headers,
timeout=self.timeout)
successful = True
# Delete
elif method == 'Delete':
logger.debug("Gatecoin - Request Delete: %s",
self.url + _urlencode(args))
response = requests.delete(self.url + command + _urlencode(args),
headers=headers,
timeout=self.timeout)
successful = True
else:
raise Exception("Gatecoin - Invalid Command!: " + command)
if successful is False or response is None:
logger.error("Gatecoin - Request not succesful")
raise Exception("Gatecoin - Request not succesful")
if response.status_code != 200:
logger.error("Gatecoin - Response error: HTTP " +
str(response.status_code) +
" returned with the content " + str(response.text) +
".")
raise Exception("Gatecoin - Response error: HTTP " +
str(response.status_code) +
" returned with the content " +
str(response.text) + ".")
# decode json
if not self.jsonNums:
response_json = json.loads(response.text, parse_float=str)
else:
response_json = json.loads(response.text,
parse_float=self.jsonNums,
parse_int=self.jsonNums)
if response_json['responseStatus']['message'] != 'OK':
logger.error("Gatecoin - Response contains a functional error: " +
str(response_json))
raise Exception(
"Gatecoin - Response contains a functional error: " +
str(response_json))
# check if gatecoin returned an error
# Commented as it should be case specific error handling
#if 'error' in response_json:
# raise Exception(response_json['error'])
return response_json
