def _build_request(self, query):
if self.method == "GET":
if '?' in self.endpoint:
separator = '&'
else:
separator = '?'
uri = self.endpoint.strip() + separator + query
return urllib2.Request(uri.encode('ASCII'))
else:
uri = self.endpoint.strip().encode('ASCII')
return urllib2.Request(uri, data=query)
def fetchBitcoinChartsData():
global ba
if 'bitcoincharts' not in ba.api_parsers.API_QUERY_CACHE:
ba.api_parsers.API_QUERY_CACHE['bitcoincharts'] = {
'last_call_timestamp': 0,
'result': None,
'call_fail_count': 0,
}
current_timestamp = int(time.time())
if (ba.api_parsers.API_QUERY_CACHE['bitcoincharts']['last_call_timestamp']
+ ba.api_parsers.API_QUERY_FREQUENCY['bitcoincharts'] >
current_timestamp):
result = ba.api_parsers.API_QUERY_CACHE['bitcoincharts']['result']
else:
with Timeout(API_CALL_TIMEOUT_THRESHOLD, CallTimeoutException):
response = urllib2.urlopen(
urllib2.Request(url=BITCOIN_CHARTS_API_URL,
headers=API_REQUEST_HEADERS)).read()
result = json.loads(response)
ba.api_parsers.API_QUERY_CACHE['bitcoincharts'] = {
'last_call_timestamp': current_timestamp,
'result': result,
'call_fail_count': 0,
}
return result
def base_request(self, method, container=None, name=None, prefix=None,
headers={}, proxy=None, contents=None, full_listing=None):
# Common request method
url = self.url
if self.token:
headers['X-Auth-Token'] = self.token
if container:
url = '%s/%s' % (url.rstrip('/'), quote(container))
if name:
url = '%s/%s' % (url.rstrip('/'), quote(name))
url += '?format=json'
if prefix:
url += '&prefix=%s' % prefix
if proxy:
proxy = urlparse.urlparse(proxy)
proxy = urllib2.ProxyHandler({proxy.scheme: proxy.netloc})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
req = urllib2.Request(url, headers=headers, data=contents)
req.get_method = lambda: method
urllib2.urlopen(req)
conn = urllib2.urlopen(req)
body = conn.read()
try:
body_data = json.loads(body)
except ValueError:
body_data = None
return [None, body_data]
def url_request(url, headers=None):
if headers is None:
headers = default_url_headers()
req = urllib2.Request(url, headers=headers)
data = urllib2.urlopen(req)
return data
def _trigger(self, check, incident_key, priority):
headers = {'Content-Type': 'application/json'}
data = json.dumps({
'service_key':
self.service_keys[priority],
'incident_key':
incident_key,
'event_type':
'trigger',
'description':
'%s on %s is DOWN' % (check['check'], check['hostname']),
'details':
check
})
try:
req = urllib2.Request(self.url, data, headers)
response = urllib2.urlopen(req, timeout=10)
result = json.loads(response.read())
response.close()
if 'status' in result:
if result['status'] == 'success':
self.logger.info('Triggered pagerduty event: %s' % result)
return True
else:
self.logger.info('Failed to trigger pagerduty event: %s' %
result)
return False
else:
self.logger.info('Failed to trigger pagerduty event: %s' %
result)
return False
except Exception:
self.logger.exception('Error triggering pagerduty event.')
return False
def _send_email(self, check):
check_name = check['check']
hostname = check['hostname']
if check['status'] is True:
status = 'UP'
else:
status = 'DOWN'
subject = "[stalker] %s on %s is %s" % (check_name, hostname, status)
data = {
"from": "Stalker <%s>" % self.from_addr,
"to": self.recipients,
"subject": subject,
"text": "%s" % check
}
headers = {
'Authorization': 'Basic %s' % self.basic_auth_creds,
'Content-Type': 'application/x-www-form-urlencoded'
}
try:
post_data = urllib.urlencode(data)
req = urllib2.Request(self.url, post_data, headers)
response = urllib2.urlopen(req)
result = response.read()
response.close()
self.logger.info('Mailgun: %s' % result)
return True
except Exception:
self.logger.exception('Mailgun notification error.')
return False
def __call__(self, req):
if req.method not in ("POST", "PUT", "DELETE", "COPY"):
return self.app
self.logger.debug("entered notification middlware")
obj = None
try:
(version, account, container, obj) = \
split_path(req.path_info, 4, 4, True)
except ValueError:
# not an object request
return self.app
resp = req.get_response(self.app)
if obj and is_success(resp.status_int) and \
req.method in ("POST", "PUT", "DELETE", "COPY"):
notification_server = self.server
if notification_server:
# create a POST request with obj name as body
data = json.dumps([{"ttl": 800, "body": {"account": account,
"container": container, "object": obj,
"method": req.method}}])
event_req = urllib2.Request(notification_server, data)
event_req.add_header('Content-Type', 'application/json')
event_req.add_header('Client-ID', self.client_id)
with Timeout(20):
try:
response = urllib2.urlopen(event_req)
except (Exception, Timeout):
self.logger.exception(
'failed to POST object %s' % obj)
else:
self.logger.info(
'successfully posted object %s' % obj)
return resp
def updater(self, coin, url_diff, reg_exp=None):
# Generic method to update the exchange rate of a given currency
if self.calculate_profit == True:
try:
#timeout = eventlet.timeout.Timeout(5, Exception(''))
logging.info('Updating Exchange Rate of ' + coin)
useragent = {
'User-Agent':
self.bitHopper.config.get('main', 'work_user_agent')
}
req = urllib2.Request(url_diff, headers=useragent)
response = urllib2.urlopen(req)
if reg_exp == None:
output = response.read()
else:
diff_str = response.read()
output = re.search(reg_exp, diff_str)
output = output.group(1)
self.rate[coin] = float(output)
logging.debug('Retrieved Exchange rate for ' + str(coin) +
': ' + output)
except Exception, e:
logging.debug('Unable to update exchange rate for ' + coin +
': ' + str(e))
self.rate[coin] = 0.0
finally:
def notify_master(self):
"""Send master our config"""
target = '%s/register' % (self.master_url)
data = json.dumps({
'hostname': self.hostname,
'checks': self.scripts,
'roles': self.roles
})
req = urllib2.Request(target, data,
{'Content-Type': 'application/json'})
req.add_header("X-REGISTER-KEY", self.register_key)
try:
r = urllib2.urlopen(req)
if r.code / 200 != 1:
self.logger.error('Error notify master: %d status' % r.code)
return False
else:
headers = r.info().dict
text = r.read()
r.close()
self.logger.info('Notified master: %s %s %d' %
(headers, text, r.code))
return True
except Exception as err:
self.logger.error('Error notifying master: %s' % err)
return False
def updater(self, coin, short_coin):
# Generic method to update the difficulty of a given currency
logging.info('Updating Difficulty of ' + coin)
config_diffcoin = [
site for site in self.diff_sites if site['coin'] == short_coin
]
#timeout = eventlet.timeout.Timeout(5, Exception(''))
useragent = {
'User-Agent': self.bitHopper.config.get('main', 'work_user_agent')
}
for site in config_diffcoin:
try:
req = urllib2.Request(site['url'], headers=useragent)
response = urllib2.urlopen(req)
if site['get_method'] == 'direct':
output = response.read()
elif site['get_method'] == 'regexp':
diff_str = response.read()
output = re.search(site['pattern'], diff_str)
output = output.group(1)
elif site['get_method'] == 'json':
pass
self.diff[short_coin] = float(output)
logging.debug('Retrieved Difficulty: ' + str(self[short_coin]))
break
except Exception, e:
logging.debug('Unable to update difficulty for ' + coin +
' from ' + site['url'] + ' : ' + str(e))
finally:
def test_copy_dest(self):
# No COPY in swiftclient. Using urllib instead...
url = os.path.join(self.url, self.container, self.storlet_file)
objname = self.storlet_file + '-copy-ex'
headers = {
'X-Auth-Token': self.token,
'X-Run-Storlet': self.storlet_name,
'Destination': '%s/%s' % (self.container, objname)
}
headers.update(self.additional_headers)
req = urllib2.Request(url, headers=headers)
req.get_method = lambda: 'COPY'
conn = urllib2.urlopen(req, timeout=10)
self.assertEqual(201, conn.getcode())
self.assertEqual('%s/%s' % (self.container, self.storlet_file),
conn.info()['x-storlet-generated-from'])
self.assertEqual(self.acct,
conn.info()['x-storlet-generated-from-account'])
self.assertIn('x-storlet-generated-from-last-modified', conn.info())
headers = client.head_object(self.url, self.token, self.container,
objname)
self.assertEqual(str(len(self.content)), headers['content-length'])
resp = dict()
client.delete_object(self.url,
self.token,
self.container,
objname,
response_dict=resp)
self.assertEqual(204, resp['status'])
def get_sync(self,
url,
data,
type=None,
content_type="application/x-www-form-urlencoded"):
try:
data = urlencode(data)
except: # data is probably a string to be send directly.
pass
headers = {"Content-Type": content_type}
if type and type.upper() not in ("POST", "GET"):
from restlib import RestfulRequest #@UnresolvedImport
req = RestfulRequest(url, data=data, method=type.upper())
else:
req = urllib2.Request(url, data, headers=headers)
opener = urllib2.build_opener(self)
eventlet.greenthread.sleep()
try:
f = opener.open(req, data=data)
if f.code is None or str(f.code)[0] == "2":
dispatcher.send(UrlGetter.HTTP_RESULT,
self,
result=f.read(),
source=url,
code=f.code)
else:
e = urllib2.HTTPError(
url, f.code,
"A code %s HTTP error has occurred when trying to send to target %s"
% (f.code, url), req.headers, f)
dispatcher.send(UrlGetter.HTTP_ERROR, self, exception=e)
# TODO: make sure we're supposed to listen to URLErrors
except (urllib2.URLError, ValueError), e:
dispatcher.send(UrlGetter.URL_ERROR, self, exception=e, url=url)
def get_data(self, argv):
# argv = {"urls" : [], "worker" : , }
content = None
error_code = None
self.logger.debug("start fetch " + argv["url"])
try:
url = argv["url"]
try:
with eventlet.Timeout(self.timeout, False):
headers = {
"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1"
}
if self.proxy is None:
req = urllib2.Request(url, headers = headers)
res = urllib2.urlopen(req)
content = res.read()
else:
proxy_handler = urllib2.ProxyHandler(self.proxy)
opener = urllib2.build_opener(proxy_handler)
header_list = []
for header in headers:
header_list.append((header, headers[header]))
opener.addheaders = header_list
res = opener.open(url)
content = res.read()
except urllib2.HTTPError, e:
raise Exception(e.code)
except urllib2.URLError, e:
raise Exception("URLError")
def scout_server_type(self, host):
"""
Obtain Server header by calling OPTIONS.
:param host: host to check
:returns: Server type, status
"""
try:
url = "http://%s:%s/" % (host[0], host[1])
req = urllib2.Request(url)
req.get_method = lambda: 'OPTIONS'
conn = urllib2.urlopen(req)
header = conn.info().getheader('Server')
server_header = header.split('/')
content = server_header[0]
status = 200
except urllib2.HTTPError as err:
if not self.suppress_errors or self.verbose:
print("-> %s: %s" % (url, err))
content = err
status = err.code
except urllib2.URLError as err:
if not self.suppress_errors or self.verbose:
print("-> %s: %s" % (url, err))
content = err
status = -1
return url, content, status
def write_html_currency_pages():
global ba
today = datetime.datetime.today()
template_file_path = os.path.join(ba.server.WWW_DOCUMENT_ROOT, '_currency_page_template.htm')
with open(template_file_path, 'r') as template_file:
template = template_file.read()
api_all_url = '{}ticker/all'.format(ba.server.API_INDEX_URL)
try:
with Timeout(API_CALL_TIMEOUT_THRESHOLD, CallTimeoutException):
response = urllib2.urlopen(urllib2.Request(url=api_all_url, headers=API_REQUEST_HEADERS)).read()
all_rates = json.loads(response)
except (CallTimeoutException,
socket.error,
urllib2.URLError,
httplib.BadStatusLine,
simplejson.decoder.JSONDecodeError):
return None
if not os.path.exists(os.path.join(ba.server.WWW_DOCUMENT_ROOT, ba.config.CURRENCY_DUMMY_PAGES_SUBFOLDER_NAME)):
os.makedirs(os.path.join(ba.server.WWW_DOCUMENT_ROOT, ba.config.CURRENCY_DUMMY_PAGES_SUBFOLDER_NAME))
for currency_code in ba.config.CURRENCY_LIST:
currency_rate = all_rates[currency_code]['last']
currency_page_contents = template
currency_page_contents = currency_page_contents.replace('$RATE$', str(Decimal(currency_rate).quantize(ba.config.DEC_PLACES)))
currency_page_contents = currency_page_contents.replace('$CURRENCY_CODE$', currency_code)
currency_page_contents = currency_page_contents.replace('$GENERATION_DATETIME$', today.strftime('%Y-%m-%dT%H:%M'))
with open(os.path.join(ba.server.WWW_DOCUMENT_ROOT,
ba.config.CURRENCY_DUMMY_PAGES_SUBFOLDER_NAME,
('%s.htm' % currency_code.lower())), 'w') as currency_page_file:
currency_page_file.write(currency_page_contents)
template_file_path = os.path.join(ba.server.WWW_DOCUMENT_ROOT, '_charts_page_template.htm')
with open(template_file_path, 'r') as template_file:
template = template_file.read()
if not os.path.exists(os.path.join(ba.server.WWW_DOCUMENT_ROOT, ba.config.CHARTS_DUMMY_PAGES_SUBFOLDER_NAME)):
os.makedirs(os.path.join(ba.server.WWW_DOCUMENT_ROOT, ba.config.CHARTS_DUMMY_PAGES_SUBFOLDER_NAME))
index = 0
for currency_code in ba.config.CURRENCY_LIST:
currency_rate = all_rates[currency_code]['last']
chart_page_contents = template
chart_page_contents = chart_page_contents.replace('$RATE$', str(Decimal(currency_rate).quantize(ba.config.DEC_PLACES)))
chart_page_contents = chart_page_contents.replace('$CURRENCY_CODE$', currency_code)
chart_page_contents = chart_page_contents.replace('$GENERATION_DATETIME$', today.strftime('%Y-%m-%dT%H:%M'))
with open(os.path.join(ba.server.WWW_DOCUMENT_ROOT,
ba.config.CHARTS_DUMMY_PAGES_SUBFOLDER_NAME,
('%s.htm' % currency_code.lower())), 'w') as chart_page_file:
chart_page_file.write(chart_page_contents)
index = index + 1
if index == ba.config.FRONTEND_MAJOR_CURRENCIES:
break
def fetch_from(method, url, headers):
print '>> Requesting: %s' % url
start = time.time()
request = urllib2.Request(url=url, headers=headers)
try:
return urllib2.urlopen(request)
finally:
print ('fetch_from(%s, %s, ..) took %s' % (method, url, (time.time() - start)))
def myfetch(myurl, i):
req = urllib2.Request(myurl)
req.add_header('User-agent', 'Mozilla 5.10')
res = urllib2.urlopen(req, timeout=4)
body = res.read()
size = len(body)
print(i, 'body size', size)
return size
def write_fiat_rates_config():
global ba
js_config_template = "var fiatCurrencies = $FIAT_CURRENCIES_DATA$;"
currencies_names_URL = 'http://openexchangerates.org/api/currencies.json'
currencies_rates_URL = 'http://openexchangerates.org/api/latest.json?app_id={app_id}'.format(
app_id=OPENEXCHANGERATES_APP_ID)
currency_data_list = {}
try:
with Timeout(API_CALL_TIMEOUT_THRESHOLD, CallTimeoutException):
response = urllib2.urlopen(
urllib2.Request(url=currencies_names_URL,
headers=API_REQUEST_HEADERS)).read()
currencies_names = json.loads(response)
with Timeout(API_CALL_TIMEOUT_THRESHOLD, CallTimeoutException):
response = urllib2.urlopen(
urllib2.Request(url=currencies_rates_URL,
headers=API_REQUEST_HEADERS)).read()
currencies_rates = json.loads(response)
except (CallTimeoutException, socket.error, urllib2.URLError,
httplib.BadStatusLine, ValueError):
return None
for currency_code in currencies_names:
try:
currency_data_list[currency_code] = {
'name': currencies_names[currency_code],
'rate': str(currencies_rates['rates'][currency_code]),
}
except (KeyError, TypeError):
return None
config_string = js_config_template
config_string = config_string.replace('$FIAT_CURRENCIES_DATA$',
json.dumps(currency_data_list))
with open(os.path.join(ba.server.WWW_DOCUMENT_ROOT, 'js', 'fiat_data.js'),
'w') as fiat_exchange_config_file:
fiat_exchange_config_file.write(config_string)
with open(os.path.join(ba.server.API_DOCUMENT_ROOT, 'fiat_data'),
'w') as fiat_exchange_api_file:
fiat_exchange_api_file.write(json.dumps(currency_data_list))
def fetch_feed(feed):
req = urllib2.Request(feed.feed_url)
if feed.etag:
req.add_header('If-None-Match', feed.etag)
try:
handle = urllib2.urlopen(req)
except (urllib2.HTTPError, urllib2.URLError):
return (None, None, None)
return feed, handle.info().getheader('ETag'), handle.read()
def Request(url, req_timeout=60, req_data=None, req_headers=settings.DEFAULT_REQUEST_HEADERS):
body, status, response = 'None', '200', None
request = urllib2.Request(url, req_data, req_headers)
try:
response = urllib2.urlopen(request)
body = response.read()
response.close()
except urllib2.HTTPError, e:
status = e.code
def get_auth(url, user, key, auth_version='1.0', **kwargs):
if auth_version != '1.0':
exit('ERROR: swiftclient missing, only auth v1.0 supported')
req = urllib2.Request(url)
req.add_header('X-Auth-User', user)
req.add_header('X-Auth-Key', key)
conn = urllib2.urlopen(req)
headers = conn.info()
return (headers.getheader('X-Storage-Url'),
headers.getheader('X-Auth-Token'))
def _send_payload(self, payload):
req = eventlet_urllib2.Request(self._url, headers=payload[1])
try:
if sys.version_info < (2, 6):
response = eventlet_urllib2.urlopen(req, payload[0]).read()
else:
response = eventlet_urllib2.urlopen(req, payload[0], self.timeout).read()
return response
except Exception as err:
return err
def _get_remote_stats(clusterid):
target = app.config['GLOBAL_CLUSTERS'][clusterid]['host'] + '/stats'
headers = {'X-API-KEY': app.config['GLOBAL_CLUSTERS'][clusterid]['key']}
try:
req = urllib2.Request(target, headers=headers)
res = urllib2.urlopen(req, timeout=app.config['REMOTE_TIMEOUT'])
return json.loads(res.read())[clusterid]
except Exception as err:
logger.exception("Error grabbing stats for %s: %s" % (clusterid, err))
return None
def calculateAllGlobalAverages(calculated_average_rates, total_currency_volumes):
def getCurrencyCrossRate(currency_from, currency_to):
if currency_from == currency_to:
return Decimal(1)
rate_from = Decimal(fiat_currencies_list[currency_from]['rate'])
rate_to = Decimal(fiat_currencies_list[currency_to]['rate'])
return (rate_from / rate_to)
fiat_exchange_rates_url = server.API_INDEX_URL + 'fiat_data'
try:
with Timeout(API_CALL_TIMEOUT_THRESHOLD, CallTimeoutException):
result = urllib2.urlopen(urllib2.Request(url=fiat_exchange_rates_url, headers=API_REQUEST_HEADERS)).read()
fiat_currencies_list = json.loads(result)
except (KeyError,ValueError,socket.error,simplejson.decoder.JSONDecodeError,urllib2.URLError,httplib.BadStatusLine,CallTimeoutException):
return {}, {}
global_volume = DEC_PLACES
for currency in CURRENCY_LIST:
global_volume = global_volume + total_currency_volumes[currency]
global_volume_percents = {}
for currency in CURRENCY_LIST:
global_volume_percents[currency] = (total_currency_volumes[currency] / global_volume * Decimal(100)).quantize(DEC_PLACES)
global_averages = {}
for currency_local in fiat_currencies_list:
global_averages[currency_local] = {'last': DEC_PLACES,
'ask': DEC_PLACES,
'bid': DEC_PLACES,
}
for currency_to_convert in CURRENCY_LIST:
global_averages[currency_local]['last'] = ( global_averages[currency_local]['last']
+ (calculated_average_rates[currency_to_convert]['last']
* global_volume_percents[currency_to_convert] / Decimal(100)
* getCurrencyCrossRate(currency_local, currency_to_convert) )
)
global_averages[currency_local]['bid'] = ( global_averages[currency_local]['bid']
+ (calculated_average_rates[currency_to_convert]['bid']
* global_volume_percents[currency_to_convert] / Decimal(100)
* getCurrencyCrossRate(currency_local, currency_to_convert) )
)
global_averages[currency_local]['ask'] = ( global_averages[currency_local]['ask']
+ (calculated_average_rates[currency_to_convert]['ask']
* global_volume_percents[currency_to_convert] / Decimal(100)
* getCurrencyCrossRate(currency_local, currency_to_convert) )
)
global_averages[currency_local]['last'] = global_averages[currency_local]['last'].quantize(DEC_PLACES)
global_averages[currency_local]['bid'] = global_averages[currency_local]['bid'].quantize(DEC_PLACES)
global_averages[currency_local]['ask'] = global_averages[currency_local]['ask'].quantize(DEC_PLACES)
currency_local_24h_avg = get24hGlobalAverage(currency_local)
if currency_local_24h_avg > DEC_PLACES:
global_averages[currency_local]['24h_avg'] = currency_local_24h_avg
return global_averages, global_volume_percents
def fetchSubscribeUrl(sURL, currposition):
pollingURL = "{}{}{}".format(sURL, '&stream_position=', currposition)
print "opening long polling URL : ", pollingURL
print "long polling ..... "
req = urllib2.Request(pollingURL, None, headers=HEADERS)
response = urllib2.urlopen(req)
html = response.read()
json_obj = json.loads(html)
print json_obj["message"]
return json_obj["message"]
def _sendJson(url, data, method='POST', noResponse=False):
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
request = urllib2.Request(url, json.dumps(data), headers)
request.get_method = lambda: method
response = urllib2.urlopen(request)
if not noResponse:
return json.load(response)
def _sendData(url,
data,
method='POST',
mime='application/octet-stream',
noResponse=False):
headers = {'Content-Type': mime}
request = urllib2.Request(url, data, headers)
request.get_method = lambda: method
response = urllib2.urlopen(request)
if not noResponse:
return response.read()
def test_get_with_retries(self, request, urlopen):
# First request fails, retry successful
request.return_value.get_type.return_value = "http"
urlopen.return_value.read.return_value = ''
req = urllib2.Request('http://127.0.0.1', method='GET')
request.side_effect = [urllib2.URLError(''), req]
sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1)
retval = sc.retry_request('GET')
self.assertEqual(request.call_count, 3)
request.assert_called_with('http://127.0.0.1?format=json',
data=None,
headers={'X-Auth-Token': 'token'})
self.assertEqual([None, None], retval)
def _get_remote_checks(clusterid, state):
endpoints = {'alerting': '/checks/state/alerting',
'pending': '/checks/state/pending',
'suspended': '/checks/state/suspended',
'in_maintenance': '/checks/state/in_maintenance'}
target = app.config['GLOBAL_CLUSTERS'][clusterid]['host'] + \
endpoints[state]
headers = {'X-API-KEY': app.config['GLOBAL_CLUSTERS'][clusterid]['key']}
try:
req = urllib2.Request(target, headers=headers)
res = urllib2.urlopen(req, timeout=app.config['REMOTE_TIMEOUT'])
return json.loads(res.read())
except Exception as err:
logger.exception("Error grabbing checks for %s: %s" % (clusterid, err))
return None
def CreateJob(self, message):
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
url = self.url / 'conversions'
request = urllib2.Request(url, json.dumps(message), headers)
response = urllib2.urlopen(request)
# Job should have been created.
assert response.code == httplib.CREATED
msg = json.load(response)
return Job(msg['id'], self.url)
