def connect(self, host, port, auth=None):
self.conn = Connection(host, port=port, verify=False)
if auth is not None:
# Use HTTP Basic authentication
s = base64.b64encode("%s:%s" % (auth.username, auth.password))
self.conn.add_headers({'Authorization': 'Basic %s' % s})
logger.info("Authenticated using BASIC")
def get_historical_prices(begin, end, symbol, measures,
resolution='day', date_obj=False):
"""Get historical prices for the given ticker symbol.
Returns a list of dicts keyed by 'date' and measures
:param string begin: begin date of the inquire interval
in the format of YYYY-MM-DD
:param string end: end date of the inquire interval
in the format of YYYY-MM-DD
:param string symbol: symbol of one stock to query
:param list measures: a list of prices that needs to be queried,
should be a subset of ["open", "high", "low", "close", "volume"]
:param string resolution: '1 day' or '5 days'
:param boolean date_obj: dates are converted to datetime objects
from date strings if True. Otherwise, dates are stored as strings
"""
try:
conn = Connection('http://ichart.finance.yahoo.com')
start_month = parse_date(begin).month - 1
start_day = parse_date(begin).day
start_year = parse_date(begin).year
end_month = parse_date(end).month - 1
end_day = parse_date(end).day
end_year = parse_date(end).year
ret = []
params = {'s': symbol,
'a': start_month,
'b': start_day,
'c': start_year,
'd': end_month,
'e': end_day,
'f': end_year,
'g': resolution[0],
'ignore': '.csv'}
resp = conn.request(method='POST', path='/table.csv', params=params)
# extract data and skip first row with column titles
data = list(resp.iter_lines())[1:]
# iterate over the data backwards as the daily prices are sorted
# backwards by the dates
for day in reversed(data):
# day is a string with date, prices, volume separated by commas,
# '<date>,<open>,<high>,<low>,<close>,<volume>,<adj_close>'
# as '2014-02-19,20.22,20.55,20.11,20.50,1599600,20.50'
day = day.split(',')
date = parse_date(day[0]) if date_obj else day[0]
daily_prices = {'date': date}
for m in measures:
if m in mapping:
daily_prices[m] = float(day[mapping[m]])
ret.append(daily_prices)
except RvbdHTTPException:
raise StockApiException("Symbol '%s' is invalid or Stock '%s' was"
" not on market on %s" % (symbol, symbol,
end))
return ret
def run_report_via_rest(host, slug, namespace, title, authfile, verify,
**kwargs):
"""Mimic the front-end javascript to run a whole report.
This also has the added benefit of adding an entry to Report History, and
adding the resulting data to the Widget Cache if those functions
are enabled.
"""
report_url = '/report/%s/%s/' % (namespace, slug)
post_header = {'Content-Type': 'application/x-www-form-urlencoded'}
criteria, options = process_criteria(kwargs)
# since we are posting in place of a criteria form, we need to split time
endtime = criteria.pop('endtime')
endtime_date, endtime_time = endtime.split('T')
criteria['endtime_0'] = endtime_date
criteria['endtime_1'] = endtime_time
conn = Connection(host, auth=get_auth(authfile), verify=verify)
logger.debug('Posting report criteria for report %s: %s' %
(title, criteria))
r = conn.request('POST',
report_url,
extra_headers=post_header,
body=criteria)
if r.ok and 'widgets' in r.json():
logger.debug('Got widgets for Report url %s' % report_url)
else:
logger.error('Error getting Widgets for Report url %s. Aborting.' %
report_url)
return
jobs = []
# create the widget jobs for each widget found
for w in r.json()['widgets']:
data = {'criteria': json.dumps(w['criteria'])}
w_response = conn.request('POST',
w['posturl'],
extra_headers=post_header,
body=data)
jobs.append(w_response.json()['joburl'])
# check until all jobs are done
timeout = options['delta']
interval = 1
wait_for_complete(conn, interval, timeout, jobs)
def run_report_via_rest(host, slug, namespace,
title, authfile, verify, **kwargs):
"""Mimic the front-end javascript to run a whole report.
This also has the added benefit of adding an entry to Report History, and
adding the resulting data to the Widget Cache if those functions
are enabled.
"""
report_url = '/report/%s/%s/' % (namespace, slug)
post_header = {'Content-Type': 'application/x-www-form-urlencoded'}
criteria, options = process_criteria(kwargs)
# since we are posting in place of a criteria form, we need to split time
endtime = criteria.pop('endtime')
endtime_date, endtime_time = endtime.split('T')
criteria['endtime_0'] = endtime_date
criteria['endtime_1'] = endtime_time
conn = Connection(host, auth=get_auth(authfile), verify=verify)
logger.debug('Posting report criteria for report %s: %s' %
(title, criteria))
r = conn.request('POST', report_url, extra_headers=post_header,
body=criteria)
if r.ok and 'widgets' in r.json():
logger.debug('Got widgets for Report url %s' % report_url)
else:
logger.error('Error getting Widgets for Report url %s. Aborting.'
% report_url)
return
jobs = []
# create the widget jobs for each widget found
for w in r.json()['widgets']:
data = {'criteria': json.dumps(w['criteria'])}
w_response = conn.request('POST', w['posturl'],
extra_headers=post_header, body=data)
jobs.append(w_response.json()['joburl'])
# check until all jobs are done
timeout = options['delta']
interval = 1
wait_for_complete(conn, interval, timeout, jobs)
class ProgressDaemon(object):
def __init__(self):
self.conn = Connection(settings.PROGRESSD_HOST,
port=settings.PROGRESSD_PORT)
def _request(self, method, url, **kwargs):
try:
return self.conn.json_request(method, url, **kwargs)
except ConnectionError:
raise RvbdException(
"Error connecting to appfwk service 'progressd': "
"try restarting the service (sudo service progressd restart) "
"or contact your administrator for assistance."
)
def get(self, id_, attr=None):
r = self._request('GET', '/jobs/items/%d/' % id_)
if attr:
return r[attr]
return r
def post(self, **kwargs):
return self._request('POST', '/jobs/', body=kwargs)
def put(self, id_, **kwargs):
self._request('PUT', '/jobs/items/%d/' % id_, body=kwargs)
def delete(self, id_):
self._request('DELETE', '/jobs/items/%d/' % id_)
def reset(self):
self._request('POST', '/jobs/reset/')
class ProgressDaemon(object):
def __init__(self):
self.conn = Connection(settings.PROGRESSD_HOST,
port=settings.PROGRESSD_PORT)
def _request(self, method, url, **kwargs):
try:
return self.conn.json_request(method, url, **kwargs)
except ConnectionError:
raise RvbdException(
"Error connecting to appfwk service 'progressd': "
"try restarting the service (sudo service progressd restart) "
"or contact your administrator for assistance.")
def get(self, id_, attr=None):
r = self._request('GET', '/jobs/items/%d/' % id_)
if attr:
return r[attr]
return r
def post(self, **kwargs):
return self._request('POST', '/jobs/', body=kwargs)
def put(self, id_, **kwargs):
self._request('PUT', '/jobs/items/%d/' % id_, body=kwargs)
def delete(self, id_):
self._request('DELETE', '/jobs/items/%d/' % id_)
def reset(self):
self._request('POST', '/jobs/reset/')
def connect(self, host, port, auth=None):
self.conn = Connection(host, port=port, verify=False)
if auth is not None:
# Use HTTP Basic authentication
s = base64.b64encode("%s:%s" % (auth.username, auth.password))
self.conn.add_headers({"Authorization": "Basic %s" % s})
logger.info("Authenticated using BASIC")
class ProgressDaemon(object):
def __init__(self):
self.conn = Connection(settings.PROGRESSD_HOST,
port=settings.PROGRESSD_PORT)
def _request(self, method, url, **kwargs):
try:
return self.conn.json_request(method, url, **kwargs)
except ConnectionError:
# In case progressd is restarting due to daily logrotate
# resending the request after 10 seconds
logger.warning("Waiting %s seconds before reconnecting progressd"
% settings.PROGRESSD_CONN_TIMEOUT)
time.sleep(settings.PROGRESSD_CONN_TIMEOUT)
try:
return self.conn.json_request(method, url, **kwargs)
except ConnectionError:
raise RvbdException(
"Error connecting to appfwk service 'progressd': "
"try restarting the service "
"(sudo service progressd restart) "
"or contact your administrator for assistance."
)
def get(self, id_, attr=None):
r = self._request('GET', '/jobs/items/%d/' % id_)
if attr:
return r[attr]
return r
def post(self, **kwargs):
return self._request('POST', '/jobs/', body=kwargs)
def put(self, id_, **kwargs):
self._request('PUT', '/jobs/items/%d/' % id_, body=kwargs)
def delete(self, id_):
self._request('DELETE', '/jobs/items/%d/' % id_)
def reset(self):
self._request('POST', '/jobs/reset/')
class ProgressDaemon(object):
def __init__(self):
self.conn = Connection(settings.PROGRESSD_HOST,
port=settings.PROGRESSD_PORT)
def _request(self, method, url, **kwargs):
try:
return self.conn.json_request(method, url, **kwargs)
except ConnectionError:
# In case progressd is restarting due to daily logrotate
# resending the request after 10 seconds
logger.warning("Waiting %s seconds before reconnecting progressd" %
settings.PROGRESSD_CONN_TIMEOUT)
time.sleep(settings.PROGRESSD_CONN_TIMEOUT)
try:
return self.conn.json_request(method, url, **kwargs)
except ConnectionError:
raise RvbdException(
"Error connecting to appfwk service 'progressd': "
"try restarting the service "
"(sudo service progressd restart) "
"or contact your administrator for assistance.")
def get(self, id_, attr=None):
r = self._request('GET', '/jobs/items/%d/' % id_)
if attr:
return r[attr]
return r
def post(self, **kwargs):
return self._request('POST', '/jobs/', body=kwargs)
def put(self, id_, **kwargs):
self._request('PUT', '/jobs/items/%d/' % id_, body=kwargs)
def delete(self, id_):
self._request('DELETE', '/jobs/items/%d/' % id_)
def reset(self):
self._request('POST', '/jobs/reset/')
def run_table_via_rest(host, url, authfile, verify, **kwargs):
criteria, options = process_criteria(kwargs)
conn = Connection(host, auth=get_auth(authfile), verify=verify)
logger.debug('POSTing new job with criteria: %s' % criteria)
r = conn.request('POST', url, body=criteria)
if r.ok:
logger.debug('Job creation successful.')
else:
logger.error('Error creating Job: %s' % r.content)
if options.get('output-file', None):
# check periodically until data is ready and write to file
url = r.headers['Location']
timeout = options['delta']
interval = 1
wait_for_complete(conn, interval, timeout,
[url], [options['output-file']])
else:
# we aren't interested in results
pass
def run_table_via_rest(host, url, authfile, verify, **kwargs):
criteria, options = process_criteria(kwargs)
conn = Connection(host, auth=get_auth(authfile), verify=verify)
logger.debug('POSTing new job with criteria: %s' % criteria)
r = conn.request('POST', url, body=criteria)
if r.ok:
logger.debug('Job creation successful.')
else:
logger.error('Error creating Job: %s' % r.content)
if options.get('output-file', None):
# check periodically until data is ready and write to file
url = r.headers['Location']
timeout = options['delta']
interval = 1
wait_for_complete(conn, interval, timeout,
[url], [options['output-file']])
else:
# we aren't interested in results
pass
def __init__(self):
self.conn = Connection(settings.PROGRESSD_HOST,
port=settings.PROGRESSD_PORT)
class Command(Application):
help = "Interactive shell for issuing REST commands"
def __init__(self, *args, **kwargs):
super(Command, self).__init__(*args, **kwargs)
self.conn = None
self.filelines = []
self.interactive = True
self.jsonmode = True
def parse_args(self):
super(Command, self).parse_args()
def add_positional_args(self):
super(Command, self).add_positional_args()
# self.add_positional_arg('host', 'Rest hostname or IP address')
def add_options(self, parser):
parser.add_option('-f', '--file', help='Read commands from a file')
super(Command, self).add_options(parser)
self.add_standard_options(conn=False, log=True)
def connect(self, host, port, auth=None):
self.conn = Connection(host, port=port, verify=False)
if auth is not None:
# Use HTTP Basic authentication
s = base64.b64encode("%s:%s" % (auth.username, auth.password))
self.conn.add_headers({'Authorization': 'Basic %s' % s})
logger.info("Authenticated using BASIC")
def raw_input_no_history(self, prompt):
if self.filelines:
return self.filelines.pop(0)
else:
input = raw_input(prompt)
readline.remove_history_item(
readline.get_current_history_length() - 1
)
return input
def main(self):
self.history = HistoryManager()
history = self.history
history.add_context('cmd',
os.path.join(os.path.expanduser('~'),
'.steelscript/rest-cmd-history'))
history.add_context('body',
os.path.join(os.path.expanduser('~'),
'.steelscript/rest-obj-history'),
obj=True)
history.set_context('cmd')
atexit.register(HistoryManager.save, history)
cmd = RootCommand()
self.cmd = cmd
GET(self, cmd)
POST(self, cmd)
PUT(self, cmd)
DELETE(self, cmd)
Connect(self, cmd)
Help(self, cmd)
Mode(self, cmd)
if self.options.file:
with open(self.options.file, 'r') as f:
self.filelines = [line.rstrip() for line in f]
print "REST Shell ('help' or 'quit' when done)"
print "Current mode is 'json', use 'mode text' to switch to raw text"
while True:
if self.filelines:
line = self.filelines.pop(0)
self.interactive = False
print "Command: %s" % line
else:
try:
if self.conn is not None:
prompt = "%s> " % self.conn.hostname
else:
prompt = "> "
line = raw_input(prompt)
self.interactive = True
except EOFError:
print ""
break
if not line:
continue
args = shlex.split(line)
if args[0].lower() == 'quit':
break
logger.info('Command: %s' % line)
try:
cmd.parse(args)
except SystemExit as e:
pass
except Exception as e:
print ('Command raised an error, see log for traceback:\n%s\n'
% str(e))
logger.exception("Command raised an error")
def __init__(self):
self.conn = Connection(settings.PROGRESSD_HOST,
port=settings.PROGRESSD_PORT)
def get_job_function(self, kwargs):
# allow for alternate methods to call table commands in future
# possibly direct API calls, or REST calls against running server
if self.options.rest_server:
if 'table-name' in kwargs:
# find id of table-name
name = kwargs['table-name']
baseurl = '/data/tables/'
verify = not self.options.insecure
conn = Connection(self.options.rest_server,
auth=get_auth(self.options.authfile),
verify=verify)
tableurl = None
url = baseurl
logger.debug('Getting available tables from server ...')
while tableurl is None:
r = conn.json_request('GET', url)
results = r['results']
urls = [t['url'] for t in results if t['name'] == name]
if len(urls) > 1:
msg = ('ERROR: Multiple tables found for name %s: %s' %
(name, urls))
logger.debug(msg)
raise ValueError(msg)
elif len(urls) == 1:
# parse out just the path component of the url
logger.debug('Found table url: %s' % urls)
fullurl = urljoin(urls[0], 'jobs/')
tableurl = urlsplit(fullurl).path
else:
url = r['next']
if url is None:
msg = ('ERROR: No table found for "%s"' % name)
logger.debug(msg)
raise ValueError(msg)
job_params = {
'func': run_table_via_rest,
'args': [self.options.rest_server, tableurl,
self.options.authfile, verify]
}
elif 'report-slug' in kwargs:
# find id of report
slug = kwargs['report-slug']
namespace = kwargs['report-namespace']
baseurl = urljoin(self.options.rest_server, '/report/')
verify = not self.options.insecure
conn = Connection(self.options.rest_server,
auth=get_auth(self.options.authfile),
verify=verify)
title = None
logger.debug('Getting available reports from server ...')
r = conn.json_request('GET', baseurl)
for report in r:
if slug in report.values():
title = report['title']
break
if title is None:
msg = ('No report found for slug %s namespace %s'
% (slug, namespace))
logger.error(msg)
raise ValueError(msg)
job_params = {
'func': run_report_via_rest,
'args': [self.options.rest_server, slug,
namespace, title, self.options.authfile, verify]
}
else:
raise ValueError('Invalid config, no "table-name" or '
'"report-slug"/"report-namespace" specified')
else:
job_params = {'func': run_table,
'args': ['table']}
logger.debug('Calculated job params: %s' % job_params)
return job_params
def get_job_function(self, kwargs):
# allow for alternate methods to call table commands in future
# possibly direct API calls, or REST calls against running server
if self.options.rest_server:
if 'table-name' in kwargs:
# find id of table-name
name = kwargs['table-name']
baseurl = '/data/tables/'
verify = not self.options.insecure
conn = Connection(self.options.rest_server,
auth=get_auth(self.options.authfile),
verify=verify)
tableurl = None
url = baseurl
logger.debug('Getting available tables from server ...')
while tableurl is None:
r = conn.json_request('GET', url)
results = r['results']
urls = [t['url'] for t in results if t['name'] == name]
if len(urls) > 1:
msg = ('ERROR: Multiple tables found for name %s: %s' %
(name, urls))
logger.debug(msg)
raise ValueError(msg)
elif len(urls) == 1:
# parse out just the path component of the url
logger.debug('Found table url: %s' % urls)
fullurl = urljoin(urls[0], 'jobs/')
tableurl = urlsplit(fullurl).path
else:
url = r['next']
if url is None:
msg = ('ERROR: No table found for "%s"' % name)
logger.debug(msg)
raise ValueError(msg)
job_params = {
'func': run_table_via_rest,
'args': [self.options.rest_server, tableurl,
self.options.authfile, verify]
}
elif 'report-slug' in kwargs:
# find id of report
slug = kwargs['report-slug']
namespace = kwargs['report-namespace']
baseurl = urljoin(self.options.rest_server, '/report/')
verify = not self.options.insecure
conn = Connection(self.options.rest_server,
auth=get_auth(self.options.authfile),
verify=verify)
title = None
logger.debug('Getting available reports from server ...')
r = conn.json_request('GET', baseurl)
for report in r:
if slug in report.values():
title = report['title']
break
if title is None:
msg = ('No report found for slug %s namespace %s'
% (slug, namespace))
logger.error(msg)
raise ValueError(msg)
job_params = {
'func': run_report_via_rest,
'args': [self.options.rest_server, slug,
namespace, title, self.options.authfile, verify]
}
else:
raise ValueError('Invalid config, no "table-name" or '
'"report-slug"/"report-namespace" specified')
else:
job_params = {'func': run_table,
'args': ['table']}
logger.debug('Calculated job params: %s' % job_params)
return job_params
class Command(Application):
help = "Interactive shell for issuing REST commands"
def __init__(self, *args, **kwargs):
super(Command, self).__init__(*args, **kwargs)
self.conn = None
self.filelines = []
self.interactive = True
self.jsonmode = True
def parse_args(self):
super(Command, self).parse_args()
def add_positional_args(self):
super(Command, self).add_positional_args()
# self.add_positional_arg('host', 'Rest hostname or IP address')
def add_options(self, parser):
parser.add_option("-f", "--file", help="Read commands from a file")
super(Command, self).add_options(parser)
self.add_standard_options(conn=False, log=True)
def connect(self, host, port, auth=None):
self.conn = Connection(host, port=port, verify=False)
if auth is not None:
# Use HTTP Basic authentication
s = base64.b64encode("%s:%s" % (auth.username, auth.password))
self.conn.add_headers({"Authorization": "Basic %s" % s})
logger.info("Authenticated using BASIC")
def raw_input_no_history(self, prompt):
if self.filelines:
return self.filelines.pop(0)
else:
input = raw_input(prompt)
readline.remove_history_item(readline.get_current_history_length() - 1)
return input
def main(self):
self.history = HistoryManager()
history = self.history
history.add_context("cmd", os.path.join(os.path.expanduser("~"), ".steelscript/rest-cmd-history"))
history.add_context("body", os.path.join(os.path.expanduser("~"), ".steelscript/rest-obj-history"), obj=True)
history.set_context("cmd")
atexit.register(HistoryManager.save, history)
cmd = RootCommand()
self.cmd = cmd
GET(self, cmd)
POST(self, cmd)
PUT(self, cmd)
DELETE(self, cmd)
Connect(self, cmd)
Help(self, cmd)
Mode(self, cmd)
if self.options.file:
with open(self.options.file, "r") as f:
self.filelines = [line.rstrip() for line in f]
print "REST Shell ('help' or 'quit' when done)"
print "Current mode is 'json', use 'mode text' to switch to raw text"
while True:
if self.filelines:
line = self.filelines.pop(0)
self.interactive = False
print "Command: %s" % line
else:
try:
if self.conn is not None:
prompt = "%s> " % self.conn.hostname
else:
prompt = "> "
line = raw_input(prompt)
self.interactive = True
except EOFError:
print ""
break
if not line:
continue
args = shlex.split(line)
if args[0].lower() == "quit":
break
logger.info("Command: %s" % line)
try:
cmd.parse(args)
except SystemExit as e:
pass
except Exception as e:
print ("Command raised an error, see log for traceback:\n%s\n" % str(e))
logger.exception("Command raised an error")
