def connect(self, host, port, auth=None): self.conn = Connection(host, port=port, verify=False) if auth is not None: # Use HTTP Basic authentication s = base64.b64encode("%s:%s" % (auth.username, auth.password)) self.conn.add_headers({'Authorization': 'Basic %s' % s}) logger.info("Authenticated using BASIC")
def get_historical_prices(begin, end, symbol, measures, resolution='day', date_obj=False): """Get historical prices for the given ticker symbol. Returns a list of dicts keyed by 'date' and measures :param string begin: begin date of the inquire interval in the format of YYYY-MM-DD :param string end: end date of the inquire interval in the format of YYYY-MM-DD :param string symbol: symbol of one stock to query :param list measures: a list of prices that needs to be queried, should be a subset of ["open", "high", "low", "close", "volume"] :param string resolution: '1 day' or '5 days' :param boolean date_obj: dates are converted to datetime objects from date strings if True. Otherwise, dates are stored as strings """ try: conn = Connection('http://ichart.finance.yahoo.com') start_month = parse_date(begin).month - 1 start_day = parse_date(begin).day start_year = parse_date(begin).year end_month = parse_date(end).month - 1 end_day = parse_date(end).day end_year = parse_date(end).year ret = [] params = {'s': symbol, 'a': start_month, 'b': start_day, 'c': start_year, 'd': end_month, 'e': end_day, 'f': end_year, 'g': resolution[0], 'ignore': '.csv'} resp = conn.request(method='POST', path='/table.csv', params=params) # extract data and skip first row with column titles data = list(resp.iter_lines())[1:] # iterate over the data backwards as the daily prices are sorted # backwards by the dates for day in reversed(data): # day is a string with date, prices, volume separated by commas, # '<date>,<open>,<high>,<low>,<close>,<volume>,<adj_close>' # as '2014-02-19,20.22,20.55,20.11,20.50,1599600,20.50' day = day.split(',') date = parse_date(day[0]) if date_obj else day[0] daily_prices = {'date': date} for m in measures: if m in mapping: daily_prices[m] = float(day[mapping[m]]) ret.append(daily_prices) except RvbdHTTPException: raise StockApiException("Symbol '%s' is invalid or Stock '%s' was" " not on market on %s" % (symbol, symbol, end)) return ret
def run_report_via_rest(host, slug, namespace, title, authfile, verify, **kwargs): """Mimic the front-end javascript to run a whole report. This also has the added benefit of adding an entry to Report History, and adding the resulting data to the Widget Cache if those functions are enabled. """ report_url = '/report/%s/%s/' % (namespace, slug) post_header = {'Content-Type': 'application/x-www-form-urlencoded'} criteria, options = process_criteria(kwargs) # since we are posting in place of a criteria form, we need to split time endtime = criteria.pop('endtime') endtime_date, endtime_time = endtime.split('T') criteria['endtime_0'] = endtime_date criteria['endtime_1'] = endtime_time conn = Connection(host, auth=get_auth(authfile), verify=verify) logger.debug('Posting report criteria for report %s: %s' % (title, criteria)) r = conn.request('POST', report_url, extra_headers=post_header, body=criteria) if r.ok and 'widgets' in r.json(): logger.debug('Got widgets for Report url %s' % report_url) else: logger.error('Error getting Widgets for Report url %s. Aborting.' % report_url) return jobs = [] # create the widget jobs for each widget found for w in r.json()['widgets']: data = {'criteria': json.dumps(w['criteria'])} w_response = conn.request('POST', w['posturl'], extra_headers=post_header, body=data) jobs.append(w_response.json()['joburl']) # check until all jobs are done timeout = options['delta'] interval = 1 wait_for_complete(conn, interval, timeout, jobs)
class ProgressDaemon(object): def __init__(self): self.conn = Connection(settings.PROGRESSD_HOST, port=settings.PROGRESSD_PORT) def _request(self, method, url, **kwargs): try: return self.conn.json_request(method, url, **kwargs) except ConnectionError: raise RvbdException( "Error connecting to appfwk service 'progressd': " "try restarting the service (sudo service progressd restart) " "or contact your administrator for assistance." ) def get(self, id_, attr=None): r = self._request('GET', '/jobs/items/%d/' % id_) if attr: return r[attr] return r def post(self, **kwargs): return self._request('POST', '/jobs/', body=kwargs) def put(self, id_, **kwargs): self._request('PUT', '/jobs/items/%d/' % id_, body=kwargs) def delete(self, id_): self._request('DELETE', '/jobs/items/%d/' % id_) def reset(self): self._request('POST', '/jobs/reset/')
class ProgressDaemon(object): def __init__(self): self.conn = Connection(settings.PROGRESSD_HOST, port=settings.PROGRESSD_PORT) def _request(self, method, url, **kwargs): try: return self.conn.json_request(method, url, **kwargs) except ConnectionError: raise RvbdException( "Error connecting to appfwk service 'progressd': " "try restarting the service (sudo service progressd restart) " "or contact your administrator for assistance.") def get(self, id_, attr=None): r = self._request('GET', '/jobs/items/%d/' % id_) if attr: return r[attr] return r def post(self, **kwargs): return self._request('POST', '/jobs/', body=kwargs) def put(self, id_, **kwargs): self._request('PUT', '/jobs/items/%d/' % id_, body=kwargs) def delete(self, id_): self._request('DELETE', '/jobs/items/%d/' % id_) def reset(self): self._request('POST', '/jobs/reset/')
def connect(self, host, port, auth=None): self.conn = Connection(host, port=port, verify=False) if auth is not None: # Use HTTP Basic authentication s = base64.b64encode("%s:%s" % (auth.username, auth.password)) self.conn.add_headers({"Authorization": "Basic %s" % s}) logger.info("Authenticated using BASIC")
class ProgressDaemon(object): def __init__(self): self.conn = Connection(settings.PROGRESSD_HOST, port=settings.PROGRESSD_PORT) def _request(self, method, url, **kwargs): try: return self.conn.json_request(method, url, **kwargs) except ConnectionError: # In case progressd is restarting due to daily logrotate # resending the request after 10 seconds logger.warning("Waiting %s seconds before reconnecting progressd" % settings.PROGRESSD_CONN_TIMEOUT) time.sleep(settings.PROGRESSD_CONN_TIMEOUT) try: return self.conn.json_request(method, url, **kwargs) except ConnectionError: raise RvbdException( "Error connecting to appfwk service 'progressd': " "try restarting the service " "(sudo service progressd restart) " "or contact your administrator for assistance." ) def get(self, id_, attr=None): r = self._request('GET', '/jobs/items/%d/' % id_) if attr: return r[attr] return r def post(self, **kwargs): return self._request('POST', '/jobs/', body=kwargs) def put(self, id_, **kwargs): self._request('PUT', '/jobs/items/%d/' % id_, body=kwargs) def delete(self, id_): self._request('DELETE', '/jobs/items/%d/' % id_) def reset(self): self._request('POST', '/jobs/reset/')
class ProgressDaemon(object): def __init__(self): self.conn = Connection(settings.PROGRESSD_HOST, port=settings.PROGRESSD_PORT) def _request(self, method, url, **kwargs): try: return self.conn.json_request(method, url, **kwargs) except ConnectionError: # In case progressd is restarting due to daily logrotate # resending the request after 10 seconds logger.warning("Waiting %s seconds before reconnecting progressd" % settings.PROGRESSD_CONN_TIMEOUT) time.sleep(settings.PROGRESSD_CONN_TIMEOUT) try: return self.conn.json_request(method, url, **kwargs) except ConnectionError: raise RvbdException( "Error connecting to appfwk service 'progressd': " "try restarting the service " "(sudo service progressd restart) " "or contact your administrator for assistance.") def get(self, id_, attr=None): r = self._request('GET', '/jobs/items/%d/' % id_) if attr: return r[attr] return r def post(self, **kwargs): return self._request('POST', '/jobs/', body=kwargs) def put(self, id_, **kwargs): self._request('PUT', '/jobs/items/%d/' % id_, body=kwargs) def delete(self, id_): self._request('DELETE', '/jobs/items/%d/' % id_) def reset(self): self._request('POST', '/jobs/reset/')
def run_table_via_rest(host, url, authfile, verify, **kwargs): criteria, options = process_criteria(kwargs) conn = Connection(host, auth=get_auth(authfile), verify=verify) logger.debug('POSTing new job with criteria: %s' % criteria) r = conn.request('POST', url, body=criteria) if r.ok: logger.debug('Job creation successful.') else: logger.error('Error creating Job: %s' % r.content) if options.get('output-file', None): # check periodically until data is ready and write to file url = r.headers['Location'] timeout = options['delta'] interval = 1 wait_for_complete(conn, interval, timeout, [url], [options['output-file']]) else: # we aren't interested in results pass
def __init__(self): self.conn = Connection(settings.PROGRESSD_HOST, port=settings.PROGRESSD_PORT)
class Command(Application): help = "Interactive shell for issuing REST commands" def __init__(self, *args, **kwargs): super(Command, self).__init__(*args, **kwargs) self.conn = None self.filelines = [] self.interactive = True self.jsonmode = True def parse_args(self): super(Command, self).parse_args() def add_positional_args(self): super(Command, self).add_positional_args() # self.add_positional_arg('host', 'Rest hostname or IP address') def add_options(self, parser): parser.add_option('-f', '--file', help='Read commands from a file') super(Command, self).add_options(parser) self.add_standard_options(conn=False, log=True) def connect(self, host, port, auth=None): self.conn = Connection(host, port=port, verify=False) if auth is not None: # Use HTTP Basic authentication s = base64.b64encode("%s:%s" % (auth.username, auth.password)) self.conn.add_headers({'Authorization': 'Basic %s' % s}) logger.info("Authenticated using BASIC") def raw_input_no_history(self, prompt): if self.filelines: return self.filelines.pop(0) else: input = raw_input(prompt) readline.remove_history_item( readline.get_current_history_length() - 1 ) return input def main(self): self.history = HistoryManager() history = self.history history.add_context('cmd', os.path.join(os.path.expanduser('~'), '.steelscript/rest-cmd-history')) history.add_context('body', os.path.join(os.path.expanduser('~'), '.steelscript/rest-obj-history'), obj=True) history.set_context('cmd') atexit.register(HistoryManager.save, history) cmd = RootCommand() self.cmd = cmd GET(self, cmd) POST(self, cmd) PUT(self, cmd) DELETE(self, cmd) Connect(self, cmd) Help(self, cmd) Mode(self, cmd) if self.options.file: with open(self.options.file, 'r') as f: self.filelines = [line.rstrip() for line in f] print "REST Shell ('help' or 'quit' when done)" print "Current mode is 'json', use 'mode text' to switch to raw text" while True: if self.filelines: line = self.filelines.pop(0) self.interactive = False print "Command: %s" % line else: try: if self.conn is not None: prompt = "%s> " % self.conn.hostname else: prompt = "> " line = raw_input(prompt) self.interactive = True except EOFError: print "" break if not line: continue args = shlex.split(line) if args[0].lower() == 'quit': break logger.info('Command: %s' % line) try: cmd.parse(args) except SystemExit as e: pass except Exception as e: print ('Command raised an error, see log for traceback:\n%s\n' % str(e)) logger.exception("Command raised an error")
def get_job_function(self, kwargs): # allow for alternate methods to call table commands in future # possibly direct API calls, or REST calls against running server if self.options.rest_server: if 'table-name' in kwargs: # find id of table-name name = kwargs['table-name'] baseurl = '/data/tables/' verify = not self.options.insecure conn = Connection(self.options.rest_server, auth=get_auth(self.options.authfile), verify=verify) tableurl = None url = baseurl logger.debug('Getting available tables from server ...') while tableurl is None: r = conn.json_request('GET', url) results = r['results'] urls = [t['url'] for t in results if t['name'] == name] if len(urls) > 1: msg = ('ERROR: Multiple tables found for name %s: %s' % (name, urls)) logger.debug(msg) raise ValueError(msg) elif len(urls) == 1: # parse out just the path component of the url logger.debug('Found table url: %s' % urls) fullurl = urljoin(urls[0], 'jobs/') tableurl = urlsplit(fullurl).path else: url = r['next'] if url is None: msg = ('ERROR: No table found for "%s"' % name) logger.debug(msg) raise ValueError(msg) job_params = { 'func': run_table_via_rest, 'args': [self.options.rest_server, tableurl, self.options.authfile, verify] } elif 'report-slug' in kwargs: # find id of report slug = kwargs['report-slug'] namespace = kwargs['report-namespace'] baseurl = urljoin(self.options.rest_server, '/report/') verify = not self.options.insecure conn = Connection(self.options.rest_server, auth=get_auth(self.options.authfile), verify=verify) title = None logger.debug('Getting available reports from server ...') r = conn.json_request('GET', baseurl) for report in r: if slug in report.values(): title = report['title'] break if title is None: msg = ('No report found for slug %s namespace %s' % (slug, namespace)) logger.error(msg) raise ValueError(msg) job_params = { 'func': run_report_via_rest, 'args': [self.options.rest_server, slug, namespace, title, self.options.authfile, verify] } else: raise ValueError('Invalid config, no "table-name" or ' '"report-slug"/"report-namespace" specified') else: job_params = {'func': run_table, 'args': ['table']} logger.debug('Calculated job params: %s' % job_params) return job_params
class Command(Application): help = "Interactive shell for issuing REST commands" def __init__(self, *args, **kwargs): super(Command, self).__init__(*args, **kwargs) self.conn = None self.filelines = [] self.interactive = True self.jsonmode = True def parse_args(self): super(Command, self).parse_args() def add_positional_args(self): super(Command, self).add_positional_args() # self.add_positional_arg('host', 'Rest hostname or IP address') def add_options(self, parser): parser.add_option("-f", "--file", help="Read commands from a file") super(Command, self).add_options(parser) self.add_standard_options(conn=False, log=True) def connect(self, host, port, auth=None): self.conn = Connection(host, port=port, verify=False) if auth is not None: # Use HTTP Basic authentication s = base64.b64encode("%s:%s" % (auth.username, auth.password)) self.conn.add_headers({"Authorization": "Basic %s" % s}) logger.info("Authenticated using BASIC") def raw_input_no_history(self, prompt): if self.filelines: return self.filelines.pop(0) else: input = raw_input(prompt) readline.remove_history_item(readline.get_current_history_length() - 1) return input def main(self): self.history = HistoryManager() history = self.history history.add_context("cmd", os.path.join(os.path.expanduser("~"), ".steelscript/rest-cmd-history")) history.add_context("body", os.path.join(os.path.expanduser("~"), ".steelscript/rest-obj-history"), obj=True) history.set_context("cmd") atexit.register(HistoryManager.save, history) cmd = RootCommand() self.cmd = cmd GET(self, cmd) POST(self, cmd) PUT(self, cmd) DELETE(self, cmd) Connect(self, cmd) Help(self, cmd) Mode(self, cmd) if self.options.file: with open(self.options.file, "r") as f: self.filelines = [line.rstrip() for line in f] print "REST Shell ('help' or 'quit' when done)" print "Current mode is 'json', use 'mode text' to switch to raw text" while True: if self.filelines: line = self.filelines.pop(0) self.interactive = False print "Command: %s" % line else: try: if self.conn is not None: prompt = "%s> " % self.conn.hostname else: prompt = "> " line = raw_input(prompt) self.interactive = True except EOFError: print "" break if not line: continue args = shlex.split(line) if args[0].lower() == "quit": break logger.info("Command: %s" % line) try: cmd.parse(args) except SystemExit as e: pass except Exception as e: print ("Command raised an error, see log for traceback:\n%s\n" % str(e)) logger.exception("Command raised an error")