def read_all_boards(self):
"""Read all data from every board for every ticker"""
markets_n_engines, boards = self._get_metadata()
try:
self.__markets_n_engines = markets_n_engines
urls = self.url # generate urls per symbols
dfs = [] # an array of pandas dataframes per symbol to concatenate
for i in range(len(urls)):
out_list = []
date_column = None
while True: # read in a loop with small date intervals
if len(out_list) > 0:
if date_column is None:
date_column = out_list[0].split(";").index("TRADEDATE")
# get the last downloaded date
start_str = out_list[-1].split(";", 4)[date_column]
start = dt.datetime.strptime(start_str, "%Y-%m-%d").date()
else:
start_str = self.start.strftime("%Y-%m-%d")
start = self.start
if start > self.end or start > dt.date.today():
break
params = self._get_params(start_str)
strings_out = self._read_url_as_String(
urls[i], params
).splitlines()[2:]
strings_out = list(filter(lambda x: x.strip(), strings_out))
if len(out_list) == 0:
out_list = strings_out
if len(strings_out) < 101: # all data received - break
break
else:
out_list += strings_out[1:] # remove a CSV head line
if len(strings_out) < 100: # all data recevied - break
break
if len(out_list) > 0:
str_io = StringIO("\r\n".join(out_list))
dfs.append(self._read_lines(str_io)) # add a new DataFrame
finally:
self.close()
if len(dfs) == 0:
raise IOError(
"{} returned no data; "
"check URL or correct a date".format(self.__class__.__name__)
)
elif len(dfs) > 1:
b = concat(dfs, axis=0, join="outer", sort=True)
else:
b = dfs[0]
return b
def _get_one_action(data):
actions = DataFrame(columns=["action", "value"])
if "Dividends" in data.columns:
# Add a label column so we can combine our two DFs
dividends = DataFrame(data["Dividends"]).dropna()
dividends["action"] = "DIVIDEND"
dividends = dividends.rename(columns={"Dividends": "value"})
actions = concat([actions, dividends], sort=True)
actions = actions.sort_index(ascending=False)
if "Splits" in data.columns:
# Add a label column so we can combine our two DFs
splits = DataFrame(data["Splits"]).dropna()
splits["action"] = "SPLIT"
splits = splits.rename(columns={"Splits": "value"})
actions = concat([actions, splits], sort=True)
actions = actions.sort_index(ascending=False)
return actions
def read(self):
"""Read data"""
try:
self.__markets, self.__engines = self._get_metadata()
urls = self.url # generate urls per symbols
dfs = [] # an array of pandas dataframes per symbol to concatenate
for i in range(len(self.symbols)):
out_list = []
date_column = None
while True: # read in a loop with small date intervals
if len(out_list) > 0:
if date_column is None:
date_column = out_list[0].split(";").index(
"TRADEDATE")
# get the last downloaded date
start_str = out_list[-1].split(";", 4)[date_column]
start = dt.datetime.strptime(start_str,
"%Y-%m-%d").date()
else:
start_str = self.start.strftime("%Y-%m-%d")
start = self.start
if start >= self.end or start >= dt.date.today():
break
params = self._get_params(start_str)
strings_out = self._read_url_as_String(
urls[i], params).splitlines()[2:]
strings_out = list(filter(lambda x: x.strip(),
strings_out))
if len(out_list) == 0:
out_list = strings_out
if len(strings_out) < 101: # all data received - break
break
else:
out_list += strings_out[1:] # remove a CSV head line
if len(strings_out) < 100: # all data recevied - break
break
str_io = StringIO("\r\n".join(out_list))
dfs.append(self._read_lines(str_io)) # add a new DataFrame
finally:
self.close()
if len(dfs) > 1:
return concat(dfs, axis=0, join="outer", sort=True)
else:
return dfs[0]