def UnzipFile(self, table_name):
fname = table_name + self.ZipTail
full_name = os.path.join(self.DataDirectory, fname)
log('Unzipping %s', full_name)
with zipfile.ZipFile(full_name, 'r') as myzip:
info = myzip.infolist()
expected_names = [table_name + '.csv', table_name + '_MetaData.csv']
for i in info:
if i.filename in expected_names:
log('Extracting file %s', i.filename)
else:
log_warning('Unexpected file name: %s', i.filename)
myzip.extract(i, self.DataDirectory)
def fetch(self, series_meta):
"""
Do the fetch.
Can only support single series queries...
:param series_meta: econ_platform_core.SeriesMetaData
:return: list
"""
query_ticker = str(series_meta.ticker_query)
try:
table_name, vector = query_ticker.split('|')
except:
raise econ_platform_core.TickerError(
'CANSIM_CSV ticker format: <table>|<vector>; invalid ticker = {0}'
.format(query_ticker))
parsed_name = self.GetTimeSeriesFile(table_name)
if not os.path.exists(parsed_name):
econ_platform_core.log(
'Table file does not exist, attempting to unzip')
try:
self.UnzipFile(table_name)
except:
raise econ_platform_core.TickerNotFoundError(
'Table {0} needs to be downloaded as a zip file'.format(
table_name))
self.ParseUnzipped(table_name)
items = []
with open(parsed_name, 'r') as f:
for row_raw in f:
row = row_raw.split('\t')
if row[0] == vector:
ddate = econ_platform_core.utils.iso_string_to_date(row[1])
items.append((ddate, float(row[2])))
if len(items) == 0:
raise econ_platform_core.TickerNotFoundError(
'Vector {0} not found in CANSIM table {1}'.format(
vector, table_name))
items.sort()
values = [x[1] for x in items]
dates = [x[0] for x in items]
data = pandas.Series(values)
data.index = dates
data.name = '{0}@{1}'.format(self.ProviderCode, query_ticker)
return [
data,
]
def fetch(self, series_meta):
"""
Do the fetch from CSV.
Increments may be via JSON.
:param series_meta: econ_platform_core.SeriesMetadata
:return: pandas.Series
"""
query_ticker = str(series_meta.ticker_query)
try:
table_name, vector = query_ticker.split('|')
except:
raise econ_platform_core.TickerError(
'CANSIM_CSV ticker format: <table>|<vector>; invalid ticker = {0}'
.format(query_ticker))
parsed_name = self.GetTimeSeriesFile(table_name)
if not os.path.exists(parsed_name):
econ_platform_core.log(
'Table file does not exist, attempting to unzip')
try:
self.UnzipFile(table_name)
except:
raise econ_platform_core.PlatformError(
'Table {0} needs to be downloaded as a zip file'.format(
table_name))
# Do the whole table
self.TableWasFetched = True
self.TableMeta = {}
self.TableSeries = {}
self.MetaMapper = {}
self.ParseUnzipped(table_name)
self.BuildSeries()
self.ArchiveFiles(table_name)
try:
ser = self.TableSeries[str(series_meta.ticker_full)]
meta = self.TableMeta[str(series_meta.ticker_full)]
except KeyError:
raise econ_platform_core.TickerNotFoundError(
'{0} was not found'.format(str(series_meta.ticker_full)))
return ser, meta
def Execute(self, cmd, *args, commit_after=False, is_many=False):
"""
Convenience function that logs commands if self.LogSQL is True.
:param cmd: str
:param args:
:param commit_after: bool
:return: sqlite3.Cursor
"""
if self.Connection is None:
self.GetConnection(test_tables=False)
if self.Cursor is None:
self.Cursor = self.Connection.cursor()
if self.LogSQL:
if len(args) > 0:
if is_many:
econ_platform_core.log('{0} : executemany()'.format(cmd))
else:
econ_platform_core.log(cmd + " : " + str(args))
else:
econ_platform_core.log(cmd)
if len(args) > 0:
if is_many:
self.Cursor.executemany(cmd, args[0])
else:
self.Cursor.execute(cmd, args)
else:
self.Cursor.execute(cmd)
if commit_after:
self.Connection.commit()
return self.Cursor
def Execute(self, cmd, *args, commit_after=False, is_many=False):
"""
Convenience function that logs commands if self.LogSQL is True.
:param cmd: str
:param args:
:param commit_after: bool
:return: sqlite3.Cursor
"""
if self.Connection is None:
self.Connect()
if self.Cursor is None:
self.Cursor = self.Connection.cursor()
if self.LogSQL:
if len(args) > 0:
if is_many:
econ_platform_core.log('{0} : executemany()'.format(cmd))
else:
econ_platform_core.log(cmd + " : " + str(args))
else:
econ_platform_core.log(cmd)
if len(args) > 0:
if is_many:
# No ticker conversion here, but unlikely to appear in an executemany()
self.Cursor.executemany(cmd, args[0])
else:
# Quietly convert tickers to str
clean_args = []
for x in args:
if issubclass(x.__class__,
econ_platform_core.tickers._TickerAbstract):
clean_args.append(str(x))
else:
clean_args.append(x)
self.Cursor.execute(cmd, clean_args)
else:
self.Cursor.execute(cmd)
if commit_after:
self.Connection.commit()
return self.Cursor