def _connect(self, db_file_path):
self._db_file_path = db_file_path
initialize = False
if not os.path.exists(db_file_path):
utils.mkdir_p(settings.DATA_DIR)
initialize = True
self._connection = sqlite3.connect(db_file_path)
self._connection.text_factory = str # FIXME: use unicode
return initialize
def __yield_open_files(self, data_contexts, mode):
'''
:param tag_file_paths: tuple(anything, file_path_to_open)
:param mode: any mode supported by the selected compression backend
'''
for data, context in data_contexts:
file_path = context['file_path']
if mode == 'w':
utils.mkdir_p(os.path.dirname(file_path))
compression = settings.DATA_COMPRESSION
if compression == 'gz':
f = gzip.open(file_path, mode)
elif not compression or compression == 'lz4':
f = open(file_path, mode)
context['_open_file'] = f
yield data, context
def __yield_open_files(self, data_contexts, mode):
'''
:param tag_file_paths: tuple(anything, file_path_to_open)
:param mode: any mode supported by the selected compression backend
'''
for data, context in data_contexts:
file_path = context['file_path']
if mode == 'w':
utils.mkdir_p(os.path.dirname(file_path))
compression = settings.DATA_COMPRESSION
if compression == 'gz':
f = gzip.open(file_path, mode)
elif not compression or compression == 'lz4':
f = open(file_path, mode)
context['_open_file'] = f
yield data, context
def save_to_csv(self, file_name=None):
ordered_columns = self.csv_column_order()
column_header_dict = dict((x, x) for x in ordered_columns)
instruments = self.get_instruments()
instrumentCmp = lambda x, y: cmp(
x[self.csv_sort_column()],
y[self.csv_sort_column()])
instruments.sort(instrumentCmp)
if self.csv_sort_descending():
instruments.reverse()
file_name = file_name or '%s.csv' % self.slug()
file_path = os.path.join(self.data_folder(), file_name)
utils.mkdir_p(os.path.dirname(file_path))
with open(file_path, 'w') as f:
csvwriter = csv.DictWriter(f, fieldnames=ordered_columns, extrasaction='ignore')
csvwriter.writerow(column_header_dict)
for instrument in instruments:
if self.csv_filter(instrument):
#csvwriter.writerow(instrument.dict())
csvwriter.writerow(instrument.get_values(ordered_columns))
def save_to_csv(self, file_name=None):
ordered_columns = self.csv_column_order()
column_header_dict = dict((x, x) for x in ordered_columns)
instruments = self.get_instruments()
instrumentCmp = lambda x, y: cmp(x[self.csv_sort_column()], y[
self.csv_sort_column()])
instruments.sort(instrumentCmp)
if self.csv_sort_descending():
instruments.reverse()
file_name = file_name or '%s.csv' % self.slug()
file_path = os.path.join(self.data_folder(), file_name)
utils.mkdir_p(os.path.dirname(file_path))
with open(file_path, 'w') as f:
csvwriter = csv.DictWriter(f,
fieldnames=ordered_columns,
extrasaction='ignore')
csvwriter.writerow(column_header_dict)
for instrument in instruments:
if self.csv_filter(instrument):
#csvwriter.writerow(instrument.dict())
csvwriter.writerow(instrument.get_values(ordered_columns))
