def _connect(self, db_file_path): self._db_file_path = db_file_path initialize = False if not os.path.exists(db_file_path): utils.mkdir_p(settings.DATA_DIR) initialize = True self._connection = sqlite3.connect(db_file_path) self._connection.text_factory = str # FIXME: use unicode return initialize
def __yield_open_files(self, data_contexts, mode): ''' :param tag_file_paths: tuple(anything, file_path_to_open) :param mode: any mode supported by the selected compression backend ''' for data, context in data_contexts: file_path = context['file_path'] if mode == 'w': utils.mkdir_p(os.path.dirname(file_path)) compression = settings.DATA_COMPRESSION if compression == 'gz': f = gzip.open(file_path, mode) elif not compression or compression == 'lz4': f = open(file_path, mode) context['_open_file'] = f yield data, context
def save_to_csv(self, file_name=None): ordered_columns = self.csv_column_order() column_header_dict = dict((x, x) for x in ordered_columns) instruments = self.get_instruments() instrumentCmp = lambda x, y: cmp( x[self.csv_sort_column()], y[self.csv_sort_column()]) instruments.sort(instrumentCmp) if self.csv_sort_descending(): instruments.reverse() file_name = file_name or '%s.csv' % self.slug() file_path = os.path.join(self.data_folder(), file_name) utils.mkdir_p(os.path.dirname(file_path)) with open(file_path, 'w') as f: csvwriter = csv.DictWriter(f, fieldnames=ordered_columns, extrasaction='ignore') csvwriter.writerow(column_header_dict) for instrument in instruments: if self.csv_filter(instrument): #csvwriter.writerow(instrument.dict()) csvwriter.writerow(instrument.get_values(ordered_columns))
def save_to_csv(self, file_name=None): ordered_columns = self.csv_column_order() column_header_dict = dict((x, x) for x in ordered_columns) instruments = self.get_instruments() instrumentCmp = lambda x, y: cmp(x[self.csv_sort_column()], y[ self.csv_sort_column()]) instruments.sort(instrumentCmp) if self.csv_sort_descending(): instruments.reverse() file_name = file_name or '%s.csv' % self.slug() file_path = os.path.join(self.data_folder(), file_name) utils.mkdir_p(os.path.dirname(file_path)) with open(file_path, 'w') as f: csvwriter = csv.DictWriter(f, fieldnames=ordered_columns, extrasaction='ignore') csvwriter.writerow(column_header_dict) for instrument in instruments: if self.csv_filter(instrument): #csvwriter.writerow(instrument.dict()) csvwriter.writerow(instrument.get_values(ordered_columns))