def initialize(self, ctx): super(CsvFileWriter, self).initialize(ctx) self._fileWriter = FileWriter(path=self.path, data="${ m['_csvdata'] }", newline=False, overwrite=self.overwrite) if (self.encoding): self._fileWriter.encoding = self.encoding ctx.comp.initialize(self._fileWriter)
def initialize(self, ctx): super(CsvFileWriter, self).initialize(ctx) self._fileWriter = FileWriter() self._fileWriter.path = self.path self._fileWriter.data = "${ m['_csvdata'] }" if (self.encoding): self._fileWriter.encoding = self.encoding self._fileWriter.overwrite = self.overwrite self._fileWriter.newline = False # Process columns for c in self.columns: if not "label" in c: c["label"] = c["name"] if not "value" in c: c["value"] = '${ m["' + c["name"] + '"] }' ctx.comp.initialize(self._fileWriter)
class CsvFileWriter(Node): """ This node writes message attributes """ # TODO: columns should be a CubETL type (as SQLColumns, etc). We could use # cubetl.table TableColumns here, which in the end are basic mappings # with name, label, value/eval (and maybe default and/or type...) # TODO: This class should possibly compose FileWriter and CsvWriter # (a CSVWriter should be able to write CSV rows to messages) def __init__(self, path="-", overwrite=False): super().__init__() self.data = '${ m }' self.headers = None self.write_headers = True self.path = path self.delimiter = "," self.row_delimiter = "\n" self.overwrite = overwrite self.encoding = None #"utf-8" self.columns = None self.auto_columns = True self._row = 0 self._output = None self._csvwriter = None def initialize(self, ctx): super(CsvFileWriter, self).initialize(ctx) self._fileWriter = FileWriter(path=self.path, data="${ m['_csvdata'] }", newline=False, overwrite=self.overwrite) if (self.encoding): self._fileWriter.encoding = self.encoding ctx.comp.initialize(self._fileWriter) def initialize_columns(self): for c in self.columns: if "label" not in c: c["label"] = c["name"] if "value" not in c: c["value"] = '${ m["' + c["name"] + '"] }' def columns_from_message(self, ctx, m): self.columns = [] for k, v in m.items(): column = {'name': k} self.columns.append(column) # Sort by name for repeatable results self.columns.sort(key=lambda c: c['name']) def finalize(self, ctx): ctx.comp.finalize(self._fileWriter) super(CsvFileWriter, self).finalize(ctx) def _csv_row(self, ctx, row): if self.encoding: row = [(r.encode(self.encoding) if isinstance(r, str) else r) for r in row] self._csvwriter.writerow(row) result = self._output.getvalue() self._output.truncate(0) self._output.seek(0) return result def process(self, ctx, m): if not self._csvwriter: self._output = io.StringIO() self._csvwriter = csv.writer(self._output, delimiter=self.delimiter, quotechar='"', quoting=csv.QUOTE_MINIMAL) if (self._row == 0): # Process columns if self.columns is None and self.auto_columns and m: self.columns_from_message(ctx, m) self.initialize_columns() # Write headers if (self.write_headers): row = [c["label"] for c in self.columns] m['_csvdata'] = self._csv_row(ctx, row) self._fileWriter.process(ctx, m) self._row = self._row + 1 row = [ctx.interpolate(c["value"], m) for c in self.columns] m['_csvdata'] = self._csv_row(ctx, row) self._fileWriter.process(ctx, m) del (m['_csvdata']) yield m
class CsvFileWriter(Node): # TODO: This class should possibly inherit from FileWriter data = '${ m }' headers = None write_headers = True path = "-" _row = 0 delimiter = "," row_delimiter = "\n" overwrite = False encoding = None #"utf-8" _output = None _csvwriter = None columns = None def initialize(self, ctx): super(CsvFileWriter, self).initialize(ctx) self._fileWriter = FileWriter() self._fileWriter.path = self.path self._fileWriter.data = "${ m['_csvdata'] }" if (self.encoding): self._fileWriter.encoding = self.encoding self._fileWriter.overwrite = self.overwrite self._fileWriter.newline = False # Process columns for c in self.columns: if not "label" in c: c["label"] = c["name"] if not "value" in c: c["value"] = '${ m["' + c["name"] + '"] }' ctx.comp.initialize(self._fileWriter) def finalize(self, ctx): ctx.comp.finalize(self._fileWriter) super(CsvFileWriter, self).finalize(ctx) def _csv_row(self, ctx, row): if self.encoding: row = [(r.encode(self.encoding) if isinstance(r, basestring) else r) for r in row] self._csvwriter.writerow(row) result = self._output.getvalue() self._output.truncate(0) return result def process(self, ctx, m): if not self._csvwriter: self._output = StringIO.StringIO() self._csvwriter = csv.writer(self._output, delimiter=self.delimiter, quotechar='"', quoting=csv.QUOTE_MINIMAL) if (self._row == 0): if (self.write_headers): row = [c["label"] for c in self.columns] m['_csvdata'] = self._csv_row(ctx, row) self._fileWriter.process(ctx, m) self._row = self._row + 1 row = [ctx.interpolate(m, c["value"]) for c in self.columns] m['_csvdata'] = self._csv_row(ctx, row) self._fileWriter.process(ctx, m) del (m['_csvdata']) yield m