def initialize(self, ctx):
super(CsvFileWriter, self).initialize(ctx)
self._fileWriter = FileWriter(path=self.path,
data="${ m['_csvdata'] }",
newline=False,
overwrite=self.overwrite)
if (self.encoding):
self._fileWriter.encoding = self.encoding
ctx.comp.initialize(self._fileWriter)
def initialize(self, ctx):
super(CsvFileWriter, self).initialize(ctx)
self._fileWriter = FileWriter()
self._fileWriter.path = self.path
self._fileWriter.data = "${ m['_csvdata'] }"
if (self.encoding):
self._fileWriter.encoding = self.encoding
self._fileWriter.overwrite = self.overwrite
self._fileWriter.newline = False
# Process columns
for c in self.columns:
if not "label" in c:
c["label"] = c["name"]
if not "value" in c:
c["value"] = '${ m["' + c["name"] + '"] }'
ctx.comp.initialize(self._fileWriter)
def initialize(self, ctx):
super(CsvFileWriter, self).initialize(ctx)
self._fileWriter = FileWriter(path=self.path,
data="${ m['_csvdata'] }",
newline=False,
overwrite=self.overwrite)
if (self.encoding):
self._fileWriter.encoding = self.encoding
ctx.comp.initialize(self._fileWriter)
def initialize(self, ctx):
super(CsvFileWriter, self).initialize(ctx)
self._fileWriter = FileWriter()
self._fileWriter.path = self.path
self._fileWriter.data = "${ m['_csvdata'] }"
if (self.encoding):
self._fileWriter.encoding = self.encoding
self._fileWriter.overwrite = self.overwrite
self._fileWriter.newline = False
# Process columns
for c in self.columns:
if not "label" in c:
c["label"] = c["name"]
if not "value" in c:
c["value"] = '${ m["' + c["name"] + '"] }'
ctx.comp.initialize(self._fileWriter)
class CsvFileWriter(Node):
"""
This node writes message attributes
"""
# TODO: columns should be a CubETL type (as SQLColumns, etc). We could use
# cubetl.table TableColumns here, which in the end are basic mappings
# with name, label, value/eval (and maybe default and/or type...)
# TODO: This class should possibly compose FileWriter and CsvWriter
# (a CSVWriter should be able to write CSV rows to messages)
def __init__(self, path="-", overwrite=False):
super().__init__()
self.data = '${ m }'
self.headers = None
self.write_headers = True
self.path = path
self.delimiter = ","
self.row_delimiter = "\n"
self.overwrite = overwrite
self.encoding = None #"utf-8"
self.columns = None
self.auto_columns = True
self._row = 0
self._output = None
self._csvwriter = None
def initialize(self, ctx):
super(CsvFileWriter, self).initialize(ctx)
self._fileWriter = FileWriter(path=self.path,
data="${ m['_csvdata'] }",
newline=False,
overwrite=self.overwrite)
if (self.encoding):
self._fileWriter.encoding = self.encoding
ctx.comp.initialize(self._fileWriter)
def initialize_columns(self):
for c in self.columns:
if "label" not in c:
c["label"] = c["name"]
if "value" not in c:
c["value"] = '${ m["' + c["name"] + '"] }'
def columns_from_message(self, ctx, m):
self.columns = []
for k, v in m.items():
column = {'name': k}
self.columns.append(column)
# Sort by name for repeatable results
self.columns.sort(key=lambda c: c['name'])
def finalize(self, ctx):
ctx.comp.finalize(self._fileWriter)
super(CsvFileWriter, self).finalize(ctx)
def _csv_row(self, ctx, row):
if self.encoding:
row = [(r.encode(self.encoding) if isinstance(r, str) else r) for r in row]
self._csvwriter.writerow(row)
result = self._output.getvalue()
self._output.truncate(0)
self._output.seek(0)
return result
def process(self, ctx, m):
if not self._csvwriter:
self._output = io.StringIO()
self._csvwriter = csv.writer(self._output, delimiter=self.delimiter,
quotechar='"', quoting=csv.QUOTE_MINIMAL)
if (self._row == 0):
# Process columns
if self.columns is None and self.auto_columns and m:
self.columns_from_message(ctx, m)
self.initialize_columns()
# Write headers
if (self.write_headers):
row = [c["label"] for c in self.columns]
m['_csvdata'] = self._csv_row(ctx, row)
self._fileWriter.process(ctx, m)
self._row = self._row + 1
row = [ctx.interpolate(c["value"], m) for c in self.columns]
m['_csvdata'] = self._csv_row(ctx, row)
self._fileWriter.process(ctx, m)
del (m['_csvdata'])
yield m
class CsvFileWriter(Node):
# TODO: This class should possibly inherit from FileWriter
data = '${ m }'
headers = None
write_headers = True
path = "-"
_row = 0
delimiter = ","
row_delimiter = "\n"
overwrite = False
encoding = None #"utf-8"
_output = None
_csvwriter = None
columns = None
def initialize(self, ctx):
super(CsvFileWriter, self).initialize(ctx)
self._fileWriter = FileWriter()
self._fileWriter.path = self.path
self._fileWriter.data = "${ m['_csvdata'] }"
if (self.encoding):
self._fileWriter.encoding = self.encoding
self._fileWriter.overwrite = self.overwrite
self._fileWriter.newline = False
# Process columns
for c in self.columns:
if not "label" in c:
c["label"] = c["name"]
if not "value" in c:
c["value"] = '${ m["' + c["name"] + '"] }'
ctx.comp.initialize(self._fileWriter)
def finalize(self, ctx):
ctx.comp.finalize(self._fileWriter)
super(CsvFileWriter, self).finalize(ctx)
def _csv_row(self, ctx, row):
if self.encoding:
row = [(r.encode(self.encoding) if isinstance(r, basestring) else r) for r in row]
self._csvwriter.writerow(row)
result = self._output.getvalue()
self._output.truncate(0)
return result
def process(self, ctx, m):
if not self._csvwriter:
self._output = StringIO.StringIO()
self._csvwriter = csv.writer(self._output, delimiter=self.delimiter,
quotechar='"', quoting=csv.QUOTE_MINIMAL)
if (self._row == 0):
if (self.write_headers):
row = [c["label"] for c in self.columns]
m['_csvdata'] = self._csv_row(ctx, row)
self._fileWriter.process(ctx, m)
self._row = self._row + 1
row = [ctx.interpolate(m, c["value"]) for c in self.columns]
m['_csvdata'] = self._csv_row(ctx, row)
self._fileWriter.process(ctx, m)
del (m['_csvdata'])
yield m
class CsvFileWriter(Node):
"""
This node writes message attributes
"""
# TODO: columns should be a CubETL type (as SQLColumns, etc). We could use
# cubetl.table TableColumns here, which in the end are basic mappings
# with name, label, value/eval (and maybe default and/or type...)
# TODO: This class should possibly compose FileWriter and CsvWriter
# (a CSVWriter should be able to write CSV rows to messages)
def __init__(self, path="-", overwrite=False):
super().__init__()
self.data = '${ m }'
self.headers = None
self.write_headers = True
self.path = path
self.delimiter = ","
self.row_delimiter = "\n"
self.overwrite = overwrite
self.encoding = None #"utf-8"
self.columns = None
self.auto_columns = True
self._row = 0
self._output = None
self._csvwriter = None
def initialize(self, ctx):
super(CsvFileWriter, self).initialize(ctx)
self._fileWriter = FileWriter(path=self.path,
data="${ m['_csvdata'] }",
newline=False,
overwrite=self.overwrite)
if (self.encoding):
self._fileWriter.encoding = self.encoding
ctx.comp.initialize(self._fileWriter)
def initialize_columns(self):
for c in self.columns:
if "label" not in c:
c["label"] = c["name"]
if "value" not in c:
c["value"] = '${ m["' + c["name"] + '"] }'
def columns_from_message(self, ctx, m):
self.columns = []
for k, v in m.items():
column = {'name': k}
self.columns.append(column)
# Sort by name for repeatable results
self.columns.sort(key=lambda c: c['name'])
def finalize(self, ctx):
ctx.comp.finalize(self._fileWriter)
super(CsvFileWriter, self).finalize(ctx)
def _csv_row(self, ctx, row):
if self.encoding:
row = [(r.encode(self.encoding) if isinstance(r, str) else r)
for r in row]
self._csvwriter.writerow(row)
result = self._output.getvalue()
self._output.truncate(0)
self._output.seek(0)
return result
def process(self, ctx, m):
if not self._csvwriter:
self._output = io.StringIO()
self._csvwriter = csv.writer(self._output,
delimiter=self.delimiter,
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
if (self._row == 0):
# Process columns
if self.columns is None and self.auto_columns and m:
self.columns_from_message(ctx, m)
self.initialize_columns()
# Write headers
if (self.write_headers):
row = [c["label"] for c in self.columns]
m['_csvdata'] = self._csv_row(ctx, row)
self._fileWriter.process(ctx, m)
self._row = self._row + 1
row = [ctx.interpolate(c["value"], m) for c in self.columns]
m['_csvdata'] = self._csv_row(ctx, row)
self._fileWriter.process(ctx, m)
del (m['_csvdata'])
yield m
class CsvFileWriter(Node):
# TODO: This class should possibly inherit from FileWriter
data = '${ m }'
headers = None
write_headers = True
path = "-"
_row = 0
delimiter = ","
row_delimiter = "\n"
overwrite = False
encoding = None #"utf-8"
_output = None
_csvwriter = None
columns = None
def initialize(self, ctx):
super(CsvFileWriter, self).initialize(ctx)
self._fileWriter = FileWriter()
self._fileWriter.path = self.path
self._fileWriter.data = "${ m['_csvdata'] }"
if (self.encoding):
self._fileWriter.encoding = self.encoding
self._fileWriter.overwrite = self.overwrite
self._fileWriter.newline = False
# Process columns
for c in self.columns:
if not "label" in c:
c["label"] = c["name"]
if not "value" in c:
c["value"] = '${ m["' + c["name"] + '"] }'
ctx.comp.initialize(self._fileWriter)
def finalize(self, ctx):
ctx.comp.finalize(self._fileWriter)
super(CsvFileWriter, self).finalize(ctx)
def _csv_row(self, ctx, row):
if self.encoding:
row = [(r.encode(self.encoding) if isinstance(r, basestring) else r) for r in row]
self._csvwriter.writerow(row)
result = self._output.getvalue()
self._output.truncate(0)
return result
def process(self, ctx, m):
if not self._csvwriter:
self._output = StringIO.StringIO()
self._csvwriter = csv.writer(self._output, delimiter=self.delimiter,
quotechar='"', quoting=csv.QUOTE_MINIMAL)
if (self._row == 0):
if (self.write_headers):
row = [c["label"] for c in self.columns]
m['_csvdata'] = self._csv_row(ctx, row)
self._fileWriter.process(ctx, m)
self._row = self._row + 1
row = [ctx.interpolate(m, c["value"]) for c in self.columns]
m['_csvdata'] = self._csv_row(ctx, row)
self._fileWriter.process(ctx, m)
del (m['_csvdata'])
yield m
