def load_table_from_csv(self, table_name, raw_schema, filename):
result_table = self.make_empty_table(table_name, raw_schema)
with open(filename, 'r') as f:
for line in f:
if line[-1] == '\n':
line = line[:-1]
tokens = line.split(',')
assert len(tokens) == len(result_table.columns), (
'Expected {} tokens on line {}, but got {}'.format(
len(result_table.columns), line, len(tokens)))
for token, column in zip(tokens,
result_table.columns.itervalues()):
# Run a casting function over the value we are given.
# CSV doesn't have a null value, so the string 'null' is
# used as the null value.
if token != 'null' or column.mode != tq_modes.NULLABLE:
token = tq_types.CAST_FUNCTION_MAP[column.type](token)
else:
token = None
if not tq_modes.check_mode(token, column.mode):
raise ValueError("Bad token for mode %s, got %s" %
(column.mode, token))
column.values.append(token)
result_table.num_rows += 1
self.load_table_or_view(result_table)
def load_table_from_csv(self, table_name, raw_schema, filename):
result_table = self.make_empty_table(table_name, raw_schema)
with open(filename, 'r') as f:
for line in f:
if line[-1] == '\n':
line = line[:-1]
tokens = line.split(',')
assert len(tokens) == len(result_table.columns), (
'Expected {} tokens on line {}, but got {}'.format(
len(result_table.columns), line, len(tokens)))
for token, column in zip(tokens,
result_table.columns.itervalues()):
# Run a casting function over the value we are given.
# CSV doesn't have a null value, so the string 'null' is
# used as the null value.
if token != 'null' or column.mode != tq_modes.NULLABLE:
token = tq_types.CAST_FUNCTION_MAP[column.type](token)
else:
token = None
if not tq_modes.check_mode(token, column.mode):
raise ValueError("Bad token for mode %s, got %s" %
(column.mode, token))
column.values.append(token)
result_table.num_rows += 1
self.load_table_or_view(result_table)
def process_row(row):
for (key, value) in row.iteritems():
mode = result_table.columns[key].mode
token = run_cast_function(key, mode, value)
if not tq_modes.check_mode(token, mode):
raise ValueError("Bad token for mode %s, got %s" %
(mode, token))
result_table.columns[key].values.append(token)
def process_row(row):
for (key, value) in row.iteritems():
mode = result_table.columns[key].mode
token = run_cast_function(key, mode, value)
if not tq_modes.check_mode(token, mode):
raise ValueError(
"Bad token for mode %s, got %s" % (
mode, token))
result_table.columns[key].values.append(
token)
