def test_hierarchical_record_with_timestamps(self):
record = {"global": "2001-01-01", "user": {"local": "2001-01-01"}}
schema_with_ts = [
{"name": "global", "type": "timestamp", "mode": "nullable"},
{"name": "user", "type": "record", "mode": "nullable",
"fields": [{
"name": "local",
"type": "timestamp",
"mode": "nullable"}]}]
schema_without_ts = [
{"name": "global", "type": "string", "mode": "nullable"},
{"name": "user", "type": "record", "mode": "nullable",
"fields": [{
"name": "local",
"type": "string",
"mode": "nullable"}]}]
self.assertItemsEqual(
schema_from_record(record),
schema_with_ts)
self.assertItemsEqual(
schema_from_record(record, timestamp_parser=lambda x: False),
schema_without_ts)
def test_nested_invalid_type_reported_correctly(self):
key = "wrong answer"
value = "wrong answer"
try:
schema_from_record({"a": {"b": [{"c": None}]}})
except InvalidTypeException, e:
key = e.key
value = e.value
def test_nested_invalid_type_reported_correctly(self):
key = "wrong answer"
value = "wrong answer"
try:
schema_from_record({"a": {"b": [{"c": None}]}})
except InvalidTypeException, e:
key = e.key
value = e.value
def test_nested_invalid_type_reported_correctly(self):
key = "wrong answer"
value = "wrong answer"
try:
schema_from_record({"a": {"b": [{"c": None}]}})
except InvalidTypeException as e:
key = e.key
value = e.value
self.assertEqual(key, "a.b.c")
self.assertEqual(value, None)
def test_nested_invalid_type_reported_correctly(self):
key = "wrong answer"
value = "wrong answer"
try:
schema_from_record({"a": {"b": [{"c": None}]}})
except InvalidTypeException as e:
key = e.key
value = e.value
self.assertEqual(key, "a.b.c")
self.assertEqual(value, None)
def test_hierarchical_record(self):
record = {"user": {"username": "******", "id": 123}}
schema = [{"name": "user", "type": "record", "mode": "nullable",
"fields": [{"name": "username", "type": "string", "mode":
"nullable"}, {"name": "id", "type": "integer",
"mode": "nullable"}]}]
self.assertItemsEqual(schema_from_record(record), schema)
def test_hierarchical_record_with_timestamps(self):
record = {"global": "2001-01-01", "user": {"local": "2001-01-01"}}
schema_with_ts = [{
"name": "global",
"type": "timestamp",
"mode": "nullable"
}, {
"name":
"user",
"type":
"record",
"mode":
"nullable",
"fields": [{
"name": "local",
"type": "timestamp",
"mode": "nullable"
}]
}]
schema_without_ts = [{
"name": "global",
"type": "string",
"mode": "nullable"
}, {
"name":
"user",
"type":
"record",
"mode":
"nullable",
"fields": [{
"name": "local",
"type": "string",
"mode": "nullable"
}]
}]
six.assertCountEqual(self, schema_from_record(record), schema_with_ts)
six.assertCountEqual(
self, schema_from_record(record, timestamp_parser=lambda x: False),
schema_without_ts)
def test_hierarchical_record(self):
record = {"user": {"username": "******", "id": 123}}
schema = [{"name": "user", "type": "record", "mode": "nullable",
"fields": [{"name": "username", "type": "string", "mode":
"nullable"}, {"name": "id", "type": "integer",
"mode": "nullable"}]}]
generated_schema = schema_from_record(record)
schema_fields = schema[0].pop('fields')
generated_fields = generated_schema[0].pop('fields')
six.assertCountEqual(self, schema_fields, generated_fields)
six.assertCountEqual(self, generated_schema, schema)
def test_simple_record(self):
record = {"username": "******", "id": 123}
schema = [{
"name": "username",
"type": "string",
"mode": "nullable"
}, {
"name": "id",
"type": "integer",
"mode": "nullable"
}]
self.assertItemsEqual(schema_from_record(record), schema)
def schema_from_record(cls, record):
"""Given a dict representing a record instance to be inserted into
BigQuery, calculate the schema.
Args:
record: dict representing a record to be inserted into big query,
where all keys are strings (representing column names in
the record) and all values are of type int, str, unicode,
float,bool, timestamp or dict. A dict value represents a
record, and must conform to the same restrictions as record
Returns:
a list representing a BigQuery schema
Note: results are undefined if a different value types are provided for
a repeated field: E.g.
{ rfield: [ { x: 1}, {x: "a string"} ] } # undefined!
"""
return schema_from_record(record)
def schema_from_record(cls, record):
"""Given a dict representing a record instance to be inserted into
BigQuery, calculate the schema.
Args:
record: dict representing a record to be inserted into big query,
where all keys are strings (representing column names in
the record) and all values are of type int, str, unicode,
float,bool, timestamp or dict. A dict value represents a
record, and must conform to the same restrictions as record
Returns:
a list representing a BigQuery schema
Note: results are undefined if a different value types are provided for
a repeated field: E.g.
{ rfield: [ { x: 1}, {x: "a string"} ] } # undefined!
"""
return schema_from_record(record)
def test_hierarchical_record(self):
record = {"user": {"username": "******", "id": 123}}
schema = [{
"name":
"user",
"type":
"record",
"mode":
"nullable",
"fields": [{
"name": "username",
"type": "string",
"mode": "nullable"
}, {
"name": "id",
"type": "integer",
"mode": "nullable"
}]
}]
self.assertItemsEqual(schema_from_record(record), schema)
def test_hierarchical_record(self):
record = {"user": {"username": "******", "id": 123}}
schema = [{
"name":
"user",
"type":
"record",
"mode":
"nullable",
"fields": [{
"name": "username",
"type": "string",
"mode": "nullable"
}, {
"name": "id",
"type": "integer",
"mode": "nullable"
}]
}]
generated_schema = schema_from_record(record)
schema_fields = schema[0].pop('fields')
generated_fields = generated_schema[0].pop('fields')
six.assertCountEqual(self, schema_fields, generated_fields)
six.assertCountEqual(self, generated_schema, schema)
def test_simple_record(self):
record = {"username": "******", "id": 123}
schema = [{"name": "username", "type": "string", "mode": "nullable"},
{"name": "id", "type": "integer", "mode": "nullable"}]
self.assertItemsEqual(schema_from_record(record), schema)
def test_repeated_field(self):
record = {"ids": [1, 2, 3, 4, 5]}
schema = [{"name": "ids", "type": "integer", "mode": "repeated"}]
self.assertItemsEqual(schema_from_record(record), schema)
def test_repeated_field(self):
record = {"ids": [1, 2, 3, 4, 5]}
schema = [{"name": "ids", "type": "integer", "mode": "repeated"}]
self.assertItemsEqual(schema_from_record(record), schema)
