def get_field_mappings(schema):
"""Maps Fiona data types to Redshift data types for each in the schema
:param schema: Fiona schema
:return: Dictionary with the data type for each field
"""
field_mappings = OrderedDict()
for property in schema['properties']:
property_type = fiona.prop_type(schema['properties'][property])
if property_type == type(int()):
# Redshift data types: INTEGER, BIGINT
field_mappings[property] = 'BIGINT'
elif property_type == type(float()):
# Redshift data types: REAL, DOUBLE PRECISION
field_mappings[property] = 'DOUBLE PRECISION'
elif property_type == type(str()):
length = fiona.prop_width(schema['properties'][property])
field_mappings[property] = 'VARCHAR({0})'.format(length)
elif property_type == type(bool()):
field_mappings[property] = 'BOOLEAN'
elif "FionaDateType" in str(property_type):
field_mappings[property] = 'DATE'
elif "FionaTimeType" in str(property_type):
field_mappings[property] = 'TIME'
elif "FionaDateTimeType" in str(property_type):
field_mappings[property] = 'TIMESTAMP'
else:
# If it is a different type, we will use VARCHAR(MAX)
field_mappings[property] = 'VARCHAR(MAX)'
return field_mappings
def validate_attribute_field_info(attribute_field_info):
# make sure it's a list
if not isinstance(attribute_field_info, list):
raise TypeError("attribute_field_info must be a list")
# make sure each element is a dictionary with the correct attributes
expected_keys = ['name', 'dtype', 'value']
for d in attribute_field_info:
if isinstance(d, dict) is False:
raise TypeError(
"Elements of attribute_field_info must be a dictionary.")
if sorted(expected_keys) != sorted(list(d.keys())):
raise KeyError(
"Keys in attributed_field_info do not match the expected set: {}"
.format(expected_keys))
try:
fiona.prop_type(d['dtype'])
fiona.prop_width(d['dtype'])
except Exception as e:
raise TypeError("Specified dtype {dtype} is invalid: {e}".format(
dtype=d['dtype'], e=str(e)))
def test_other():
assert prop_width('int') == None
assert prop_width('float') == None
assert prop_width('date') == None
def test_str():
assert prop_width('str:254') == 254
assert prop_width('str') == 80
# -*- coding: utf-8 -*-
###############################################################################
from pprint import pprint
import fiona
c = fiona.open('/gdata/GSHHS_c.shp')
pprint(c.schema)
###############################################################################
rec = next(c)
set(rec.keys()) - set(c.schema.keys())
set(rec['properties'].keys()) == set(c.schema['properties'].keys())
###############################################################################
type(rec['properties']['source'])
c.schema['properties']['source']
###############################################################################
from fiona import prop_width
prop_width('str:25')
prop_width('str')
###############################################################################
from fiona import prop_type
prop_type('int')
prop_type('float')
prop_type('str:25')
###############################################################################
c = fiona.open('/gdata/GSHHS_c.shp')
rec = c.next()
pprint(rec)
###############################################################################
c.close()
rec['id']
###############################################################################
c = fiona.open('/gdata/GSHHS_c.shp')
def test_width_other():
assert prop_width('int') == None
assert prop_width('float') == None
assert prop_width('date') == None
def test_width_str():
assert prop_width('str:254') == 254
assert prop_width('str') == 80
def test_width_other():
assert prop_width("int") == None
assert prop_width("float") == None
assert prop_width("date") == None
def test_width_str():
assert prop_width("str:254") == 254
assert prop_width("str") == 80