def handle(self, *args, **options):
try:
db_path = args[0]
assert os.path.exists(db_path)
except (AssertionError, IndexError):
raise CommandError("Specify path for gyb sqlite database (data.db)")
# confirm that database contains a viable trees_fvsaggregate table
gybconn = sqlite3.connect(db_path)
gybcursor = gybconn.cursor()
gybcursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
tables = [x[0] for x in gybcursor.fetchall()]
if 'trees_fvsaggregate' not in tables:
raise CommandError("trees_fvsaggregate table not found in {}".format(db_path))
# Confirm that gyb's schema is sufficient to provide all FP schema fields
gybcursor.execute("PRAGMA table_info(trees_fvsaggregate);")
gyb_fieldnames = [x[1] for x in gybcursor.fetchall()]
pgcursor = connection.cursor()
try:
pgcursor.execute("SELECT * FROM trees_fvsaggregate LIMIT 0;")
# don't track id, autosequenced
pg_fields = [desc for desc in pgcursor.description if desc[0] != 'id']
finally:
pgcursor.close()
pg_fieldnames = [x[0] for x in pg_fields]
# postgres schema is only allowed to deviate from gyb as follows...
# special cases described in match_case function below
assert set(pg_fieldnames) - set([x.lower() for x in gyb_fieldnames]) == set(['pp_btl', 'lp_btl'])
pg_insert_cols = ", ".join(['"{}"'.format(f) for f in pg_fieldnames])
def match_case(pgname):
"""return case-sensitive key name from sqlite given a postgres field name"""
for gybfield in gyb_fieldnames:
if pgname == gybfield.lower():
return gybfield
# not present, special cases
# If forestplanner schema calls for pp_btl or lp_btl
# substitute PINEBTL
if pgname == 'pp_btl':
return "PINEBTL"
if pgname == 'lp_btl':
return "PINEBTL"
raise Exception("Can't find {} in sqlite fields".format(pgname))
gyb_values_template = ", ".join(
["%({})s".format(match_case(x[0])) for x in pg_fields]
)
query_template = """INSERT INTO trees_fvsaggregate ({})
VALUES ({});""".format(pg_insert_cols, gyb_values_template)
pgcursor = connection.cursor()
try:
with transaction.commit_on_success():
pgcursor.executemany(query_template, get_gyb_rows(db_path, gyb_fieldnames))
finally:
pgcursor.close()
print("Recaching valid_condids.")
FVSAggregate.recache()
def handle(self, *args, **options):
try:
db_path = args[0]
assert os.path.exists(db_path)
except (AssertionError, IndexError):
raise CommandError(
"Specify path for gyb sqlite database (data.db)")
# confirm that database contains a viable trees_fvsaggregate table
gybconn = sqlite3.connect(db_path)
gybcursor = gybconn.cursor()
gybcursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
tables = [x[0] for x in gybcursor.fetchall()]
if 'trees_fvsaggregate' not in tables:
raise CommandError(
"trees_fvsaggregate table not found in {}".format(db_path))
# Confirm that gyb's schema is sufficient to provide all FP schema fields
gybcursor.execute("PRAGMA table_info(trees_fvsaggregate);")
gyb_fieldnames = [x[1] for x in gybcursor.fetchall()]
pgcursor = connection.cursor()
try:
pgcursor.execute("SELECT * FROM trees_fvsaggregate LIMIT 0;")
# don't track id, autosequenced
pg_fields = [
desc for desc in pgcursor.description if desc[0] != 'id'
]
finally:
pgcursor.close()
pg_fieldnames = [x[0] for x in pg_fields]
# postgres schema is only allowed to deviate from gyb as follows...
# special cases described in match_case function below
assert set(pg_fieldnames) - set([x.lower()
for x in gyb_fieldnames]) == set(
['pp_btl', 'lp_btl'])
pg_insert_cols = ", ".join(['"{}"'.format(f) for f in pg_fieldnames])
def match_case(pgname):
"""return case-sensitive key name from sqlite given a postgres field name"""
for gybfield in gyb_fieldnames:
if pgname == gybfield.lower():
return gybfield
# not present, special cases
# If forestplanner schema calls for pp_btl or lp_btl
# substitute PINEBTL
if pgname == 'pp_btl':
return "PINEBTL"
if pgname == 'lp_btl':
return "PINEBTL"
raise Exception("Can't find {} in sqlite fields".format(pgname))
gyb_values_template = ", ".join(
["%({})s".format(match_case(x[0])) for x in pg_fields])
query_template = """INSERT INTO trees_fvsaggregate ({})
VALUES ({});""".format(pg_insert_cols, gyb_values_template)
pgcursor = connection.cursor()
try:
with transaction.commit_on_success():
pgcursor.executemany(query_template,
get_gyb_rows(db_path, gyb_fieldnames))
finally:
pgcursor.close()
print("Recaching valid_condids.")
FVSAggregate.recache()
def import_ogr(self, shp_path, field_mapping=None, layer_num=0,
forest_property=None, new_property_name=None, pre_impute=False):
ds = DataSource(shp_path)
layer = ds[0]
num_features = len(layer)
field_mapping = self._validate_field_mapping(layer, field_mapping)
if not forest_property and not new_property_name:
raise Exception(
"Must provide either existing forest_property OR new_property_name")
if new_property_name:
# Calculating property outline from stands
stands = []
for feature in layer:
stands.append(wkt.loads(feature.geom.wkt))
casc_poly = cascaded_union(stands)
if casc_poly.type == 'MultiPolygon':
polys = []
for c in casc_poly:
# Identify small 'slivers' or areas of empty space b/t polygons that are unintentional
# If they're smaller than the threshold, remove them
interiors = [x for x in c.interiors if Polygon(
x).area > settings.SLIVER_THRESHOLD]
polys.append(Polygon(shell=c.exterior, holes=interiors))
elif casc_poly.type == 'Polygon':
# Identify small 'slivers' or areas of empty space b/t polygons that are unintentional
# If they're smaller than the threshold, remove them
interiors = [x for x in casc_poly.interiors if Polygon(
x).area > settings.SLIVER_THRESHOLD]
polys = [Polygon(shell=casc_poly.exterior, holes=interiors)]
casc = MultiPolygon(polys)
# Creating Property
self.forest_property = ForestProperty.objects.create(
user=self.user, name=new_property_name, geometry_final=casc.wkt)
else:
self.forest_property = forest_property
try:
# special case for user-inventory situation
# if there is a condid field, it is implicitly required.
use_condid = False
if 'condid' in layer.fields:
use_condid = True
variant = self.forest_property.variant
valid_condids = FVSAggregate.valid_condids(variant)
stands = []
stratum = {}
for feature in layer:
stand = Stand(
user=self.user,
name=str(datetime_to_unix(datetime.datetime.now())),
geometry_orig=feature.geom.geos)
for fname in self.optional_fields:
if fname in field_mapping.keys():
try:
stand.__dict__[fname] = feature.get(
field_mapping[fname])
except OGRIndexError:
pass
# If user inventory case, check each feature which must contain integer condids
# that refer to valid fvsaggregate records for that variant
if use_condid:
condid = feature.get('condid')
if condid not in valid_condids:
raise Exception('Condition id {} is not valid for the {} variant (check fvsaggregate table)'.format(condid, variant))
if condid in stratum.keys():
# use cached
strata = stratum[condid]
else:
# create it
kwargs = condid_strata(condid, self.forest_property.uid)
strata = Strata(user=self.user, name=condid, **kwargs)
strata.save(skip_validation=True) # no need for NN validation
self.forest_property.add(strata)
stratum[condid] = strata
stand.cond_id = condid
stand.locked_cond_id = condid
stand.strata = strata
stand.full_clean()
stands.append(stand)
del stand
for stand in stands:
stand.save()
self.forest_property.add(stand)
if pre_impute:
# Technically locked stands won't need terrain variables
# ... but terrain info is nice to have anyways for all stands.
# Note the work is done asynchronously to ensure fast uploads
stand.preimpute()
except:
# Any failures? Rollback and re-raise...
for stand in stands:
stand.delete()
if new_property_name:
self.forest_property.delete()
raise
return True