def merge_where_field(self, m):
"""
Ultimately used to build a unique where clause from the source data. If specified, any values in this field
in the source dataset (i.e. the country field for a MEX layer) will be used to delete from the global output
(DELETE FROM gfw_mining WHERE country = 'MEX') and then the source will be appended
:param m: the merge feild
:return:
"""
if m:
if m not in util.list_fields(self.source, self.gfw_env):
logging.debug(
"Where clause field {0} specified for merge_where_field but "
"field not in source dataset".format(m))
if m not in util.list_fields(self.esri_service_output,
self.gfw_env):
logging.error(
"Where clause field {0} specified for merge_where_field but "
"field not in esri_service_output. Data from this field will not be"
"appended due to the NO_TEST approach. Exiting ".format(m))
sys.exit(1)
if m not in util.list_fields(self.cartodb_service_output,
self.gfw_env):
logging.error(
"Where clause field {0} specified for merge_where_field but "
"field not in cartodb_service_output. Data from this field will not be"
"appended due to the NO_TEST approach. Exiting ".format(m))
sys.exit(1)
else:
m = None
self._merge_where_field = m
def merge_where_field(self, m):
"""
Ultimately used to build a unique where clause from the source data. If specified, any values in this field
in the source dataset (i.e. the country field for a MEX layer) will be used to delete from the global output
(DELETE FROM gfw_mining WHERE country = 'MEX') and then the source will be appended
:param m: the merge feild
:return:
"""
if m:
if m not in util.list_fields(self.source, self.gfw_env):
logging.debug("Where clause field {0} specified for merge_where_field but "
"field not in source dataset".format(m))
if m not in util.list_fields(self.esri_service_output, self.gfw_env):
logging.error("Where clause field {0} specified for merge_where_field but "
"field not in esri_service_output. Data from this field will not be"
"appended due to the NO_TEST approach. Exiting ".format(m))
sys.exit(1)
if m not in util.list_fields(self.cartodb_service_output, self.gfw_env):
logging.error("Where clause field {0} specified for merge_where_field but "
"field not in cartodb_service_output. Data from this field will not be"
"appended due to the NO_TEST approach. Exiting ".format(m))
sys.exit(1)
else:
m = None
self._merge_where_field = m
def check_country_fields(self, in_fc_list):
"""
Check the input feature classes to be sure they have a country field
Important if we're going to use country as a way to append/delete features
:param in_fc_list: List of FCs/cartoDB tables to check
:return:
"""
result_list = []
for fc in in_fc_list:
if 'country' in util.list_fields(fc, self.gfw_env):
result_list.append(True)
else:
logging.debug("No country code found for {0}".format(fc))
result_list.append(False)
# If all result_list values are True, return True
if len(set(result_list)) == 1 and result_list[0]:
result = True
else:
result = False
return result
def update_gfwid(self):
"""
For each row, take the hash of the well known text representation of the geometry
This will be used in the API to cache analysis results for geometries previously analyzed
:return:
"""
logging.debug('Starting vector_layer.update_gfwid for {0}'.format(
self.name))
if "gfwid" not in util.list_fields(self.source, self.gfw_env):
arcpy.AddField_management(self.source,
"gfwid",
"TEXT",
field_length=50,
field_alias="GFW ID")
# Required to prevent calcuate field failures-- will likely fail to hash the !Shape! object if there are
# null geometries
logging.debug('Starting repair geometry')
arcpy.RepairGeometry_management(self.source, "DELETE_NULL")
logging.debug('Starting to calculate gfwid')
arcpy.CalculateField_management(self.source,
"gfwid",
"md5(!Shape!.WKT)",
"PYTHON_9.3",
code_block="import hashlib\n"
"def md5(shape):\n"
" hash = hashlib.md5()\n"
" hash.update(shape)\n"
" return hash.hexdigest()")
def delete_features_input_where_clause(self, f):
"""
Used to filter the input dataset. If this exists, the source will be copied locally, then use this where
clause to delete records before integrating with the rest of the update process
:param f: the where clause
:return:
"""
if f:
if "'" not in f and '"' not in f:
logging.debug(
"delete_features_input_where_clause {0} doesn't have quoted strings. "
"It probably should.".format(f))
where_clause_field_name = f.split()[0].replace("'", "").replace(
'"', "")
if where_clause_field_name in util.list_fields(
self.source, self.gfw_env):
try:
arcpy.MakeFeatureLayer_management(self.source, self.name,
f)
# Clean up temporary feature layer after we create it
arcpy.Delete_management(self.name)
except arcpy.ExecuteError:
logging.error(
"delete_features_input_where_clause '{0!s}' is invalide "
"or delete FL failed".format(f))
sys.exit(1)
self._delete_features_input_where_clause = f
def check_country_fields(self, in_fc_list):
"""
Check the input feature classes to be sure they have a country field
Important if we're going to use country as a way to append/delete features
:param in_fc_list: List of FCs/cartoDB tables to check
:return:
"""
result_list = []
for fc in in_fc_list:
if 'country' in util.list_fields(fc, self.gfw_env):
result_list.append(True)
else:
logging.debug("No country code found for {0}".format(fc))
result_list.append(False)
# If all result_list values are True, return True
if len(set(result_list)) == 1 and result_list[0]:
result = True
else:
result = False
return result
def delete_features_input_where_clause(self, f):
"""
Used to filter the input dataset. If this exists, the source will be copied locally, then use this where
clause to delete records before integrating with the rest of the update process
:param f: the where clause
:return:
"""
if f:
if "'" not in f and '"' not in f:
logging.debug("delete_features_input_where_clause {0} doesn't have quoted strings. "
"It probably should.".format(f))
where_clause_field_name = f.split()[0].replace("'", "").replace('"', "")
if where_clause_field_name in util.list_fields(self.source, self.gfw_env):
try:
arcpy.MakeFeatureLayer_management(self.source, self.name, f)
# Clean up temporary feature layer after we create it
arcpy.Delete_management(self.name)
except arcpy.ExecuteError:
logging.error("delete_features_input_where_clause '{0!s}' is invalide "
"or delete FL failed".format(f))
sys.exit(1)
self._delete_features_input_where_clause = f
def add_and_populate_country_field(self, in_fc):
"""
Add a field for country if it doesn't exist and populate it with country value
:param in_fc: fc to add 'country' to
:return: True/False based on whether country added; will be used later to delete the field if added
"""
country_added = False
if 'country' not in util.list_fields(in_fc, self.gfw_env):
util.add_field_and_calculate(in_fc, 'country', 'TEXT', 3, self.add_country_value, self.gfw_env)
country_added = True
return country_added
def add_and_populate_country_field(self, in_fc):
"""
Add a field for country if it doesn't exist and populate it with country value
:param in_fc: fc to add 'country' to
:return: True/False based on whether country added; will be used later to delete the field if added
"""
country_added = False
if 'country' not in util.list_fields(in_fc, self.gfw_env):
util.add_field_and_calculate(in_fc, 'country', 'TEXT', 3,
self.add_country_value, self.gfw_env)
country_added = True
return country_added
def update_gfwid(self):
"""
For each row, take the hash of the well known text representation of the geometry
This will be used in the API to cache analysis results for geometries previously analyzed
:return:
"""
logging.debug('Starting vector_layer.update_gfwid for {0}'.format(self.name))
if "gfwid" not in util.list_fields(self.source, self.gfw_env):
arcpy.AddField_management(self.source, "gfwid", "TEXT", field_length=50, field_alias="GFW ID")
# Required to prevent calcuate field failures-- will likely fail to hash the !Shape! object if there are
# null geometries
logging.debug('Starting repair geometry')
arcpy.RepairGeometry_management(self.source, "DELETE_NULL")
logging.debug('Starting to calculate gfwid')
arcpy.CalculateField_management(self.source, "gfwid", "md5(!Shape!.WKT)", "PYTHON_9.3",
code_block="import hashlib\n"
"def md5(shape):\n"
" hash = hashlib.md5()\n"
" hash.update(shape)\n"
" return hash.hexdigest()"
)