def process(self):
start_seconds = clock()
workspace = arcpy.env.workspace
arcpy.env.workspace = self.cadastre
self.log.debug('removing index')
try:
arcpy.RemoveIndex_management(in_table='PLSSPoint_AGRC',
index_name='webquery')
except Exception as e:
self.log.warn('error removing PLSS index: %s', e)
self.log.debug('adding index')
try:
arcpy.AddIndex_management(in_table='PLSSPoint_AGRC',
fields='POINTID',
index_name='webquery')
except Exception as e:
self.log.warn('error adding parcel index: %s', e)
arcpy.env.workspace = workspace
self.log.debug('finished PLSS processing %s',
seat.format_time(clock() - start_seconds))
def run(self):
print('compressing and analyzing: ' + self.name)
start_time = clock()
result = PoolResult(self.name)
arcpy.management.Compress(self.sde_connection)
arcpy.management.AnalyzeDatasets(self.sde_connection, 'SYSTEM')
for connection in self.user_connections:
user_name = arcpy.Describe(connection).connectionProperties.user
print('user: '******'.')[1].upper() == user_name.upper()]
if len(tables) > 0:
temp_start = clock()
arcpy.management.AnalyzeDatasets(connection, 'NO_SYSTEM', tables)
print(format_time(clock() - temp_start))
else:
message = 'no tables found for user connection: ' + connection
result.success = False
result.error_message = '{}: {}'.format(self.name, message)
result.elapsed_time = clock() - start_time
return result
def process(self):
#: squash 29 county layers into 1 with 9 attributes used by the app
start_seconds = clock()
workspace = arcpy.env.workspace
arcpy.env.workspace = self.destination_workspace
self.log.info("creating statewide parcel layer %s", self.destination_fc_name)
self._create_destination_table(self.destination_workspace, self.destination_fc_name)
for crate in self._crates:
self.log.info("appending crate %s", crate.source_name)
arcpy.Append_management(inputs=crate.destination, target=self.destination_fc_name, schema_type="NO_TEST")
county_name = crate.destination_name.replace("Parcels_", "")
self.log.debug("updating field names")
with arcpy.da.UpdateCursor(
in_table=self.destination_fc_name, field_names="County", where_clause="County IS NULL OR County = ''"
) as cursor:
for row in cursor:
row[0] = county_name
cursor.updateRow(row)
self.log.debug("removing index")
try:
arcpy.RemoveIndex_management(in_table=self.destination_fc_name, index_name="webquery")
except Exception as e:
self.log.warn("error removing parcel index: %s", e.message)
self.log.debug("adding index")
try:
arcpy.AddIndex_management(
in_table=self.destination_fc_name, fields="PARCEL_ID;County", index_name="webquery"
)
except Exception as e:
self.log.warn("error adding parcel index: %s", e.message)
self.log.debug("compacting %s", self.temporary_workspace)
arcpy.Compact_management(self.temporary_workspace)
arcpy.env.workspace = workspace
self.log.debug("finished parcel processing %s", seat.format_time(clock() - start_seconds))
def process(self):
start_seconds = clock()
workspace = arcpy.env.workspace
arcpy.env.workspace = self.cadastre
self.log.debug('removing index')
try:
arcpy.RemoveIndex_management(in_table='PLSSPoint_AGRC', index_name='webquery')
except Exception as e:
self.log.warn('error removing PLSS index: %s', e)
self.log.debug('adding index')
try:
arcpy.AddIndex_management(in_table='PLSSPoint_AGRC', fields='POINTID', index_name='webquery')
except Exception as e:
self.log.warn('error adding parcel index: %s', e)
arcpy.env.workspace = workspace
self.log.debug('finished PLSS processing %s', seat.format_time(clock() - start_seconds))
def create_locators(self):
#: address points
fields = [
"'Primary Table:Point Address ID' AddressPoints:OBJECTID VISIBLE NONE;'Primary Table:Street ID' <None> VISIBLE NONE;",
"'*Primary Table:House Number' AddressPoints:AddNum VISIBLE NONE;'Primary Table:Side' <None> VISIBLE NONE;",
"'Primary Table:Full Street Name' <None> VISIBLE NONE;'Primary Table:Prefix Direction' AddressPoints:PrefixDir VISIBLE NONE;",
"'Primary Table:Prefix Type' <None> VISIBLE NONE;'*Primary Table:Street Name' AddressPoints:STREETNAME VISIBLE NONE;",
"'Primary Table:Suffix Type' AddressPoints:STREETTYPE VISIBLE NONE;'Primary Table:Suffix Direction' AddressPoints:SUFFIXDIR VISIBLE NONE;",
"'Primary Table:City or Place' AddressPoints:AddSystem VISIBLE NONE;'Primary Table:County' <None> VISIBLE NONE;",
"'Primary Table:State' <None> VISIBLE NONE;'Primary Table:State Abbreviation' <None> VISIBLE NONE;'Primary Table:ZIP Code' <None> VISIBLE NONE;",
"'Primary Table:Country Code' <None> VISIBLE NONE;'Primary Table:3-Digit Language Code' <None> VISIBLE NONE;",
"'Primary Table:2-Digit Language Code' <None> VISIBLE NONE;'Primary Table:Admin Language Code' <None> VISIBLE NONE;",
"'Primary Table:Block ID' <None> VISIBLE NONE;'Primary Table:Street Rank' <None> VISIBLE NONE;'Primary Table:Display X' <None> VISIBLE NONE;",
"'Primary Table:Display Y' <None> VISIBLE NONE;'Primary Table:Min X value for extent' <None> VISIBLE NONE;",
"'Primary Table:Max X value for extent' <None> VISIBLE NONE;'Primary Table:Min Y value for extent' <None> VISIBLE NONE;",
"'Primary Table:Max Y value for extent' <None> VISIBLE NONE;'Primary Table:Additional Field' <None> VISIBLE NONE;",
"'*Primary Table:Altname JoinID' AddressPoints:UTAddPtID VISIBLE NONE;'Primary Table:City Altname JoinID' <None> VISIBLE NONE;",
"'*Alternate Name Table:JoinID' AtlNamesAddrPnts:UTAddPtID VISIBLE NONE;'Alternate Name Table:Full Street Name' <None> VISIBLE NONE;",
"'Alternate Name Table:Prefix Direction' AtlNamesAddrPnts:PrefixDir VISIBLE NONE;'Alternate Name Table:Prefix Type' <None> VISIBLE NONE;",
"'Alternate Name Table:Street Name' AtlNamesAddrPnts:STREETNAME VISIBLE NONE;",
"'Alternate Name Table:Suffix Type' AtlNamesAddrPnts:STREETTYPE VISIBLE NONE;",
"'Alternate Name Table:Suffix Direction' AtlNamesAddrPnts:SUFFIXDIR VISIBLE NONE"
]
start_seconds = clock()
process_seconds = clock()
self.log.info('creating the %s locator', 'address point')
try:
output_location = join(self.output_location, 'AddressPoints_AddressSystem')
arcpy.geocoding.CreateAddressLocator(
in_address_locator_style='US Address - Single House',
in_reference_data='{0}/{1};{0}/{2}'.format(self.locators, "AtlNamesAddrPnts 'Alternate Name Table'", "AddressPoints 'Primary Table'"),
in_field_map=''.join(fields),
out_address_locator=output_location,
config_keyword='',
enable_suggestions='DISABLED')
self.update_locator_properties(output_location, template.us_single_house_addresses)
except Exception as e:
self.log.error(e)
self.log.info('finished %s', format_time(clock() - process_seconds))
process_seconds = clock()
#: streets
fields = [
"'Primary Table:Feature ID' GeocodeRoads:OBJECTID VISIBLE NONE;'*Primary Table:From Left' GeocodeRoads:FROMADDR_L VISIBLE NONE;",
"'*Primary Table:To Left' GeocodeRoads:TOADDR_L VISIBLE NONE;'*Primary Table:From Right' GeocodeRoads:FROMADDR_R VISIBLE NONE;",
"'*Primary Table:To Right' GeocodeRoads:TOADDR_R VISIBLE NONE;'Primary Table:Left Parity' <None> VISIBLE NONE;",
"'Primary Table:Right Parity' <None> VISIBLE NONE;'Primary Table:Full Street Name' <None> VISIBLE NONE;",
"'Primary Table:Prefix Direction' GeocodeRoads:PREDIR VISIBLE NONE;'Primary Table:Prefix Type' <None> VISIBLE NONE;",
"'*Primary Table:Street Name' GeocodeRoads:NAME VISIBLE NONE;'Primary Table:Suffix Type' GeocodeRoads:POSTTYPE VISIBLE NONE;",
"'Primary Table:Suffix Direction' GeocodeRoads:POSTDIR VISIBLE NONE;", "'Primary Table:Left City or Place' GeocodeRoads:ADDRSYS_L VISIBLE NONE;",
"'Primary Table:Right City or Place' GeocodeRoads:ADDRSYS_R VISIBLE NONE;'Primary Table:Left County' <None> VISIBLE NONE;",
"'Primary Table:Right County' <None> VISIBLE NONE;'Primary Table:Left State' <None> VISIBLE NONE;'Primary Table:Right State' <None> VISIBLE NONE;",
"'Primary Table:Left State Abbreviation' <None> VISIBLE NONE;'Primary Table:Right State Abbreviation' <None> VISIBLE NONE;",
"'Primary Table:Left ZIP Code' <None> VISIBLE NONE;'Primary Table:Right ZIP Code' <None> VISIBLE NONE;'Primary Table:Country Code' <None> VISIBLE NONE;",
"'Primary Table:3-Digit Language Code' <None> VISIBLE NONE;'Primary Table:2-Digit Language Code' <None> VISIBLE NONE;",
"'Primary Table:Admin Language Code' <None> VISIBLE NONE;'Primary Table:Left Block ID' <None> VISIBLE NONE;",
"'Primary Table:Right Block ID' <None> VISIBLE NONE;'Primary Table:Left Street ID' <None> VISIBLE NONE;",
"'Primary Table:Right Street ID' <None> VISIBLE NONE;'Primary Table:Street Rank' <None> VISIBLE NONE;",
"'Primary Table:Min X value for extent' <None> VISIBLE NONE;'Primary Table:Max X value for extent' <None> VISIBLE NONE;",
"'Primary Table:Min Y value for extent' <None> VISIBLE NONE;'Primary Table:Max Y value for extent' <None> VISIBLE NONE;",
"'Primary Table:Left Additional Field' <None> VISIBLE NONE;'Primary Table:Right Additional Field' <None> VISIBLE NONE;",
"'*Primary Table:Altname JoinID' GeocodeRoads:GLOBALID_SGID VISIBLE NONE;'Primary Table:City Altname JoinID' <None> VISIBLE NONE;",
"'*Alternate Name Table:JoinID' AtlNamesRoads:GLOBALID_SGID VISIBLE NONE;'Alternate Name Table:Full Street Name' <None> VISIBLE NONE;",
"'Alternate Name Table:Prefix Direction' AtlNamesRoads:PREDIR VISIBLE NONE;'Alternate Name Table:Prefix Type' <None> VISIBLE NONE;",
"'Alternate Name Table:Street Name' AtlNamesRoads:NAME VISIBLE NONE;'Alternate Name Table:Suffix Type' AtlNamesRoads:POSTTYPE VISIBLE NONE;",
"'Alternate Name Table:Suffix Direction' AtlNamesRoads:POSTDIR VISIBLE NONE"
]
self.log.info('creating the %s locator', 'streets')
try:
output_location = join(self.output_location, 'Roads_AddressSystem_STREET')
arcpy.geocoding.CreateAddressLocator(
in_address_locator_style='US Address - Dual Ranges',
in_reference_data='{0}/{1};{0}/{2}'.format(self.locators, "GeocodeRoads 'Primary Table'", "AtlNamesRoads 'Alternate Name Table'"),
in_field_map=''.join(fields),
out_address_locator=output_location,
config_keyword='',
enable_suggestions='DISABLED')
self.update_locator_properties(output_location, template.us_dual_range_addresses)
except Exception as e:
self.log.error(e)
self.log.info('finished %s', format_time(clock() - process_seconds))
process_seconds = clock()
self.log.info('finished %s', format_time(clock() - process_seconds))
self.log.info('done %s', format_time(clock() - start_seconds))
def ship(self):
database_infos = [
#: (name, sde connection, [user connections])
#: Name corresponds with the subfolder with the garage folder where the
#: connection files are.
('FiberVerification', 'DC_sde@[email protected]',
['DC_FiberAdmin@[email protected]']),
('FFSL', 'DNR_sde@[email protected]',
['DNR_Fire@[email protected]',
'DNR_Forestry@[email protected]',
'DNR_Lands@[email protected]']),
('PLSSFabric', 'sde.sde', ['admin.sde']),
('SGID10', 'SGID_sde@[email protected]',
['SGID_Biosciense@[email protected]',
'SGID_Boundaries@[email protected]',
'SGID_Cadastre@[email protected]',
'SGID_Climate@[email protected]',
'SGID_Demographics@[email protected]',
'SGID_Economy@[email protected]',
'SGID_Elevation@[email protected]',
'SGID_Energy@[email protected]',
'SGID_Environment@[email protected]',
'SGID_Farming@[email protected]',
'SGID_Geoscience@[email protected]',
'SGID_Health@[email protected]',
'SGID_History@[email protected]',
'SGID_Indices@[email protected]',
'SGID_Location@[email protected]',
'SGID_Planning@[email protected]',
'SGID_Political@[email protected]',
'SGID_Raster@[email protected]',
'SGID_Recreation@[email protected]',
'SGID_Society@[email protected]',
'SGID_Transportation@[email protected]',
'SGID_Utilities@[email protected]',
'SGID_Water@[email protected]']),
('UDNR', 'DNR_sde@[email protected]',
['DNR_DPR@[email protected]',
'DNR_DWR@[email protected]',
'DNR_FFSL@[email protected]',
'DNR_OGM@[email protected]',
'DNR_UGS@[email protected]',
'DNR_WRE@[email protected]',
'DNR_WRT@[email protected]']),
('UDPR', 'DNR_sde@[email protected]',
['DNR_OHV@[email protected]',
'DNR_susanz@[email protected]']),
('UDWR', 'DNR_sde@[email protected]', ['DNR_DWRADMIN@[email protected]']),
('UEMP', 'sde@[email protected]', ['uempadmin@[email protected]']),
('UGGP', 'sde@[email protected]', ['uggpadmin@[email protected]']),
('UGHP', 'sde@[email protected]', ['ughpadmin@[email protected]']),
('UGIO', 'DNR_sde@[email protected]', ['DNR_ugioadmin@[email protected]']),
('UGMP', 'DC_sde@[email protected]', ['DNR_UGMPADMIN@[email protected]']),
('UTAX', 'DC_sde@[email protected]', ['DC_ustccass@[email protected]']),
('UTRANS', 'DC_sde@[email protected]', ['DC_TRANSADMIN@[email protected]'])
]
with Pool(MAX_PROCESSES) as pool:
results = list(pool.map(run, [(self.garage,) + info for info in database_infos]))
#: log times
for result in results:
self.log.info('Compress/Analyze time for {}: {}'.format(result.name, format_time(result.elapsed_time)))
if False in [result.success for result in results]:
message = '\n\n'.join([result.error_message for result in results if result.error_message is not None])
self.log.error('Errors during multiprocessing: ' + message)
self.success = (False, message)
def test_format_time_hours(self):
self.assertEqual(seat.format_time(7200.0), '2.0 hours')
self.assertEqual(seat.format_time(5410.0), '1.5 hours')
def test_format_time_seconds(self):
self.assertEqual(seat.format_time(80.0), '80.0 seconds')
self.assertEqual(seat.format_time(50.0), '50.0 seconds')
def test_format_time_minutes(self):
self.assertEqual(seat.format_time(91.0), '1.52 minutes')
self.assertEqual(seat.format_time(1800.0), '30.0 minutes')
def test_format_time_milliseconds(self):
self.assertEqual(seat.format_time(5.5), '5500 ms')
def test_format_time_hours(self):
self.assertEqual(seat.format_time(7200), '2.0 hours')
self.assertEqual(seat.format_time(5410), '1.5 hours')
def test_format_time_minutes(self):
self.assertEqual(seat.format_time(91), '1.52 minutes')
self.assertEqual(seat.format_time(1800), '30.0 minutes')
def test_format_time_seconds(self):
self.assertEqual(seat.format_time(80), '80.0 seconds')
self.assertEqual(seat.format_time(50), '50.0 seconds')
def test_format_time_milliseconds(self):
self.assertEqual(seat.format_time(5.5), '5500 ms')
