def process(self): start_seconds = clock() workspace = arcpy.env.workspace arcpy.env.workspace = self.cadastre self.log.debug('removing index') try: arcpy.RemoveIndex_management(in_table='PLSSPoint_AGRC', index_name='webquery') except Exception as e: self.log.warn('error removing PLSS index: %s', e) self.log.debug('adding index') try: arcpy.AddIndex_management(in_table='PLSSPoint_AGRC', fields='POINTID', index_name='webquery') except Exception as e: self.log.warn('error adding parcel index: %s', e) arcpy.env.workspace = workspace self.log.debug('finished PLSS processing %s', seat.format_time(clock() - start_seconds))
def run(self): print('compressing and analyzing: ' + self.name) start_time = clock() result = PoolResult(self.name) arcpy.management.Compress(self.sde_connection) arcpy.management.AnalyzeDatasets(self.sde_connection, 'SYSTEM') for connection in self.user_connections: user_name = arcpy.Describe(connection).connectionProperties.user print('user: '******'.')[1].upper() == user_name.upper()] if len(tables) > 0: temp_start = clock() arcpy.management.AnalyzeDatasets(connection, 'NO_SYSTEM', tables) print(format_time(clock() - temp_start)) else: message = 'no tables found for user connection: ' + connection result.success = False result.error_message = '{}: {}'.format(self.name, message) result.elapsed_time = clock() - start_time return result
def process(self): #: squash 29 county layers into 1 with 9 attributes used by the app start_seconds = clock() workspace = arcpy.env.workspace arcpy.env.workspace = self.destination_workspace self.log.info("creating statewide parcel layer %s", self.destination_fc_name) self._create_destination_table(self.destination_workspace, self.destination_fc_name) for crate in self._crates: self.log.info("appending crate %s", crate.source_name) arcpy.Append_management(inputs=crate.destination, target=self.destination_fc_name, schema_type="NO_TEST") county_name = crate.destination_name.replace("Parcels_", "") self.log.debug("updating field names") with arcpy.da.UpdateCursor( in_table=self.destination_fc_name, field_names="County", where_clause="County IS NULL OR County = ''" ) as cursor: for row in cursor: row[0] = county_name cursor.updateRow(row) self.log.debug("removing index") try: arcpy.RemoveIndex_management(in_table=self.destination_fc_name, index_name="webquery") except Exception as e: self.log.warn("error removing parcel index: %s", e.message) self.log.debug("adding index") try: arcpy.AddIndex_management( in_table=self.destination_fc_name, fields="PARCEL_ID;County", index_name="webquery" ) except Exception as e: self.log.warn("error adding parcel index: %s", e.message) self.log.debug("compacting %s", self.temporary_workspace) arcpy.Compact_management(self.temporary_workspace) arcpy.env.workspace = workspace self.log.debug("finished parcel processing %s", seat.format_time(clock() - start_seconds))
def create_locators(self): #: address points fields = [ "'Primary Table:Point Address ID' AddressPoints:OBJECTID VISIBLE NONE;'Primary Table:Street ID' <None> VISIBLE NONE;", "'*Primary Table:House Number' AddressPoints:AddNum VISIBLE NONE;'Primary Table:Side' <None> VISIBLE NONE;", "'Primary Table:Full Street Name' <None> VISIBLE NONE;'Primary Table:Prefix Direction' AddressPoints:PrefixDir VISIBLE NONE;", "'Primary Table:Prefix Type' <None> VISIBLE NONE;'*Primary Table:Street Name' AddressPoints:STREETNAME VISIBLE NONE;", "'Primary Table:Suffix Type' AddressPoints:STREETTYPE VISIBLE NONE;'Primary Table:Suffix Direction' AddressPoints:SUFFIXDIR VISIBLE NONE;", "'Primary Table:City or Place' AddressPoints:AddSystem VISIBLE NONE;'Primary Table:County' <None> VISIBLE NONE;", "'Primary Table:State' <None> VISIBLE NONE;'Primary Table:State Abbreviation' <None> VISIBLE NONE;'Primary Table:ZIP Code' <None> VISIBLE NONE;", "'Primary Table:Country Code' <None> VISIBLE NONE;'Primary Table:3-Digit Language Code' <None> VISIBLE NONE;", "'Primary Table:2-Digit Language Code' <None> VISIBLE NONE;'Primary Table:Admin Language Code' <None> VISIBLE NONE;", "'Primary Table:Block ID' <None> VISIBLE NONE;'Primary Table:Street Rank' <None> VISIBLE NONE;'Primary Table:Display X' <None> VISIBLE NONE;", "'Primary Table:Display Y' <None> VISIBLE NONE;'Primary Table:Min X value for extent' <None> VISIBLE NONE;", "'Primary Table:Max X value for extent' <None> VISIBLE NONE;'Primary Table:Min Y value for extent' <None> VISIBLE NONE;", "'Primary Table:Max Y value for extent' <None> VISIBLE NONE;'Primary Table:Additional Field' <None> VISIBLE NONE;", "'*Primary Table:Altname JoinID' AddressPoints:UTAddPtID VISIBLE NONE;'Primary Table:City Altname JoinID' <None> VISIBLE NONE;", "'*Alternate Name Table:JoinID' AtlNamesAddrPnts:UTAddPtID VISIBLE NONE;'Alternate Name Table:Full Street Name' <None> VISIBLE NONE;", "'Alternate Name Table:Prefix Direction' AtlNamesAddrPnts:PrefixDir VISIBLE NONE;'Alternate Name Table:Prefix Type' <None> VISIBLE NONE;", "'Alternate Name Table:Street Name' AtlNamesAddrPnts:STREETNAME VISIBLE NONE;", "'Alternate Name Table:Suffix Type' AtlNamesAddrPnts:STREETTYPE VISIBLE NONE;", "'Alternate Name Table:Suffix Direction' AtlNamesAddrPnts:SUFFIXDIR VISIBLE NONE" ] start_seconds = clock() process_seconds = clock() self.log.info('creating the %s locator', 'address point') try: output_location = join(self.output_location, 'AddressPoints_AddressSystem') arcpy.geocoding.CreateAddressLocator( in_address_locator_style='US Address - Single House', in_reference_data='{0}/{1};{0}/{2}'.format(self.locators, "AtlNamesAddrPnts 'Alternate Name Table'", "AddressPoints 'Primary Table'"), in_field_map=''.join(fields), out_address_locator=output_location, config_keyword='', enable_suggestions='DISABLED') self.update_locator_properties(output_location, template.us_single_house_addresses) except Exception as e: self.log.error(e) self.log.info('finished %s', format_time(clock() - process_seconds)) process_seconds = clock() #: streets fields = [ "'Primary Table:Feature ID' GeocodeRoads:OBJECTID VISIBLE NONE;'*Primary Table:From Left' GeocodeRoads:FROMADDR_L VISIBLE NONE;", "'*Primary Table:To Left' GeocodeRoads:TOADDR_L VISIBLE NONE;'*Primary Table:From Right' GeocodeRoads:FROMADDR_R VISIBLE NONE;", "'*Primary Table:To Right' GeocodeRoads:TOADDR_R VISIBLE NONE;'Primary Table:Left Parity' <None> VISIBLE NONE;", "'Primary Table:Right Parity' <None> VISIBLE NONE;'Primary Table:Full Street Name' <None> VISIBLE NONE;", "'Primary Table:Prefix Direction' GeocodeRoads:PREDIR VISIBLE NONE;'Primary Table:Prefix Type' <None> VISIBLE NONE;", "'*Primary Table:Street Name' GeocodeRoads:NAME VISIBLE NONE;'Primary Table:Suffix Type' GeocodeRoads:POSTTYPE VISIBLE NONE;", "'Primary Table:Suffix Direction' GeocodeRoads:POSTDIR VISIBLE NONE;", "'Primary Table:Left City or Place' GeocodeRoads:ADDRSYS_L VISIBLE NONE;", "'Primary Table:Right City or Place' GeocodeRoads:ADDRSYS_R VISIBLE NONE;'Primary Table:Left County' <None> VISIBLE NONE;", "'Primary Table:Right County' <None> VISIBLE NONE;'Primary Table:Left State' <None> VISIBLE NONE;'Primary Table:Right State' <None> VISIBLE NONE;", "'Primary Table:Left State Abbreviation' <None> VISIBLE NONE;'Primary Table:Right State Abbreviation' <None> VISIBLE NONE;", "'Primary Table:Left ZIP Code' <None> VISIBLE NONE;'Primary Table:Right ZIP Code' <None> VISIBLE NONE;'Primary Table:Country Code' <None> VISIBLE NONE;", "'Primary Table:3-Digit Language Code' <None> VISIBLE NONE;'Primary Table:2-Digit Language Code' <None> VISIBLE NONE;", "'Primary Table:Admin Language Code' <None> VISIBLE NONE;'Primary Table:Left Block ID' <None> VISIBLE NONE;", "'Primary Table:Right Block ID' <None> VISIBLE NONE;'Primary Table:Left Street ID' <None> VISIBLE NONE;", "'Primary Table:Right Street ID' <None> VISIBLE NONE;'Primary Table:Street Rank' <None> VISIBLE NONE;", "'Primary Table:Min X value for extent' <None> VISIBLE NONE;'Primary Table:Max X value for extent' <None> VISIBLE NONE;", "'Primary Table:Min Y value for extent' <None> VISIBLE NONE;'Primary Table:Max Y value for extent' <None> VISIBLE NONE;", "'Primary Table:Left Additional Field' <None> VISIBLE NONE;'Primary Table:Right Additional Field' <None> VISIBLE NONE;", "'*Primary Table:Altname JoinID' GeocodeRoads:GLOBALID_SGID VISIBLE NONE;'Primary Table:City Altname JoinID' <None> VISIBLE NONE;", "'*Alternate Name Table:JoinID' AtlNamesRoads:GLOBALID_SGID VISIBLE NONE;'Alternate Name Table:Full Street Name' <None> VISIBLE NONE;", "'Alternate Name Table:Prefix Direction' AtlNamesRoads:PREDIR VISIBLE NONE;'Alternate Name Table:Prefix Type' <None> VISIBLE NONE;", "'Alternate Name Table:Street Name' AtlNamesRoads:NAME VISIBLE NONE;'Alternate Name Table:Suffix Type' AtlNamesRoads:POSTTYPE VISIBLE NONE;", "'Alternate Name Table:Suffix Direction' AtlNamesRoads:POSTDIR VISIBLE NONE" ] self.log.info('creating the %s locator', 'streets') try: output_location = join(self.output_location, 'Roads_AddressSystem_STREET') arcpy.geocoding.CreateAddressLocator( in_address_locator_style='US Address - Dual Ranges', in_reference_data='{0}/{1};{0}/{2}'.format(self.locators, "GeocodeRoads 'Primary Table'", "AtlNamesRoads 'Alternate Name Table'"), in_field_map=''.join(fields), out_address_locator=output_location, config_keyword='', enable_suggestions='DISABLED') self.update_locator_properties(output_location, template.us_dual_range_addresses) except Exception as e: self.log.error(e) self.log.info('finished %s', format_time(clock() - process_seconds)) process_seconds = clock() self.log.info('finished %s', format_time(clock() - process_seconds)) self.log.info('done %s', format_time(clock() - start_seconds))
def ship(self): database_infos = [ #: (name, sde connection, [user connections]) #: Name corresponds with the subfolder with the garage folder where the #: connection files are. ('FiberVerification', 'DC_sde@[email protected]', ['DC_FiberAdmin@[email protected]']), ('FFSL', 'DNR_sde@[email protected]', ['DNR_Fire@[email protected]', 'DNR_Forestry@[email protected]', 'DNR_Lands@[email protected]']), ('PLSSFabric', 'sde.sde', ['admin.sde']), ('SGID10', 'SGID_sde@[email protected]', ['SGID_Biosciense@[email protected]', 'SGID_Boundaries@[email protected]', 'SGID_Cadastre@[email protected]', 'SGID_Climate@[email protected]', 'SGID_Demographics@[email protected]', 'SGID_Economy@[email protected]', 'SGID_Elevation@[email protected]', 'SGID_Energy@[email protected]', 'SGID_Environment@[email protected]', 'SGID_Farming@[email protected]', 'SGID_Geoscience@[email protected]', 'SGID_Health@[email protected]', 'SGID_History@[email protected]', 'SGID_Indices@[email protected]', 'SGID_Location@[email protected]', 'SGID_Planning@[email protected]', 'SGID_Political@[email protected]', 'SGID_Raster@[email protected]', 'SGID_Recreation@[email protected]', 'SGID_Society@[email protected]', 'SGID_Transportation@[email protected]', 'SGID_Utilities@[email protected]', 'SGID_Water@[email protected]']), ('UDNR', 'DNR_sde@[email protected]', ['DNR_DPR@[email protected]', 'DNR_DWR@[email protected]', 'DNR_FFSL@[email protected]', 'DNR_OGM@[email protected]', 'DNR_UGS@[email protected]', 'DNR_WRE@[email protected]', 'DNR_WRT@[email protected]']), ('UDPR', 'DNR_sde@[email protected]', ['DNR_OHV@[email protected]', 'DNR_susanz@[email protected]']), ('UDWR', 'DNR_sde@[email protected]', ['DNR_DWRADMIN@[email protected]']), ('UEMP', 'sde@[email protected]', ['uempadmin@[email protected]']), ('UGGP', 'sde@[email protected]', ['uggpadmin@[email protected]']), ('UGHP', 'sde@[email protected]', ['ughpadmin@[email protected]']), ('UGIO', 'DNR_sde@[email protected]', ['DNR_ugioadmin@[email protected]']), ('UGMP', 'DC_sde@[email protected]', ['DNR_UGMPADMIN@[email protected]']), ('UTAX', 'DC_sde@[email protected]', ['DC_ustccass@[email protected]']), ('UTRANS', 'DC_sde@[email protected]', ['DC_TRANSADMIN@[email protected]']) ] with Pool(MAX_PROCESSES) as pool: results = list(pool.map(run, [(self.garage,) + info for info in database_infos])) #: log times for result in results: self.log.info('Compress/Analyze time for {}: {}'.format(result.name, format_time(result.elapsed_time))) if False in [result.success for result in results]: message = '\n\n'.join([result.error_message for result in results if result.error_message is not None]) self.log.error('Errors during multiprocessing: ' + message) self.success = (False, message)
def test_format_time_hours(self): self.assertEqual(seat.format_time(7200.0), '2.0 hours') self.assertEqual(seat.format_time(5410.0), '1.5 hours')
def test_format_time_seconds(self): self.assertEqual(seat.format_time(80.0), '80.0 seconds') self.assertEqual(seat.format_time(50.0), '50.0 seconds')
def test_format_time_minutes(self): self.assertEqual(seat.format_time(91.0), '1.52 minutes') self.assertEqual(seat.format_time(1800.0), '30.0 minutes')
def test_format_time_milliseconds(self): self.assertEqual(seat.format_time(5.5), '5500 ms')
def test_format_time_hours(self): self.assertEqual(seat.format_time(7200), '2.0 hours') self.assertEqual(seat.format_time(5410), '1.5 hours')
def test_format_time_minutes(self): self.assertEqual(seat.format_time(91), '1.52 minutes') self.assertEqual(seat.format_time(1800), '30.0 minutes')
def test_format_time_seconds(self): self.assertEqual(seat.format_time(80), '80.0 seconds') self.assertEqual(seat.format_time(50), '50.0 seconds')