def setUp(self): logging.basicConfig( filename='test/unittest.log', format= "%(asctime)s - %(levelname)s - %(name)s:%(funcName)s - %(message)s", level=logging.DEBUG, filemode='w') self.config = Config('test/configuration_examples/configuration.json') self.ns_servers = ['localhost', 'root.localhost'] self.ignore_rules_list = sorted( ['lb-node.*', 'cmts.*', 'rfgw.*', 'test', 'xxx']) self.ignore_rules = { "test": [], "lb-node.*": [], "rfgw.*": [], "cmts.*": ["Et0/0/0"], "xxx": ["eth0", "eth1", "eth2.*"] } self.ip_ignore_rules = [ "127.0.0.0/8", "256.0.0.0", "109.0.0.0/33", "192.0.2.22", "1.1.1.1" ] self.default_community = 'public' self.retries = 0 self.timeout = 1
def test_get_ip_ignored(self): self.assertListEqual(sorted(self.ip_ignore_rules), sorted(self.config.data['ignored']['ip'])) self.assertListEqual(sorted(self.ip_ignore_rules), sorted(self.config.get_ip_ignore_rules())) config = Config('test/configuration_examples/simple.json') self.assertEquals(config.get_ip_ignore_rules(), [])
def setUp(self): logging.basicConfig(level=Config.get('LOG_LEVEL'), format=Config.get('LOG_FORMAT'), datefmt=Config.get('LOG_DATE_FORMAT'), filename=Config.get('LOG_FILE'), filemode='w') self.logger_r = logging.getLogger('TestRasteriser')
def test_rasterise_from_shp(self): """ Read inland water data from shapefile """ self.logger_r.info( 'Rasteriser with Inland Water MasterMap data from shapefile...') output_file = 'test_water_output_raster.tif' output_path = '{}/{}'.format(Config.get('DATA_DIRECTORY'), output_file) if path.exists(output_path): remove(output_path) try: # Get MasterMap data, requesting classification codes 'General Surface', 'Natural Environment' gdf = GeoDataFrame.from_file( '{}/inland_water_e08000021.shp'.format( Config.get('DATA_DIRECTORY'))) # Call rasteriser self.logger_r.info('Calling rasteriser...') Rasteriser(gdf.to_json(), area_codes=['E08000021'], output_filename=output_file, area_threshold=50.0).create() self.logger_r.info('Written output to {}/{}'.format( Config.get('DATA_DIRECTORY'), output_file)) self.assertTrue(path.exists(output_path)) self.assertTrue(path.getsize(output_path) > 0) self.logger_r.info('Completed') except: self.logger_r.warning(traceback.format_exc()) self.fail('Failing test due to unexpected exception')
def test_is_interface_ignored(self): self.assertTrue( self.config.is_interface_ignored('lb-node1.vektor.net', 'eth0')) self.assertTrue( self.config.is_interface_ignored('cmts-sc-1.vektor.net', 'Et0/0/0')) self.assertFalse( self.config.is_interface_ignored('cmts-sc-1.vektor.net', 'Et1/0/0')) self.assertFalse( self.config.is_interface_ignored('cmts-sc-1.vektor.net', 'Et0/0/01')) self.assertFalse( self.config.is_interface_ignored('cmts-sc-1.vektor.net', '0Et0/0/0')) self.assertFalse( self.config.is_interface_ignored('cmts-sc-1.vektor.net', '')) self.assertTrue(self.config.is_interface_ignored('xxx', 'eth0')) self.assertTrue(self.config.is_interface_ignored('xxx', 'eth0')) self.assertTrue(self.config.is_interface_ignored('xxx', 'eth2')) self.assertTrue(self.config.is_interface_ignored('xxx', 'eth21')) self.assertFalse(self.config.is_interface_ignored('xxx', 'eth01')) self.assertFalse(self.config.is_interface_ignored('xxx', '1eth0')) self.assertTrue(self.config.is_interface_ignored('test', 'ethasdasd')) config = Config('test/configuration_examples/invalid_values.json') self.assertFalse(config.is_interface_ignored('localhost', 'interface'))
def setUp(self): logging.basicConfig( level=Config.get("LOG_LEVEL"), format=Config.get("LOG_FORMAT"), datefmt=Config.get("LOG_DATE_FORMAT"), filename=Config.get("LOG_FILE"), filemode="w", ) self.logger = logging.getLogger("TestPointSet")
def test_is_device_ignored(self): self.assertTrue(self.config.is_device_ignored('lb-node1.vektor.net')) self.assertFalse(self.config.is_device_ignored('cmts-sc-1.vektor.net')) self.assertTrue(self.config.is_device_ignored('test')) self.assertFalse(self.config.is_device_ignored('test1')) self.assertFalse(self.config.is_device_ignored('1test')) self.assertFalse(self.config.is_device_ignored('')) config = Config('test/configuration_examples/invalid_values.json') self.assertFalse(config.is_device_ignored('localhost'))
def setUp(self): logging.basicConfig( filename='test/unittest.log', format="%(asctime)s - %(levelname)s - %(name)s:%(funcName)s - %(message)s", level=logging.DEBUG, filemode='w' ) self.dispatcher = Dispatcher(Config(filename='test/configuration_examples/configuration.json')) self.connector = DnsConnector(self.dispatcher) self.dns = DnsCheck(config=Config('test/configuration_examples/configuration.json'))
def test_file_open(self): self.assertIsInstance( Config('test/configuration_examples/configuration.json'), Config) self.assertIsInstance( Config('test/configuration_examples/simple.json'), Config) self.assertIsInstance( Config('test/configuration_examples/simple.json'), Config) self.assertRaises(IOError, Config, 'config.json') self.assertRaises(SystemExit, Config, 'test/configuration_examples/bad_json_file.json')
def test_get_device_ignored(self): self.assertListEqual( sorted(self.config.data['ignored']['device'].keys()), self.ignore_rules_list) self.assertListEqual( sorted(self.config.get_device_ignore_rules().keys()), self.ignore_rules_list) self.assertEquals(self.config.get_device_ignore_rules(), self.ignore_rules) config = Config('test/configuration_examples/simple.json') self.assertEquals(config.get_device_ignore_rules(), {})
def test_get_connector_config(self): connector = TestConnector(self.dispatcher) self.assertDictEqual( { 'enabled': True }, connector.config) dispatcher = Dispatcher(Config(filename='test/configuration_examples/configuration.json'), auto_load=False) connector = TestConnector(dispatcher) self.assertDictEqual( { 'enabled': True, 'key': 'value' }, connector.config) connector2 = Test2Connector(dispatcher) self.assertDictEqual( { 'enabled': True }, connector2.config) connector3 = Test3Connector(dispatcher) dispatcher.get_connector_config(connector3)
def test_get_snmp(self): self.assertEquals(self.config.get_snmp_community('hostname'), self.default_community) self.assertEquals(self.config.get_snmp_community('1custom-host1'), self.default_community) self.assertEquals(self.config.get_snmp_community(), self.default_community) self.assertEquals(self.config.get_snmp_community('custom-host1'), 'custom_community') self.assertEquals(self.config.get_snmp_community('asd.domain.example'), 'custom_domain') config = Config('test/configuration_examples/simple.json') self.assertEquals(config.get_snmp_community('1custom-host1'), self.default_community) self.assertEquals(config.get_snmp_community('custom-host1'), self.default_community) self.assertEquals(config.get_snmp_community('asd.domain.example'), self.default_community)
def debug_dump_geojson_to_file(self, filename, json_data): """ Dump the given JSON data to a file for examination """ filepath = '{}/{}'.format(Config.get('DATA_DIRECTORY'), filename) if path.exists(filepath): remove(filepath) with open(filepath, 'w') as file_out: file_out.write(dumps(json_data))
def test_rasterise_from_nismod(self): """ Get MasterMap data from NISMOD API as input GeoJSON data """ self.logger_r.info('Rasteriser with API MasterMap data...') output_file = 'test_output_raster.tif' output_path = '{}/{}'.format(Config.get('DATA_DIRECTORY'), output_file) if path.exists(output_path): remove(output_path) try: # Get MasterMap data, requesting classification codes 'General Surface', 'Natural Environment' api_parms = { 'scale': 'lad', 'area_codes': ['E07000004', 'E07000008', 'E07000009', 'E07000011'], 'classification_codes': ['10056', '10111'], 'export_format': 'geojson' } api_url = '{}/mastermap/areas'.format( Config.get('NISMOD_DB_API_URL')) auth_username = Config.get('NISMOD_DB_USERNAME') auth_password = Config.get('NISMOD_DB_PASSWORD') self.logger_r.info('Calling API to extract input GeoJSON data...') r = requests.get(api_url, params=api_parms, auth=(auth_username, auth_password)) input_geojson = r.json() # Call rasteriser self.logger_r.info('Calling rasteriser...') Rasteriser(input_geojson, area_codes=[ 'E07000004', 'E07000008', 'E07000009', 'E07000011' ], output_filename=output_file).create() self.assertTrue(path.exists(output_path)) self.assertTrue(path.getsize(output_path) > 0) self.logger_r.info('Written output to {}/{}'.format( Config.get('DATA_DIRECTORY'), output_file)) self.logger_r.info('Completed') except: self.logger_r.warning(traceback.format_exc()) self.fail('Failing test due to unexpected exception')
def setUp(self): logging.basicConfig( filename='test/unittest.log', format="%(asctime)s - %(levelname)s - %(name)s:%(funcName)s - %(message)s", level=logging.DEBUG, filemode='w' ) self.dispatcher = Dispatcher( Config(filename='test/configuration_examples/simple.json'), auto_load=False )
def test_rasterise_from_shp_and_fishnet_file(self): """ Read inland water data from shapefile, using a fishnet generated previously """ self.logger_r.info( 'Rasteriser with Inland Water MasterMap data from shapefile, using pre-generated fishnet...' ) try: self.logger_r.info( 'Generate fishnet with GeoJSON string return...') fishnet_geojson = FishNet(outfile=None, outformat='GeoJSON', bbox=[414650, 563500, 429600, 575875]).create() self.assertFalse(fishnet_geojson is None) output_file = 'test_water_output_raster_ex_fishnet.tif' output_path = '{}/{}'.format(Config.get('DATA_DIRECTORY'), output_file) if path.exists(output_path): remove(output_path) # Get MasterMap data, requesting classification codes 'General Surface', 'Natural Environment' gdf = GeoDataFrame.from_file( '{}/inland_water_e08000021.shp'.format( Config.get('DATA_DIRECTORY'))) # Call rasteriser self.logger_r.info('Calling rasteriser...') Rasteriser(gdf.to_json(), fishnet=fishnet_geojson, output_filename=output_file, area_threshold=50.0).create() self.assertTrue(path.exists(output_path)) self.assertTrue(path.getsize(output_path) > 0) self.logger_r.info('Written output to {}/{}'.format( Config.get('DATA_DIRECTORY'), output_file)) self.logger_r.info('Completed') except: self.logger_r.warning(traceback.format_exc()) self.fail('Failing test due to unexpected exception')
def __init__(self): ServerFactory.__init__(self) self.pack = pack self.l = Obj() self.c = Config(config) self.dfa = self.do_for_all self.worlds = [World()] self.w = self.worlds lib = __import__('lib') for i in listdir(curdir+'lib'): tmp = __import__('lib.{}'.format(i),lib) exec('self.l.{} = tmp'.format(i),{'self':self,'tmp':tmp})
def test_fishnet_area_codes(self): """ Tests fishnet generation with a list of area codes """ self.logger.info('Fishnet with list of area codes...') output_file = '{}.json'.format(uuid.uuid4().hex) output_path = '{}/{}'.format(Config.get('DATA_DIRECTORY'), output_file) FishNet(outfile=output_file, outformat='GeoJSON', lad=['E07000004']).create() self.assertTrue(path.exists(output_path)) self.assertTrue(path.getsize(output_path) > 0) remove(output_path) self.logger.info('Completed')
def setUp(self): logging.basicConfig( filename='test/unittest.log', format= "%(asctime)s - %(levelname)s - %(name)s:%(funcName)s - %(message)s", level=logging.DEBUG, filemode='w') self.device = Device( 'localhost', Config(filename='test/configuration_examples/simple.json')) self.device.get_interfaces() for if_index, di in self.device.interfaces.iteritems(): self.di = di break
def test_fishnet_bbox(self): """ Tests fishnet generation with a bounding box """ self.logger.info('Fishnet with bounding box...') output_file = '{}.json'.format(uuid.uuid4().hex) output_path = '{}/{}'.format(Config.get('DATA_DIRECTORY'), output_file) FishNet(outfile=output_file, outformat='GeoJSON', bbox=[414650, 563500, 429600, 575875]).create() self.assertTrue(path.exists(output_path)) self.assertTrue(path.getsize(output_path) > 0) remove(output_path) self.logger.info('Completed')
def test_fishnet_shapefile(self): """ Tests fishnet generation with a shapefile output """ self.logger.info('Fishnet with ESRI shapefile output...') output_file = '{}.shp'.format(uuid.uuid4().hex) output_path = '{}/{}'.format(Config.get('DATA_DIRECTORY'), output_file) FishNet(outfile=output_file, outformat='ESRI Shapefile', lad=['E07000004']).create() self.assertTrue(path.exists(output_path)) self.assertTrue(path.getsize(output_path) > 0) remove(output_path) self.logger.info('Completed')
def test_pointset_extract(self): """ Generate a point set with points extracted from an existing dataset """ set_size = 500 out_file = "extracted_points.gpkg" out_path = self.absolute_file_output_path(out_file) out_gdf = ExtractGrid( set_size, ["E08000021"], "{}/newcastle_lamp_posts.gpkg".format( Config.get("DATA_DIRECTORY")), ).generate() self.assertTrue(len(out_gdf) == set_size) self.write_geopackage(out_gdf, out_file) self.assertTrue(path.exists(out_path) and path.getsize(out_path) > 0)
def setUp(self): logging.basicConfig( filename='test/unittest.log', format= "%(asctime)s - %(levelname)s - %(name)s:%(funcName)s - %(message)s", level=logging.DEBUG, filemode='w') self.dispatcher = Dispatcher( Config(filename='test/configuration_examples/configuration.json')) self.connector = SqliteConnector(self.dispatcher) self.connector.create_ptr_table() ptr = { 'ip_address': u'10.10.10.10', 'hostname': 'cmts-sc-1.domain.example', 'if_name': 'Ethernet0/0/0', 'ptr': 'cmts-sc-1-et0-0-0.domain.example' } self.connector.save_ptr(Ptr(**ptr))
def create(self): """ | Generate the output raster dataset """ gdal.UseExceptions() temp_shp = '{}/{}.shp'.format(Config.get('DATA_DIRECTORY'), uuid.uuid4().hex) try: # Read the supplied GeoJSON data into a DataFrame self.logger.info('Creating GeoDataFrame from input...') #self.logger.debug('GeoJSON follows:') #elf.logger.debug(self.geojson_data) #self.logger.debug('GeoJSON end') if isinstance(self.geojson_data, str): self.logger.info( 'Input GeoJSON is a string, not a dict => converting...') self.geojson_data = loads(self.geojson_data) #self.debug_dump_geojson_to_file('rasteriser_input_data_dump.json', self.geojson_data) input_data = GeoDataFrame.from_features(self.geojson_data) self.logger.debug(input_data.head(10)) self.logger.info('Done') # Create the fishnet if necessary if self.bounding_box is not None: # Use the supplied British National Grid bounding box self.logger.info( 'Generate fishnet GeoDataFrame from supplied bounding box...' ) fishnet_geojson = FishNet(bbox=self.bounding_box, netsize=self.resolution).create() elif self.fishnet is not None: # Use a supplied fishnet output self.logger.info( 'Generate fishnet GeoDataFrame from supplied GeoJSON...') if isinstance(self.fishnet, str): self.logger.info( 'Input fishnet GeoJSON is a string, not a dict => converting...' ) self.fishnet = loads(self.fishnet) fishnet_geojson = self.fishnet elif len(self.area_codes) > 0: # Use the LAD codes self.logger.info( 'Generate fishnet GeoDataFrame from supplied LAD codes...') fishnet_geojson = FishNet(lad=self.area_codes, netsize=self.resolution).create() else: raise ValueError( 'No boundary information supplied - please supply fishnet GeoJSON, bounding box, or list of LAD codes' ) #self.debug_dump_geojson_to_file('rasteriser_fishnet_data_dump.json', fishnet_geojson) fishnet = GeoDataFrame.from_features(fishnet_geojson) x_min, y_min, x_max, y_max = fishnet.total_bounds self.logger.debug(fishnet.head(10)) self.logger.info('Done') # Overlay intersection self.logger.info('Overlay data on fishnet using intersection...') intersection = overlay(fishnet, input_data, how='intersection') self.logger.info('Done') # Write area attribute into frame self.logger.info('Computing areas...') intersection['area'] = intersection.geometry.area self.logger.info('Done') # Create grid to rasterize via merge and assign an 'include' field based on the threshold self.logger.info('Doing merge...') self.logger.debug(intersection.head(10)) int_merge = fishnet.merge( intersection.groupby(['FID']).area.sum() / 100.0, on='FID') for i, row in int_merge.iterrows(): self.logger.debug('{} has area {}'.format(i, row['area'])) if row['area'] > self.area_threshold: int_merge.at[i, 'include_me'] = int( 0) if self.invert else int(1) else: int_merge.at[i, 'include_me'] = int( 1) if self.invert else int(0) self.logger.info('Done') self.logger.info('Compute bounds of dataset...') #x_min, y_min, x_max, y_max = int_merge.total_bounds xdim = int((x_max - x_min) / self.resolution) ydim = int((y_max - y_min) / self.resolution) self.logger.info( 'xmin = {}, ymin = {}, xmax = {}, ymax = {}'.format( x_min, y_min, x_max, y_max)) # Save as temporary shapefile (TO DO - understand what information is gained by doing this that is not present in GeoJSON) self.logger.info('Write out temporary shapefile...') int_merge.to_file(temp_shp) self.logger.info('Written to {}'.format(temp_shp)) # Open OGR dataset ogr_source = ogr.Open(temp_shp) output_file = '{}/{}'.format(Config.get('DATA_DIRECTORY'), self.output_filename) self.logger.info( 'Will write output raster to {}'.format(output_file)) # Create raster dataset and set projection driver = gdal.GetDriverByName('GTiff') rasterised = driver.Create(output_file, xdim, ydim, 1, gdal.GDT_Byte) rasterised.SetGeoTransform( (x_min, self.resolution, 0, y_max, 0, -self.resolution)) srs = osr.SpatialReference() srs.ImportFromEPSG(27700) rasterised.SetProjection(srs.ExportToWkt()) # Set nodata values band = rasterised.GetRasterBand(1) #band.SetNoDataValue(self.nodata) band.Fill(self.nodata) # Do rasterisation self.logger.info( 'Set transform and projection, about to rasterise layer...') gdal.RasterizeLayer(rasterised, [1], ogr_source.GetLayer(0), options=["ATTRIBUTE=include_me"]) self.logger.info('Done') rasterised.FlushCache() rasterised = None ogr_source = None except: self.logger.warning(traceback.format_exc()) finally: self.logger.info('Removing temporary files...') filestem = Path(temp_shp).stem for shpf in Path(Config.get('DATA_DIRECTORY')).glob( '{}.*'.format(filestem)): self.logger.info('Cleaning up {}'.format(shpf)) shpf.unlink()
def test_autoload(self): self.dispatcher = Dispatcher( Config(filename='test/configuration_examples/simple.json'), connectors_dir='/../test/connectors/modules' )
class TestConfig(unittest.TestCase): def setUp(self): logging.basicConfig( filename='test/unittest.log', format= "%(asctime)s - %(levelname)s - %(name)s:%(funcName)s - %(message)s", level=logging.DEBUG, filemode='w') self.config = Config('test/configuration_examples/configuration.json') self.ns_servers = ['localhost', 'root.localhost'] self.ignore_rules_list = sorted( ['lb-node.*', 'cmts.*', 'rfgw.*', 'test', 'xxx']) self.ignore_rules = { "test": [], "lb-node.*": [], "rfgw.*": [], "cmts.*": ["Et0/0/0"], "xxx": ["eth0", "eth1", "eth2.*"] } self.ip_ignore_rules = [ "127.0.0.0/8", "256.0.0.0", "109.0.0.0/33", "192.0.2.22", "1.1.1.1" ] self.default_community = 'public' self.retries = 0 self.timeout = 1 def test_file_open(self): self.assertIsInstance( Config('test/configuration_examples/configuration.json'), Config) self.assertIsInstance( Config('test/configuration_examples/simple.json'), Config) self.assertIsInstance( Config('test/configuration_examples/simple.json'), Config) self.assertRaises(IOError, Config, 'config.json') self.assertRaises(SystemExit, Config, 'test/configuration_examples/bad_json_file.json') def test_empty_config_file(self): # Test empty self.assertRaises(SystemExit, Config, 'test/configuration_examples/empty.json') def test_get_ns_list(self): self.assertListEqual(self.config.data['dns']['servers'], self.ns_servers) self.assertListEqual(self.config.get_ns_servers(), self.ns_servers) def test_get_device_ignored(self): self.assertListEqual( sorted(self.config.data['ignored']['device'].keys()), self.ignore_rules_list) self.assertListEqual( sorted(self.config.get_device_ignore_rules().keys()), self.ignore_rules_list) self.assertEquals(self.config.get_device_ignore_rules(), self.ignore_rules) config = Config('test/configuration_examples/simple.json') self.assertEquals(config.get_device_ignore_rules(), {}) def test_get_ip_ignored(self): self.assertListEqual(sorted(self.ip_ignore_rules), sorted(self.config.data['ignored']['ip'])) self.assertListEqual(sorted(self.ip_ignore_rules), sorted(self.config.get_ip_ignore_rules())) config = Config('test/configuration_examples/simple.json') self.assertEquals(config.get_ip_ignore_rules(), []) def test_is_device_ignored(self): self.assertTrue(self.config.is_device_ignored('lb-node1.vektor.net')) self.assertFalse(self.config.is_device_ignored('cmts-sc-1.vektor.net')) self.assertTrue(self.config.is_device_ignored('test')) self.assertFalse(self.config.is_device_ignored('test1')) self.assertFalse(self.config.is_device_ignored('1test')) self.assertFalse(self.config.is_device_ignored('')) config = Config('test/configuration_examples/invalid_values.json') self.assertFalse(config.is_device_ignored('localhost')) def test_is_interface_ignored(self): self.assertTrue( self.config.is_interface_ignored('lb-node1.vektor.net', 'eth0')) self.assertTrue( self.config.is_interface_ignored('cmts-sc-1.vektor.net', 'Et0/0/0')) self.assertFalse( self.config.is_interface_ignored('cmts-sc-1.vektor.net', 'Et1/0/0')) self.assertFalse( self.config.is_interface_ignored('cmts-sc-1.vektor.net', 'Et0/0/01')) self.assertFalse( self.config.is_interface_ignored('cmts-sc-1.vektor.net', '0Et0/0/0')) self.assertFalse( self.config.is_interface_ignored('cmts-sc-1.vektor.net', '')) self.assertTrue(self.config.is_interface_ignored('xxx', 'eth0')) self.assertTrue(self.config.is_interface_ignored('xxx', 'eth0')) self.assertTrue(self.config.is_interface_ignored('xxx', 'eth2')) self.assertTrue(self.config.is_interface_ignored('xxx', 'eth21')) self.assertFalse(self.config.is_interface_ignored('xxx', 'eth01')) self.assertFalse(self.config.is_interface_ignored('xxx', '1eth0')) self.assertTrue(self.config.is_interface_ignored('test', 'ethasdasd')) config = Config('test/configuration_examples/invalid_values.json') self.assertFalse(config.is_interface_ignored('localhost', 'interface')) def test_is_ip_ignored(self): self.assertTrue(self.config.is_ip_ignored('127.0.0.0')) self.assertTrue(self.config.is_ip_ignored('127.0.0.1')) self.assertTrue(self.config.is_ip_ignored('127.255.255.255')) self.assertFalse(self.config.is_ip_ignored('126.255.255.255')) self.assertFalse(self.config.is_ip_ignored('109.122.96.23')) self.assertTrue(self.config.is_ip_ignored('1.1.1.1')) self.assertFalse(self.config.is_ip_ignored('1.1.1.2')) self.assertFalse(self.config.is_ip_ignored('1.1.1.0')) def test_get_snmp(self): self.assertEquals(self.config.get_snmp_community('hostname'), self.default_community) self.assertEquals(self.config.get_snmp_community('1custom-host1'), self.default_community) self.assertEquals(self.config.get_snmp_community(), self.default_community) self.assertEquals(self.config.get_snmp_community('custom-host1'), 'custom_community') self.assertEquals(self.config.get_snmp_community('asd.domain.example'), 'custom_domain') config = Config('test/configuration_examples/simple.json') self.assertEquals(config.get_snmp_community('1custom-host1'), self.default_community) self.assertEquals(config.get_snmp_community('custom-host1'), self.default_community) self.assertEquals(config.get_snmp_community('asd.domain.example'), self.default_community) def test_get_snmp_retries(self): self.assertEquals(self.config.get_snmp_retries(), 143) config = Config('test/configuration_examples/simple.json') self.assertEquals(config.get_snmp_retries(), 0) self.assertEquals(config.get_snmp_retries(443), 443) self.assertEquals(config.get_snmp_retries(1.0), 1) self.assertEquals(config.get_snmp_retries(1.9), 1) self.assertEquals(config.get_snmp_retries(2.4), 2) config = Config('test/configuration_examples/invalid_values.json') self.assertEquals(config.get_snmp_retries(123), 123) config = Config('test/configuration_examples/invalid_values_2.json') self.assertEquals(config.get_snmp_retries(123), 123) def test_get_snmp_timeoeut(self): self.assertEquals(self.config.get_snmp_timeout(), 23) config = Config('test/configuration_examples/simple.json') self.assertEquals(config.get_snmp_timeout(), 1) self.assertEquals(config.get_snmp_timeout(443), 443) self.assertEquals(config.get_snmp_timeout(0.1), 0.1) self.assertEquals(config.get_snmp_timeout(1.9), 1.9) config = Config('test/configuration_examples/invalid_values.json') self.assertEquals(config.get_snmp_timeout(123.4), 123.4) config = Config('test/configuration_examples/invalid_values_2.json') self.assertEquals(config.get_snmp_timeout(123.4), 123.4) def test_get_email_server(self): self.assertEqual(self.config.get_email_server(), 'smtp.domain.example') def test_get_email_to(self): self.assertListEqual(self.config.get_email_to(), ["*****@*****.**"]) def test_get_email_from(self): self.assertEqual(self.config.get_email_from(), '*****@*****.**')
def test_get_snmp_timeoeut(self): self.assertEquals(self.config.get_snmp_timeout(), 23) config = Config('test/configuration_examples/simple.json') self.assertEquals(config.get_snmp_timeout(), 1) self.assertEquals(config.get_snmp_timeout(443), 443) self.assertEquals(config.get_snmp_timeout(0.1), 0.1) self.assertEquals(config.get_snmp_timeout(1.9), 1.9) config = Config('test/configuration_examples/invalid_values.json') self.assertEquals(config.get_snmp_timeout(123.4), 123.4) config = Config('test/configuration_examples/invalid_values_2.json') self.assertEquals(config.get_snmp_timeout(123.4), 123.4)
action='store_true', help='Prints the version') parser.add_argument('-q', '--quiet', action='store_true', help='Limits the output to minimal responses') parser.add_argument('-da', '--download_archive', action='store', help='Download only links not listed in the file, ' 'and appending to the file the links downloaded') parser.add_argument('-thi', '--threading_instances', action='store', help='Number of threading instances to ') args = parser.parse_args() """ Main """ if args.version: print(f'Download manager version: {version}') else: conf = Config(dl_link=args.link, dl_file=args.file, output=args.output, audio_only=args.audio_only, archive=args.download_archive, threading_instances=args.threading_instances, quiet=args.quiet) manager = Manager(config=conf) manager.start_downloads()
def create(self): """ | Generate the fishnet dataset, based on the code at | https://pcjericks.github.io/py-gdalogr-cookbook/vector_layers.html#create-fishnet-grid """ gdal.UseExceptions() try: # Get bounding values aoi = self.bbox if self.lad: # Get the bounds from a (list of) Local Authority District boundary(s)/all self.logger.info('Get boundary from list of LAD codes...') try: kvp = { 'lad_codes': ','.join(self.lad), 'export_format': 'geojson', 'year': 2016 } api = '{}/{}/{}'.format(Config.get('NISMOD_DB_API_URL'), 'boundaries', 'lads') auth_username = Config.get('NISMOD_DB_USERNAME') auth_password = Config.get('NISMOD_DB_PASSWORD') r = requests.get(api, params=kvp, auth=(auth_username, auth_password)) # Note: should be able to simply read r.json() into a GeoDataFrame, however it throws a ValueError # 'Mixing dicts with non-Series may lead to ambiguous ordering' which makes very little sense to me! # So we do it a roundabout way via the recipe at # https://gis.stackexchange.com/questions/225586/reading-raw-data-into-geopandas self.logger.info('NISMOD API call completed') gdf = geopandas.read_file(BytesIO(r.content)) aoi = gdf.total_bounds except ValueError: raise xmin, ymin, xmax, ymax = [float(value) for value in aoi] self.logger.info( 'Fishnet bounds : xmin {}, ymin {}, xmax {}, ymax {}'.format( xmin, ymin, xmax, ymax)) grid_width = grid_height = float(self.netsize) # Number of rows x columns rows = ceil((ymax - ymin) / grid_height) cols = ceil((xmax - xmin) / grid_width) self.logger.info('Fishnet has {} rows and {} columns'.format( rows, cols)) # Start grid cell envelope ring_x_left_origin = xmin ring_x_right_origin = xmin + grid_width ring_y_top_origin = ymax ring_y_bottom_origin = ymax - grid_height out_driver = ogr.GetDriverByName(self.outformat) output_file = self.outfile if output_file is None: # Stream the data to memory output_file = '/vsimem/{}.geojson'.format(uuid.uuid4().hex) else: # Create output file if not path.isabs(output_file): # Relative path => so prepend data directory (does NOT handle making subdirectories here) data_dir = Config.get('DATA_DIRECTORY') self.logger.info( 'Relative path supplied, assume relative to data directory {}' .format(data_dir)) output_file = path.join(data_dir, output_file) else: # Absolute path => ensure all directories are present before writing try: makedirs(path.dirname(output_file), exist_ok=True) except OSError: self.logger.warning( 'Failed to create subdirectory for output file') raise # Delete any pre-existing version of output file if path.exists(output_file): remove(output_file) out_data_source = out_driver.CreateDataSource(output_file) srs = osr.SpatialReference() srs.ImportFromEPSG(27700) out_layer = out_data_source.CreateLayer(output_file, srs=srs, geom_type=ogr.wkbPolygon) # Add a FID field id_field = ogr.FieldDefn('FID', ogr.OFTInteger) out_layer.CreateField(id_field) feature_defn = out_layer.GetLayerDefn() # Create grid cells fid = 1 countcols = 0 while countcols < cols: countcols += 1 #self.logger.info('Generating column {}...'.format(countcols)) # Reset envelope for rows ring_y_top = ring_y_top_origin ring_y_bottom = ring_y_bottom_origin countrows = 0 while countrows < rows: countrows += 1 #self.logger.info('Row {}'.format(countrows)) ring = ogr.Geometry(ogr.wkbLinearRing) ring.AddPoint(ring_x_left_origin, ring_y_top) ring.AddPoint(ring_x_right_origin, ring_y_top) ring.AddPoint(ring_x_right_origin, ring_y_bottom) ring.AddPoint(ring_x_left_origin, ring_y_bottom) ring.AddPoint(ring_x_left_origin, ring_y_top) poly = ogr.Geometry(ogr.wkbPolygon) poly.AddGeometry(ring) # Add new geom to layer out_feature = ogr.Feature(feature_defn) out_feature.SetGeometry(poly) out_feature.SetField('FID', fid) out_layer.CreateFeature(out_feature) out_feature = None fid += 1 # New envelope for next poly ring_y_top = ring_y_top - grid_height ring_y_bottom = ring_y_bottom - grid_height # New envelope for next poly ring_x_left_origin = ring_x_left_origin + grid_width ring_x_right_origin = ring_x_right_origin + grid_width # Save and close data sources out_data_source = None fishnet_output = None if self.outfile is None: # Read the memory buffer GeoJSON into Python dict structure memfile_json = self.read_file(output_file).decode('utf-8') fishnet_output = loads(memfile_json) else: fishnet_output = output_file self.logger.info('Finished writing fishnet output') return fishnet_output except: self.logger.warning(traceback.format_exc()) return None
def test_get_snmp_retries(self): self.assertEquals(self.config.get_snmp_retries(), 143) config = Config('test/configuration_examples/simple.json') self.assertEquals(config.get_snmp_retries(), 0) self.assertEquals(config.get_snmp_retries(443), 443) self.assertEquals(config.get_snmp_retries(1.0), 1) self.assertEquals(config.get_snmp_retries(1.9), 1) self.assertEquals(config.get_snmp_retries(2.4), 2) config = Config('test/configuration_examples/invalid_values.json') self.assertEquals(config.get_snmp_retries(123), 123) config = Config('test/configuration_examples/invalid_values_2.json') self.assertEquals(config.get_snmp_retries(123), 123)