def handle(self, *args, **options): verbosity = int(options.get('verbosity')) ignore_errors = options.get('ignore_errors') user = options.get('user') overwrite = options.get('overwrite') if verbosity > 0: console = sys.stdout else: console = None if overwrite == True: skip = False else: skip = True keywords = options.get('keywords').split(',') if len(keywords) == 1 and keywords[0] == '': keywords = [] start = datetime.datetime.now() output = [] for path in args: out = upload(path, user=user, overwrite=overwrite, skip=skip, keywords=keywords, verbosity=verbosity, console=console) output.extend(out) updated = [dict_['file'] for dict_ in output if dict_['status']=='updated'] created = [dict_['file'] for dict_ in output if dict_['status']=='created'] skipped = [dict_['file'] for dict_ in output if dict_['status']=='skipped'] failed = [dict_['file'] for dict_ in output if dict_['status']=='failed'] finish = datetime.datetime.now() td = finish - start duration = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600 duration_rounded = round(duration, 2) if verbosity > 1: print "\nDetailed report of failures:" for dict_ in output: if dict_['status'] == 'failed': print "\n\n", dict_['file'], "\n================" traceback.print_exception(dict_['exception_type'], dict_['error'], dict_['traceback']) if verbosity > 0: print "\n\nFinished processing %d layers in %s seconds.\n" % ( len(output), duration_rounded) print "%d Created layers" % len(created) print "%d Updated layers" % len(updated) print "%d Skipped layers" % len(skipped) print "%d Failed layers" % len(failed) if len(output) > 0: print "%f seconds per layer" % (duration * 1.0 / len(output))
def test_layer_upload(self): """Test that layers can be uploaded to running GeoNode/GeoServer """ layers = {} expected_layers = [] not_expected_layers = [] for filename in os.listdir(gisdata.GOOD_DATA): basename, extension = os.path.splitext(filename) if extension.lower() in ['.tif', '.shp', '.zip']: expected_layers.append( os.path.join( gisdata.GOOD_DATA, filename)) for filename in os.listdir(gisdata.BAD_DATA): not_expected_layers.append( os.path.join(gisdata.BAD_DATA, filename) ) uploaded = upload(gisdata.DATA_DIR, console=None) for item in uploaded: errors = 'error' in item if errors: # should this file have been uploaded? if item['file'] in not_expected_layers: continue else: msg = ('Could not upload file "%s", ' 'and it is not in %s' % ( item['file'], not_expected_layers)) assert errors, msg else: msg = ('Upload should have returned either "name" or ' '"errors" for file %s.' % item['file']) assert 'name' in item, msg layers[item['file']] = item['name'] msg = ('There were %s compatible layers in the directory,' ' but only %s were sucessfully uploaded' % (len(expected_layers), len(layers))) # assert len(layers) == len(expected_layers), msg for layer in expected_layers: msg = ('The following file should have been uploaded' 'but was not: %s. ' % layer) assert layer in layers, msg layer_name = layers[layer] # Check the layer is in the Django database Layer.objects.get(name=layer_name) # Check that layer is in geoserver found = False gs_username, gs_password = settings.OGC_SERVER['default'][ 'USER'], settings.OGC_SERVER['default']['PASSWORD'] page = get_web_page( os.path.join( settings.OGC_SERVER['default']['LOCATION'], 'rest/layers'), username=gs_username, password=gs_password) if page.find('rest/layers/%s.html' % layer_name) > 0: found = True if not found: msg = ( 'Upload could not be verified, the layer %s is not ' 'in geoserver %s, but GeoNode did not raise any errors, ' 'this should never happen.' % (layer_name, settings.OGC_SERVER['default']['LOCATION'])) raise GeoNodeException(msg) # server_url = settings.OGC_SERVER['default']['LOCATION'] + 'ows?' # Verify that the GeoServer GetCapabilities record is accessible: # metadata = get_layers_metadata(server_url, '1.0.0') # msg = ('The metadata list should not be empty in server %s' # % server_url) # assert len(metadata) > 0, msg # Check the keywords are recognized too # Clean up and completely delete the layers for layer in expected_layers: layer_name = layers[layer] Layer.objects.get(name=layer_name).delete()
def handle(self, *args, **options): verbosity = int(options.get('verbosity')) # ignore_errors = options.get('ignore_errors') username = options.get('user') user = get_valid_user(username) overwrite = options.get('overwrite') category = options.get('category', None) private = options.get('private', False) title = options.get('title', None) date = options.get('date', None) metadata_uploaded_preserve = options.get('metadata_uploaded_preserve', False) if verbosity > 0: console = self.stdout else: console = None if overwrite: skip = False else: skip = True keywords = options.get('keywords').split(',') if len(keywords) == 1 and keywords[0] == '': keywords = [] else: keywords = map(str.strip, keywords) regions = options.get('regions').split(',') if len(regions) == 1 and regions[0] == '': regions = [] else: regions = map(str.strip, regions) start = datetime.datetime.now() output = [] for path in args: out = upload( path, user=user, overwrite=overwrite, skip=skip, keywords=keywords, verbosity=verbosity, console=console, category=category, regions=regions, title=title, date=date, private=private, metadata_uploaded_preserve=metadata_uploaded_preserve) output.extend(out) updated = [dict_['file'] for dict_ in output if dict_['status'] == 'updated'] created = [dict_['file'] for dict_ in output if dict_['status'] == 'created'] skipped = [dict_['file'] for dict_ in output if dict_['status'] == 'skipped'] failed = [dict_['file'] for dict_ in output if dict_['status'] == 'failed'] finish = datetime.datetime.now() td = finish - start duration = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600 duration_rounded = round(duration, 2) if verbosity > 1: print "\nDetailed report of failures:" for dict_ in output: if dict_['status'] == 'failed': print "\n\n", dict_['file'], "\n================" traceback.print_exception(dict_['exception_type'], dict_['error'], dict_['traceback']) if verbosity > 0: print "\n\nFinished processing %d layers in %s seconds.\n" % ( len(output), duration_rounded) print "%d Created layers" % len(created) print "%d Updated layers" % len(updated) print "%d Skipped layers" % len(skipped) print "%d Failed layers" % len(failed) if len(output) > 0: print "%f seconds per layer" % (duration * 1.0 / len(output))
def setUp(self): call_command('loaddata', 'people_data', verbosity=0) upload(gisdata.VECTOR_DATA, console=None)
def test_layer_upload(self): """Test that layers can be uploaded to running GeoNode/GeoServer """ layers = {} expected_layers = [] not_expected_layers = [] for filename in os.listdir(gisdata.GOOD_DATA): basename, extension = os.path.splitext(filename) if extension.lower() in [".tif", ".shp", ".zip"]: expected_layers.append(os.path.join(gisdata.GOOD_DATA, filename)) for filename in os.listdir(gisdata.BAD_DATA): not_expected_layers.append(os.path.join(gisdata.BAD_DATA, filename)) uploaded = upload(gisdata.DATA_DIR, console=None) for item in uploaded: errors = "error" in item if errors: # should this file have been uploaded? if item["file"] in not_expected_layers: continue else: msg = 'Could not upload file "%s", ' "and it is not in %s" % (item["file"], not_expected_layers) assert errors == True, msg else: msg = 'Upload should have returned either "name" or ' '"errors" for file %s.' % item["file"] assert "name" in item, msg layers[item["file"]] = item["name"] msg = "There were %s compatible layers in the directory," " but only %s were sucessfully uploaded" % ( len(expected_layers), len(layers), ) # assert len(layers) == len(expected_layers), msg uploaded_layers = [layer for layer in layers.items()] for layer in expected_layers: msg = "The following file should have been uploaded" "but was not: %s. " % layer assert layer in layers, msg layer_name = layers[layer] # Check the layer is in the Django database Layer.objects.get(name=layer_name) # Check that layer is in geoserver found = False gs_username, gs_password = settings.GEOSERVER_CREDENTIALS page = get_web_page( os.path.join(settings.GEOSERVER_BASE_URL, "rest/layers"), username=gs_username, password=gs_password ) if page.find("rest/layers/%s.html" % layer_name) > 0: found = True if not found: msg = ( "Upload could not be verified, the layer %s is not " "in geoserver %s, but GeoNode did not raise any errors, " "this should never happen." % (layer_name, settings.GEOSERVER_BASE_URL) ) raise GeoNodeException(msg) server_url = settings.GEOSERVER_BASE_URL + "ows?" # Verify that the GeoServer GetCapabilities record is accesible: # metadata = get_layers_metadata(server_url, '1.0.0') # msg = ('The metadata list should not be empty in server %s' # % server_url) # assert len(metadata) > 0, msg # Check the keywords are recognized too # Clean up and completely delete the layers for layer in expected_layers: layer_name = layers[layer] Layer.objects.get(name=layer_name).delete()
def handle(self, *args, **options): verbosity = int(options.get('verbosity')) ignore_errors = options.get('ignore_errors') user = options.get('user') overwrite = options.get('overwrite') if verbosity > 0: console = sys.stdout else: console = None if overwrite == True: skip = False else: skip = True keywords = options.get('keywords').split() start = datetime.datetime.now() output = [] for path in args: out = upload(path, user=user, overwrite=overwrite, skip=skip, keywords=keywords, verbosity=verbosity, console=console) output.extend(out) updated = [ dict_['file'] for dict_ in output if dict_['status'] == 'updated' ] created = [ dict_['file'] for dict_ in output if dict_['status'] == 'created' ] skipped = [ dict_['file'] for dict_ in output if dict_['status'] == 'skipped' ] failed = [ dict_['file'] for dict_ in output if dict_['status'] == 'failed' ] finish = datetime.datetime.now() td = finish - start duration = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600 duration_rounded = round(duration, 2) if verbosity > 1: print "\nDetailed report of failures:" for dict_ in output: if dict_['status'] == 'failed': print "\n\n", dict_['file'], "\n================" traceback.print_exception(dict_['exception_type'], dict_['error'], dict_['traceback']) if verbosity > 0: print "\n\nFinished processing %d layers in %s seconds.\n" % ( len(output), duration_rounded) print "%d Created layers" % len(created) print "%d Updated layers" % len(updated) print "%d Skipped layers" % len(skipped) print "%d Failed layers" % len(failed) if len(output) > 0: print "%f seconds per layer" % (duration * 1.0 / len(output))
def handle(self, *args, **options): verbosity = int(options.get('verbosity')) # ignore_errors = options.get('ignore_errors') username = options.get('user') user = get_valid_user(username) overwrite = options.get('overwrite') category = options.get('category', None) private = options.get('private', False) title = options.get('title', None) if verbosity > 0: console = self.stdout else: console = None if overwrite: skip = False else: skip = True keywords = options.get('keywords').split(',') if len(keywords) == 1 and keywords[0] == '': keywords = [] else: keywords = map(str.strip, keywords) regions = options.get('regions').split(',') if len(regions) == 1 and regions[0] == '': regions = [] else: regions = map(str.strip, regions) start = datetime.datetime.now() output = [] for path in args: out = upload(path, user=user, overwrite=overwrite, skip=skip, keywords=keywords, verbosity=verbosity, console=console, category=category, regions=regions, title=title, private=private) output.extend(out) updated = [ dict_['file'] for dict_ in output if dict_['status'] == 'updated' ] created = [ dict_['file'] for dict_ in output if dict_['status'] == 'created' ] skipped = [ dict_['file'] for dict_ in output if dict_['status'] == 'skipped' ] failed = [ dict_['file'] for dict_ in output if dict_['status'] == 'failed' ] finish = datetime.datetime.now() td = finish - start duration = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600 duration_rounded = round(duration, 2) if verbosity > 1: print "\nDetailed report of failures:" for dict_ in output: if dict_['status'] == 'failed': print "\n\n", dict_['file'], "\n================" traceback.print_exception(dict_['exception_type'], dict_['error'], dict_['traceback']) if verbosity > 0: print "\n\nFinished processing %d layers in %s seconds.\n" % ( len(output), duration_rounded) print "%d Created layers" % len(created) print "%d Updated layers" % len(updated) print "%d Skipped layers" % len(skipped) print "%d Failed layers" % len(failed) if len(output) > 0: print "%f seconds per layer" % (duration * 1.0 / len(output))
def handle(self, *args, **options): verbosity = int(options.get('verbosity')) # ignore_errors = options.get('ignore_errors') username = options.get('user') user = get_valid_user(username) overwrite = options.get('overwrite') name = options.get('layername', None) title = options.get('title', None) abstract = options.get('abstract', None) date = options.get('date', None) license = options.get('license', None) category = options.get('category', None) private = options.get('private', False) metadata_uploaded_preserve = options.get('metadata_uploaded_preserve', False) charset = options.get('charset', 'UTF-8') if verbosity > 0: console = self.stdout else: console = None if overwrite: skip = False else: skip = True keywords = options.get('keywords').split(',') if len(keywords) == 1 and keywords[0] == '': keywords = [] else: keywords = [k.strip() for k in keywords] regions = options.get('regions').split(',') if len(regions) == 1 and regions[0] == '': regions = [] else: regions = [r.strip() for r in regions] start = datetime.datetime.now(timezone.get_current_timezone()) output = [] for path in options['path']: out = upload(path, user=user, overwrite=overwrite, skip=skip, name=name, title=title, abstract=abstract, date=date, keywords=keywords, verbosity=verbosity, console=console, license=license, category=category, regions=regions, private=private, metadata_uploaded_preserve=metadata_uploaded_preserve, charset=charset) output.extend(out) updated = [ dict_['file'] for dict_ in output if dict_['status'] == 'updated' ] created = [ dict_['file'] for dict_ in output if dict_['status'] == 'created' ] skipped = [ dict_['file'] for dict_ in output if dict_['status'] == 'skipped' ] failed = [ dict_['file'] for dict_ in output if dict_['status'] == 'failed' ] finish = datetime.datetime.now(timezone.get_current_timezone()) td = finish - start duration = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600 duration_rounded = round(duration, 2) if verbosity > 1: print("\nDetailed report of failures:") for dict_ in output: if dict_['status'] == 'failed': print("\n\n", dict_['file'], "\n================") traceback.print_exception(dict_['exception_type'], dict_['error'], dict_['traceback']) if verbosity > 0: print("\n\nFinished processing {} layers in {} seconds.\n".format( len(output), duration_rounded)) print("{} Created layers".format(len(created))) print("{} Updated layers".format(len(updated))) print("{} Skipped layers".format(len(skipped))) print("{} Failed layers".format(len(failed))) if len(output) > 0: print("{} seconds per layer".format(duration * 1.0 / len(output)))