def handle(self, *args, **options):
verbosity = int(options.get('verbosity'))
ignore_errors = options.get('ignore_errors')
user = options.get('user')
overwrite = options.get('overwrite')
if verbosity > 0:
console = sys.stdout
else:
console = None
if overwrite == True:
skip = False
else:
skip = True
keywords = options.get('keywords').split(',')
if len(keywords) == 1 and keywords[0] == '': keywords = []
start = datetime.datetime.now()
output = []
for path in args:
out = upload(path, user=user, overwrite=overwrite, skip=skip,
keywords=keywords, verbosity=verbosity, console=console)
output.extend(out)
updated = [dict_['file'] for dict_ in output if dict_['status']=='updated']
created = [dict_['file'] for dict_ in output if dict_['status']=='created']
skipped = [dict_['file'] for dict_ in output if dict_['status']=='skipped']
failed = [dict_['file'] for dict_ in output if dict_['status']=='failed']
finish = datetime.datetime.now()
td = finish - start
duration = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600
duration_rounded = round(duration, 2)
if verbosity > 1:
print "\nDetailed report of failures:"
for dict_ in output:
if dict_['status'] == 'failed':
print "\n\n", dict_['file'], "\n================"
traceback.print_exception(dict_['exception_type'],
dict_['error'],
dict_['traceback'])
if verbosity > 0:
print "\n\nFinished processing %d layers in %s seconds.\n" % (
len(output), duration_rounded)
print "%d Created layers" % len(created)
print "%d Updated layers" % len(updated)
print "%d Skipped layers" % len(skipped)
print "%d Failed layers" % len(failed)
if len(output) > 0:
print "%f seconds per layer" % (duration * 1.0 / len(output))
def test_layer_upload(self):
"""Test that layers can be uploaded to running GeoNode/GeoServer
"""
layers = {}
expected_layers = []
not_expected_layers = []
for filename in os.listdir(gisdata.GOOD_DATA):
basename, extension = os.path.splitext(filename)
if extension.lower() in ['.tif', '.shp', '.zip']:
expected_layers.append(
os.path.join(
gisdata.GOOD_DATA,
filename))
for filename in os.listdir(gisdata.BAD_DATA):
not_expected_layers.append(
os.path.join(gisdata.BAD_DATA, filename)
)
uploaded = upload(gisdata.DATA_DIR, console=None)
for item in uploaded:
errors = 'error' in item
if errors:
# should this file have been uploaded?
if item['file'] in not_expected_layers:
continue
else:
msg = ('Could not upload file "%s", '
'and it is not in %s' % (
item['file'], not_expected_layers))
assert errors, msg
else:
msg = ('Upload should have returned either "name" or '
'"errors" for file %s.' % item['file'])
assert 'name' in item, msg
layers[item['file']] = item['name']
msg = ('There were %s compatible layers in the directory,'
' but only %s were sucessfully uploaded' %
(len(expected_layers), len(layers)))
# assert len(layers) == len(expected_layers), msg
for layer in expected_layers:
msg = ('The following file should have been uploaded'
'but was not: %s. ' % layer)
assert layer in layers, msg
layer_name = layers[layer]
# Check the layer is in the Django database
Layer.objects.get(name=layer_name)
# Check that layer is in geoserver
found = False
gs_username, gs_password = settings.OGC_SERVER['default'][
'USER'], settings.OGC_SERVER['default']['PASSWORD']
page = get_web_page(
os.path.join(
settings.OGC_SERVER['default']['LOCATION'],
'rest/layers'),
username=gs_username,
password=gs_password)
if page.find('rest/layers/%s.html' % layer_name) > 0:
found = True
if not found:
msg = (
'Upload could not be verified, the layer %s is not '
'in geoserver %s, but GeoNode did not raise any errors, '
'this should never happen.' %
(layer_name, settings.OGC_SERVER['default']['LOCATION']))
raise GeoNodeException(msg)
# server_url = settings.OGC_SERVER['default']['LOCATION'] + 'ows?'
# Verify that the GeoServer GetCapabilities record is accessible:
# metadata = get_layers_metadata(server_url, '1.0.0')
# msg = ('The metadata list should not be empty in server %s'
# % server_url)
# assert len(metadata) > 0, msg
# Check the keywords are recognized too
# Clean up and completely delete the layers
for layer in expected_layers:
layer_name = layers[layer]
Layer.objects.get(name=layer_name).delete()
def handle(self, *args, **options):
verbosity = int(options.get('verbosity'))
# ignore_errors = options.get('ignore_errors')
username = options.get('user')
user = get_valid_user(username)
overwrite = options.get('overwrite')
category = options.get('category', None)
private = options.get('private', False)
title = options.get('title', None)
date = options.get('date', None)
metadata_uploaded_preserve = options.get('metadata_uploaded_preserve',
False)
if verbosity > 0:
console = self.stdout
else:
console = None
if overwrite:
skip = False
else:
skip = True
keywords = options.get('keywords').split(',')
if len(keywords) == 1 and keywords[0] == '':
keywords = []
else:
keywords = map(str.strip, keywords)
regions = options.get('regions').split(',')
if len(regions) == 1 and regions[0] == '':
regions = []
else:
regions = map(str.strip, regions)
start = datetime.datetime.now()
output = []
for path in args:
out = upload(
path,
user=user,
overwrite=overwrite,
skip=skip,
keywords=keywords,
verbosity=verbosity,
console=console,
category=category,
regions=regions,
title=title,
date=date,
private=private,
metadata_uploaded_preserve=metadata_uploaded_preserve)
output.extend(out)
updated = [dict_['file']
for dict_ in output if dict_['status'] == 'updated']
created = [dict_['file']
for dict_ in output if dict_['status'] == 'created']
skipped = [dict_['file']
for dict_ in output if dict_['status'] == 'skipped']
failed = [dict_['file']
for dict_ in output if dict_['status'] == 'failed']
finish = datetime.datetime.now()
td = finish - start
duration = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600
duration_rounded = round(duration, 2)
if verbosity > 1:
print "\nDetailed report of failures:"
for dict_ in output:
if dict_['status'] == 'failed':
print "\n\n", dict_['file'], "\n================"
traceback.print_exception(dict_['exception_type'],
dict_['error'],
dict_['traceback'])
if verbosity > 0:
print "\n\nFinished processing %d layers in %s seconds.\n" % (
len(output), duration_rounded)
print "%d Created layers" % len(created)
print "%d Updated layers" % len(updated)
print "%d Skipped layers" % len(skipped)
print "%d Failed layers" % len(failed)
if len(output) > 0:
print "%f seconds per layer" % (duration * 1.0 / len(output))
def setUp(self):
call_command('loaddata', 'people_data', verbosity=0)
upload(gisdata.VECTOR_DATA, console=None)
def test_layer_upload(self):
"""Test that layers can be uploaded to running GeoNode/GeoServer
"""
layers = {}
expected_layers = []
not_expected_layers = []
for filename in os.listdir(gisdata.GOOD_DATA):
basename, extension = os.path.splitext(filename)
if extension.lower() in [".tif", ".shp", ".zip"]:
expected_layers.append(os.path.join(gisdata.GOOD_DATA, filename))
for filename in os.listdir(gisdata.BAD_DATA):
not_expected_layers.append(os.path.join(gisdata.BAD_DATA, filename))
uploaded = upload(gisdata.DATA_DIR, console=None)
for item in uploaded:
errors = "error" in item
if errors:
# should this file have been uploaded?
if item["file"] in not_expected_layers:
continue
else:
msg = 'Could not upload file "%s", ' "and it is not in %s" % (item["file"], not_expected_layers)
assert errors == True, msg
else:
msg = 'Upload should have returned either "name" or ' '"errors" for file %s.' % item["file"]
assert "name" in item, msg
layers[item["file"]] = item["name"]
msg = "There were %s compatible layers in the directory," " but only %s were sucessfully uploaded" % (
len(expected_layers),
len(layers),
)
# assert len(layers) == len(expected_layers), msg
uploaded_layers = [layer for layer in layers.items()]
for layer in expected_layers:
msg = "The following file should have been uploaded" "but was not: %s. " % layer
assert layer in layers, msg
layer_name = layers[layer]
# Check the layer is in the Django database
Layer.objects.get(name=layer_name)
# Check that layer is in geoserver
found = False
gs_username, gs_password = settings.GEOSERVER_CREDENTIALS
page = get_web_page(
os.path.join(settings.GEOSERVER_BASE_URL, "rest/layers"), username=gs_username, password=gs_password
)
if page.find("rest/layers/%s.html" % layer_name) > 0:
found = True
if not found:
msg = (
"Upload could not be verified, the layer %s is not "
"in geoserver %s, but GeoNode did not raise any errors, "
"this should never happen." % (layer_name, settings.GEOSERVER_BASE_URL)
)
raise GeoNodeException(msg)
server_url = settings.GEOSERVER_BASE_URL + "ows?"
# Verify that the GeoServer GetCapabilities record is accesible:
# metadata = get_layers_metadata(server_url, '1.0.0')
# msg = ('The metadata list should not be empty in server %s'
# % server_url)
# assert len(metadata) > 0, msg
# Check the keywords are recognized too
# Clean up and completely delete the layers
for layer in expected_layers:
layer_name = layers[layer]
Layer.objects.get(name=layer_name).delete()
def test_layer_upload(self):
"""Test that layers can be uploaded to running GeoNode/GeoServer
"""
layers = {}
expected_layers = []
not_expected_layers = []
for filename in os.listdir(gisdata.GOOD_DATA):
basename, extension = os.path.splitext(filename)
if extension.lower() in ['.tif', '.shp', '.zip']:
expected_layers.append(
os.path.join(
gisdata.GOOD_DATA,
filename))
for filename in os.listdir(gisdata.BAD_DATA):
not_expected_layers.append(
os.path.join(gisdata.BAD_DATA, filename)
)
uploaded = upload(gisdata.DATA_DIR, console=None)
for item in uploaded:
errors = 'error' in item
if errors:
# should this file have been uploaded?
if item['file'] in not_expected_layers:
continue
else:
msg = ('Could not upload file "%s", '
'and it is not in %s' % (
item['file'], not_expected_layers))
assert errors, msg
else:
msg = ('Upload should have returned either "name" or '
'"errors" for file %s.' % item['file'])
assert 'name' in item, msg
layers[item['file']] = item['name']
msg = ('There were %s compatible layers in the directory,'
' but only %s were sucessfully uploaded' %
(len(expected_layers), len(layers)))
# assert len(layers) == len(expected_layers), msg
for layer in expected_layers:
msg = ('The following file should have been uploaded'
'but was not: %s. ' % layer)
assert layer in layers, msg
layer_name = layers[layer]
# Check the layer is in the Django database
Layer.objects.get(name=layer_name)
# Check that layer is in geoserver
found = False
gs_username, gs_password = settings.OGC_SERVER['default'][
'USER'], settings.OGC_SERVER['default']['PASSWORD']
page = get_web_page(
os.path.join(
settings.OGC_SERVER['default']['LOCATION'],
'rest/layers'),
username=gs_username,
password=gs_password)
if page.find('rest/layers/%s.html' % layer_name) > 0:
found = True
if not found:
msg = (
'Upload could not be verified, the layer %s is not '
'in geoserver %s, but GeoNode did not raise any errors, '
'this should never happen.' %
(layer_name, settings.OGC_SERVER['default']['LOCATION']))
raise GeoNodeException(msg)
# server_url = settings.OGC_SERVER['default']['LOCATION'] + 'ows?'
# Verify that the GeoServer GetCapabilities record is accessible:
# metadata = get_layers_metadata(server_url, '1.0.0')
# msg = ('The metadata list should not be empty in server %s'
# % server_url)
# assert len(metadata) > 0, msg
# Check the keywords are recognized too
# Clean up and completely delete the layers
for layer in expected_layers:
layer_name = layers[layer]
Layer.objects.get(name=layer_name).delete()
def handle(self, *args, **options):
verbosity = int(options.get('verbosity'))
ignore_errors = options.get('ignore_errors')
user = options.get('user')
overwrite = options.get('overwrite')
if verbosity > 0:
console = sys.stdout
else:
console = None
if overwrite == True:
skip = False
else:
skip = True
keywords = options.get('keywords').split()
start = datetime.datetime.now()
output = []
for path in args:
out = upload(path,
user=user,
overwrite=overwrite,
skip=skip,
keywords=keywords,
verbosity=verbosity,
console=console)
output.extend(out)
updated = [
dict_['file'] for dict_ in output if dict_['status'] == 'updated'
]
created = [
dict_['file'] for dict_ in output if dict_['status'] == 'created'
]
skipped = [
dict_['file'] for dict_ in output if dict_['status'] == 'skipped'
]
failed = [
dict_['file'] for dict_ in output if dict_['status'] == 'failed'
]
finish = datetime.datetime.now()
td = finish - start
duration = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600
duration_rounded = round(duration, 2)
if verbosity > 1:
print "\nDetailed report of failures:"
for dict_ in output:
if dict_['status'] == 'failed':
print "\n\n", dict_['file'], "\n================"
traceback.print_exception(dict_['exception_type'],
dict_['error'],
dict_['traceback'])
if verbosity > 0:
print "\n\nFinished processing %d layers in %s seconds.\n" % (
len(output), duration_rounded)
print "%d Created layers" % len(created)
print "%d Updated layers" % len(updated)
print "%d Skipped layers" % len(skipped)
print "%d Failed layers" % len(failed)
if len(output) > 0:
print "%f seconds per layer" % (duration * 1.0 / len(output))
def handle(self, *args, **options):
verbosity = int(options.get('verbosity'))
# ignore_errors = options.get('ignore_errors')
username = options.get('user')
user = get_valid_user(username)
overwrite = options.get('overwrite')
category = options.get('category', None)
private = options.get('private', False)
title = options.get('title', None)
if verbosity > 0:
console = self.stdout
else:
console = None
if overwrite:
skip = False
else:
skip = True
keywords = options.get('keywords').split(',')
if len(keywords) == 1 and keywords[0] == '':
keywords = []
else:
keywords = map(str.strip, keywords)
regions = options.get('regions').split(',')
if len(regions) == 1 and regions[0] == '':
regions = []
else:
regions = map(str.strip, regions)
start = datetime.datetime.now()
output = []
for path in args:
out = upload(path,
user=user,
overwrite=overwrite,
skip=skip,
keywords=keywords,
verbosity=verbosity,
console=console,
category=category,
regions=regions,
title=title,
private=private)
output.extend(out)
updated = [
dict_['file'] for dict_ in output if dict_['status'] == 'updated'
]
created = [
dict_['file'] for dict_ in output if dict_['status'] == 'created'
]
skipped = [
dict_['file'] for dict_ in output if dict_['status'] == 'skipped'
]
failed = [
dict_['file'] for dict_ in output if dict_['status'] == 'failed'
]
finish = datetime.datetime.now()
td = finish - start
duration = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600
duration_rounded = round(duration, 2)
if verbosity > 1:
print "\nDetailed report of failures:"
for dict_ in output:
if dict_['status'] == 'failed':
print "\n\n", dict_['file'], "\n================"
traceback.print_exception(dict_['exception_type'],
dict_['error'],
dict_['traceback'])
if verbosity > 0:
print "\n\nFinished processing %d layers in %s seconds.\n" % (
len(output), duration_rounded)
print "%d Created layers" % len(created)
print "%d Updated layers" % len(updated)
print "%d Skipped layers" % len(skipped)
print "%d Failed layers" % len(failed)
if len(output) > 0:
print "%f seconds per layer" % (duration * 1.0 / len(output))
def handle(self, *args, **options):
verbosity = int(options.get('verbosity'))
# ignore_errors = options.get('ignore_errors')
username = options.get('user')
user = get_valid_user(username)
overwrite = options.get('overwrite')
name = options.get('layername', None)
title = options.get('title', None)
abstract = options.get('abstract', None)
date = options.get('date', None)
license = options.get('license', None)
category = options.get('category', None)
private = options.get('private', False)
metadata_uploaded_preserve = options.get('metadata_uploaded_preserve',
False)
charset = options.get('charset', 'UTF-8')
if verbosity > 0:
console = self.stdout
else:
console = None
if overwrite:
skip = False
else:
skip = True
keywords = options.get('keywords').split(',')
if len(keywords) == 1 and keywords[0] == '':
keywords = []
else:
keywords = [k.strip() for k in keywords]
regions = options.get('regions').split(',')
if len(regions) == 1 and regions[0] == '':
regions = []
else:
regions = [r.strip() for r in regions]
start = datetime.datetime.now(timezone.get_current_timezone())
output = []
for path in options['path']:
out = upload(path,
user=user,
overwrite=overwrite,
skip=skip,
name=name,
title=title,
abstract=abstract,
date=date,
keywords=keywords,
verbosity=verbosity,
console=console,
license=license,
category=category,
regions=regions,
private=private,
metadata_uploaded_preserve=metadata_uploaded_preserve,
charset=charset)
output.extend(out)
updated = [
dict_['file'] for dict_ in output if dict_['status'] == 'updated'
]
created = [
dict_['file'] for dict_ in output if dict_['status'] == 'created'
]
skipped = [
dict_['file'] for dict_ in output if dict_['status'] == 'skipped'
]
failed = [
dict_['file'] for dict_ in output if dict_['status'] == 'failed'
]
finish = datetime.datetime.now(timezone.get_current_timezone())
td = finish - start
duration = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600
duration_rounded = round(duration, 2)
if verbosity > 1:
print("\nDetailed report of failures:")
for dict_ in output:
if dict_['status'] == 'failed':
print("\n\n", dict_['file'], "\n================")
traceback.print_exception(dict_['exception_type'],
dict_['error'],
dict_['traceback'])
if verbosity > 0:
print("\n\nFinished processing {} layers in {} seconds.\n".format(
len(output), duration_rounded))
print("{} Created layers".format(len(created)))
print("{} Updated layers".format(len(updated)))
print("{} Skipped layers".format(len(skipped)))
print("{} Failed layers".format(len(failed)))
if len(output) > 0:
print("{} seconds per layer".format(duration * 1.0 /
len(output)))
