def import_maps(gs_data_dir, conn, zipfile,
no_password=False, chown_to=None, do_django_layer_save=True,
from_string=None, to_string=None):
tempdir = tempfile.mkdtemp()
temppath = lambda *p: os.path.join(tempdir, *p)
os.system('unzip %s -d %s' % (zipfile, tempdir))
for layer_name in os.listdir(temppath('layers')):
import_layer(gs_data_dir, conn,
temppath('layers', layer_name), layer_name,
no_password, chown_to, do_django_layer_save)
conn.commit()
print 'layers import complete'
def import_models(path, add_owner=False):
with open(path, 'r') as f:
models = serializers.deserialize('json', f)
for model in models:
if add_owner:
owner = User.objects.filter(pk=model.object.owner_id)
if not owner:
model.object.owner = User.objects.get(pk=1)
model.save()
return models
print 'importing maps'
import_models(temppath('maps.json'), add_owner=True)
print 'importing map layers'
maplayer_models = import_models(temppath('maplayers.json'))
print 'importing map thumb specs'
with open(temppath('map_thumb_specs.json')) as f:
map_thumbs = json.load(f)
thumb_updater = (make_thumbnail_updater(from_string, to_string)
if (from_string is not None and to_string is not None)
else lambda m:m.save())
for mapid, thumb_spec in map_thumbs:
try:
m = Map.objects.get(pk=mapid)
except Map.DoesNotExist:
print 'No map "%s" for importing thumb spec' % mapid
else:
m.set_thumbnail(thumb_spec)
t = m.get_thumbnail()
thumb_updater(t)
if from_string is not None and to_string is not None:
print 'adjusting maplayer params'
updater = make_maplayer_updater(from_string, to_string)
for maplayer_model in maplayer_models:
try:
maplayer = MapLayer.objects.get(pk=maplayer_model.object.id)
except MapLayer.DoesNotExist:
print 'No maplayer "%s" for updating layer params' % maplayer.object.id
else:
updater(maplayer)
else:
thumb_spec_path = temppath('thumb_spec.json')
thumbnail = layer.get_thumbnail()
if thumbnail:
print 'thumbnail already exists for: %s ... skipping creation' % layer_name
else:
if os.path.isfile(thumb_spec_path):
with open(thumb_spec_path) as f:
thumb_spec = json.load(f)
layer.set_thumbnail(thumb_spec)
# rename thumb_spec if asked to
if (th_from_string is not None and th_to_string is not None):
thumbnail = layer.get_thumbnail()
if thumbnail:
updater = make_thumbnail_updater(th_from_string, th_to_string)
updater(thumbnail)
else:
print 'No thumbnail to update spec for layer: %s' % layer_name
cursor.close()
# Load layer status
with open(temppath('publishingstatus.json'), 'r') as f:
statuses = serializers.deserialize('json', f)
for status in statuses:
try:
# Is there already a publishing status?
ps = PublishingStatus.objects.get(layer=status.object.layer)
ps.status = status.object.status
ps.save()
def import_maps(conn, zipfile,
no_password=False, chown_to=None, do_django_layer_save=True,
from_string=None, to_string=None):
# unless specified, assume importing to local setup
if to_string is None:
to_string = settings.SITEURL
tempdir = tempfile.mkdtemp()
temppath = lambda *p: os.path.join(tempdir, *p)
os.system('unzip %s -d %s' % (zipfile, tempdir))
for layer_name in os.listdir(temppath('layers')):
import_layer(conn,
temppath('layers', layer_name), layer_name,
no_password, chown_to, do_django_layer_save)
conn.commit()
print 'layers import complete'
def import_models(path, add_owner=False):
with open(path, 'r') as f:
# deserialize returns a generator, read everything
models = list(serializers.deserialize('json', f))
for model in models:
if add_owner:
owner = User.objects.filter(pk=model.object.owner_id)
if not owner:
model.object.owner = User.objects.get(pk=1)
model.save()
print 'imported %s=%s' % (model.object.__class__.__name__, model.object.pk)
return models
print 'importing maps'
import_models(temppath('maps.json'), add_owner=True)
print 'importing map layers'
maplayer_models = import_models(temppath('maplayers.json'))
print 'importing users'
import_models(temppath('comment_users.json'))
print 'importing map comments'
import_models(temppath('map_comments.json'))
print 'importing map publishing status'
# make sure we don't end up with duplicates, just copy the status value
with open(temppath('map_publishing_status.json'), 'r') as f:
models = list(serializers.deserialize('json', f))
for model in models:
real_object = model.object
PublishingStatus.objects.set_status(real_object.map or real_object.layer, real_object.status)
print 'importing map thumb specs'
with open(temppath('map_thumb_specs.json')) as f:
map_thumbs = json.load(f)
thumb_updater = (make_thumbnail_updater(from_string, to_string)
if (from_string is not None and to_string is not None)
else lambda m:m.save())
for mapid, thumb_spec in map_thumbs:
try:
m = Map.objects.get(pk=mapid)
except Map.DoesNotExist:
print 'No map "%s" for importing thumb spec' % mapid
else:
m.set_thumbnail(thumb_spec)
t = m.get_thumbnail()
thumb_updater(t)
if from_string is not None and to_string is not None:
print 'adjusting maplayer params'
if to_string[-1] != '/':
to_string += '/'
updater = make_maplayer_updater(from_string, to_string)
for maplayer_model in maplayer_models:
try:
maplayer = MapLayer.objects.get(pk=maplayer_model.object.id)
except MapLayer.DoesNotExist:
print 'No maplayer "%s" for updating layer params' % maplayer.object.id
else:
updater(maplayer)
def import_layer(gs_data_dir, conn, layer_tempdir, layer_name,
no_password=False, chown_to=None, do_django_layer_save=True,
th_from_string=None, th_to_string=None):
print 'importing layer: %s' % layer_name
gspath = lambda *p: os.path.join(gs_data_dir, *p)
temppath = lambda *p: os.path.join(layer_tempdir, *p)
restore_string = 'pg_restore --host=%s --dbname=%s --clean --username=%s %s < %s' % (
settings.DB_DATASTORE_HOST, settings.DB_DATASTORE_DATABASE, settings.DB_DATASTORE_USER,
no_password and '--no-password' or '',
temppath('layer.dump'),
)
# can't check return value since pg_restore will complain if any drop statements fail :(
os.system(restore_string)
# rebuild the geometry columns entry
with open(temppath("geom.info")) as fp:
s = fp.read()
geom_cols = eval(s)[0]
cursor = conn.cursor()
# f_table_catalog, f_table_schema, f_table_name
cursor.execute("delete from geometry_columns where f_table_schema='%s' and f_table_name='%s'" % (geom_cols[1],geom_cols[2]))
cursor.execute('insert into geometry_columns VALUES(%s)' % ','.join(["'%s'" % v for v in geom_cols]))
#copy in styles
for f in os.listdir(temppath('styles')):
shutil.copy(temppath('styles',f),gspath('styles'))
# and other geoserver info
for ws in os.listdir(temppath('workspaces')):
for store_name in os.listdir(temppath('workspaces',ws)):
for layer_name in os.listdir(temppath('workspaces',ws,store_name)):
dest_dir = gspath('workspaces', ws, store_name, layer_name)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
fdir = temppath('workspaces', ws, store_name, layer_name)
for f in os.listdir(fdir):
shutil.copy(os.path.join(fdir,f), dest_dir)
if chown_to:
from pwd import getpwnam
userid = getpwnam(chown_to)[2]
for root, dirs, files in os.walk(gspath()):
os.chown(root, userid, -1)
for f in files:
os.chown(os.path.join(root, f), userid, -1)
# reload catalog
Layer.objects.gs_catalog.http.request(settings.GEOSERVER_BASE_URL + "rest/reload",'POST')
if do_django_layer_save:
# now we can create the django model - must be done last when gscatalog is ready
with open(temppath("model.json")) as fp:
model_json = fp.read()
layer = serializers.deserialize("json", model_json).next()
owner = User.objects.filter(pk=layer.object.owner_id)
if not owner:
layer.object.owner = User.objects.get(pk=1)
layer_exists = Layer.objects.filter(typename=layer.object.typename)
if not layer_exists:
layer.save()
else:
print 'Layer %s already exists ... skipping model save' % layer_name
# add thumbnail if exists in src and not destination
try:
layer = Layer.objects.get(typename='geonode:%s' % layer_name)
except Layer.DoesNotExist:
print 'Layer %s does not exist. Could not update thumbnail spec' % layer_name
else:
thumb_spec_path = temppath('thumb_spec.json')
thumbnail = layer.get_thumbnail()
if thumbnail:
print 'thumbnail already exists for: %s ... skipping creation' % layer_name
else:
if os.path.isfile(thumb_spec_path):
with open(thumb_spec_path) as f:
thumb_spec = json.load(f)
layer.set_thumbnail(thumb_spec)
# rename thumb_spec if asked to
if (th_from_string is not None and th_to_string is not None):
thumbnail = layer.get_thumbnail()
if thumbnail:
updater = make_thumbnail_updater(th_from_string, th_to_string)
updater(thumbnail)
else:
print 'No thumbnail to update spec for layer: %s' % layer_name
cursor.close()
else:
thumb_spec_path = temppath('thumb_spec.json')
thumbnail = layer.get_thumbnail()
if thumbnail:
print 'thumbnail already exists for: %s ... skipping creation' % layer_name
else:
if os.path.isfile(thumb_spec_path):
with open(thumb_spec_path) as f:
thumb_spec = json.load(f)
layer.set_thumbnail(thumb_spec)
# rename thumb_spec if asked to
if (th_from_string is not None and th_to_string is not None):
thumbnail = layer.get_thumbnail()
if thumbnail:
updater = make_thumbnail_updater(th_from_string,
th_to_string)
updater(thumbnail)
else:
print 'No thumbnail to update spec for layer: %s' % layer_name
cursor.close()
# Load layer status
with open(temppath('publishingstatus.json'), 'r') as f:
statuses = serializers.deserialize('json', f)
for status in statuses:
try:
# Is there already a publishing status?
ps = PublishingStatus.objects.get(layer=status.object.layer)
ps.status = status.object.status
ps.save()
def import_maps(conn,
zipfile,
no_password=False,
chown_to=None,
do_django_layer_save=True,
from_string=None,
to_string=None):
# unless specified, assume importing to local setup
if to_string is None:
to_string = settings.SITEURL
tempdir = tempfile.mkdtemp()
temppath = lambda *p: os.path.join(tempdir, *p)
os.system('unzip %s -d %s' % (zipfile, tempdir))
for layer_name in os.listdir(temppath('layers')):
import_layer(conn, temppath('layers', layer_name), layer_name,
no_password, chown_to, do_django_layer_save)
conn.commit()
print 'layers import complete'
def import_models(path, add_owner=False):
with open(path, 'r') as f:
# deserialize returns a generator, read everything
models = list(serializers.deserialize('json', f))
for model in models:
if add_owner:
owner = User.objects.filter(pk=model.object.owner_id)
if not owner:
model.object.owner = User.objects.get(pk=1)
model.save()
print 'imported %s=%s' % (model.object.__class__.__name__,
model.object.pk)
return models
print 'importing maps'
import_models(temppath('maps.json'), add_owner=True)
print 'importing map layers'
maplayer_models = import_models(temppath('maplayers.json'))
print 'importing users'
import_models(temppath('comment_users.json'))
print 'importing map comments'
import_models(temppath('map_comments.json'))
print 'importing map publishing status'
# make sure we don't end up with duplicates, just copy the status value
with open(temppath('map_publishing_status.json'), 'r') as f:
models = list(serializers.deserialize('json', f))
for model in models:
real_object = model.object
PublishingStatus.objects.set_status(
real_object.map or real_object.layer, real_object.status)
print 'importing map thumb specs'
with open(temppath('map_thumb_specs.json')) as f:
map_thumbs = json.load(f)
thumb_updater = (make_thumbnail_updater(from_string, to_string) if
(from_string is not None
and to_string is not None) else lambda m: m.save())
for mapid, thumb_spec in map_thumbs:
try:
m = Map.objects.get(pk=mapid)
except Map.DoesNotExist:
print 'No map "%s" for importing thumb spec' % mapid
else:
m.set_thumbnail(thumb_spec)
t = m.get_thumbnail()
thumb_updater(t)
if from_string is not None and to_string is not None:
print 'adjusting maplayer params'
if to_string[-1] != '/':
to_string += '/'
updater = make_maplayer_updater(from_string, to_string)
for maplayer_model in maplayer_models:
try:
maplayer = MapLayer.objects.get(pk=maplayer_model.object.id)
except MapLayer.DoesNotExist:
print 'No maplayer "%s" for updating layer params' % maplayer.object.id
else:
updater(maplayer)
