def dump_geoserver_raster_data(self, config, settings, target_folder):
if (config.gs_data_dir):
if (config.gs_dump_raster_data):
# Dump '$config.gs_data_dir/geonode'
gs_data_root = os.path.join(config.gs_data_dir, 'geonode')
if not os.path.isabs(gs_data_root):
gs_data_root = os.path.join(settings.PROJECT_ROOT, '..', gs_data_root)
logger.info("Dumping GeoServer Uploaded Data from '"+gs_data_root+"'.")
if os.path.exists(gs_data_root):
gs_data_folder = os.path.join(target_folder, 'gs_data_dir', 'geonode')
if not os.path.exists(gs_data_folder):
os.makedirs(gs_data_folder)
copy_tree(gs_data_root, gs_data_folder,
ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1]))
logger.info("Dumped GeoServer Uploaded Data from '"+gs_data_root+"'.")
else:
logger.info("Skipped GeoServer Uploaded Data '"+gs_data_root+"'.")
# Dump '$config.gs_data_dir/data/geonode'
gs_data_root = os.path.join(config.gs_data_dir, 'data', 'geonode')
if not os.path.isabs(gs_data_root):
gs_data_root = os.path.join(settings.PROJECT_ROOT, '..', gs_data_root)
logger.info("Dumping GeoServer Uploaded Data from '"+gs_data_root+"'.")
if os.path.exists(gs_data_root):
gs_data_folder = os.path.join(target_folder, 'gs_data_dir', 'data', 'geonode')
if not os.path.exists(gs_data_folder):
os.makedirs(gs_data_folder)
copy_tree(gs_data_root, gs_data_folder,
ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1]))
logger.info("Dumped GeoServer Uploaded Data from '" + gs_data_root + "'.")
else:
logger.info("Skipped GeoServer Uploaded Data '"+gs_data_root+"'.")
def restore_geoserver_raster_data(self, config, settings, target_folder):
if (config.gs_data_dir):
if (config.gs_dump_raster_data):
# Restore '$config.gs_data_dir/geonode'
gs_data_folder = os.path.join(target_folder, 'gs_data_dir', 'geonode')
if os.path.exists(gs_data_folder):
gs_data_root = os.path.join(config.gs_data_dir, 'geonode')
if not os.path.isabs(gs_data_root):
gs_data_root = os.path.join(settings.PROJECT_ROOT, '..', gs_data_root)
if not os.path.exists(gs_data_root):
os.makedirs(gs_data_root)
copy_tree(gs_data_folder, gs_data_root)
print(f"GeoServer Uploaded Raster Data Restored to '{gs_data_root}'.")
else:
print(('Skipping geoserver raster data restore: ' +
f'directory "{gs_data_folder}" not found.'))
# Restore '$config.gs_data_dir/data/geonode'
gs_data_folder = os.path.join(target_folder, 'gs_data_dir', 'data', 'geonode')
if os.path.exists(gs_data_folder):
gs_data_root = os.path.join(config.gs_data_dir, 'data', 'geonode')
if not os.path.isabs(gs_data_root):
gs_data_root = os.path.join(settings.PROJECT_ROOT, '..', gs_data_root)
if not os.path.exists(gs_data_root):
os.makedirs(gs_data_root)
copy_tree(gs_data_folder, gs_data_root)
print(f"GeoServer Uploaded Data Restored to '{gs_data_root}'.")
else:
print(('Skipping geoserver raster data restore: ' +
f'directory "{gs_data_folder}" not found.'))
def test_backup_of_child_directories(
self, patch_isdir, patch_listdir, patch_getmtime, patch_shutil_copytree, path_os_exists):
"""
Test that all directories which meet the 'ignore criteria are backed-up'
"""
copy_tree('/src', '/dst', ignore=ignore_time('>=', datetime.now().isoformat()))
self.assertTrue(patch_shutil_copytree.called)
def test_skipped_backup_of_root_files_with_modification_dates_not_meeting_greater_than_filter_criteria(
self, patch_isdir, patch_listdir, patch_getmtime, patch_shutil_copy2):
"""
Test that all root directories whose modification dates do not meet the 'data_dt_filter'
less-than iso timestamp are not backed-up
"""
copy_tree('/src', '/dst', ignore=ignore_time('>', datetime.now().isoformat()))
self.assertTrue(patch_shutil_copy2.called)
def restore_geoserver_raster_data(self, config, settings, target_folder):
if (config.gs_data_dir):
if (config.gs_dump_raster_data):
gs_data_folder = os.path.join(target_folder, 'gs_data_dir',
'data', 'geonode')
if not os.path.exists(gs_data_folder):
print(('Skipping geoserver raster data restore: ' +
'directory "{}" not found.'.format(gs_data_folder)))
return
# Restore '$config.gs_data_dir/data/geonode'
gs_data_root = os.path.join(config.gs_data_dir, 'data',
'geonode')
if not os.path.isabs(gs_data_root):
gs_data_root = os.path.join(settings.PROJECT_ROOT, '..',
gs_data_root)
try:
chmod_tree(gs_data_root)
except Exception:
print(
'Original GeoServer Data Dir "{}" must be writable by the current user. \
Do not forget to copy it first. It will be wiped-out by the Restore procedure!'
.format(gs_data_root))
raise
try:
shutil.rmtree(gs_data_root)
print('Cleaned out old GeoServer Data Dir: ' +
gs_data_root)
except Exception:
pass
if not os.path.exists(gs_data_root):
os.makedirs(gs_data_root)
copy_tree(gs_data_folder, gs_data_root)
chmod_tree(gs_data_root)
print("GeoServer Uploaded Data Restored to '" + gs_data_root +
"'.")
# Cleanup '$config.gs_data_dir/gwc-layers'
gwc_layers_root = os.path.join(config.gs_data_dir,
'gwc-layers')
if not os.path.isabs(gwc_layers_root):
gwc_layers_root = os.path.join(settings.PROJECT_ROOT, '..',
gwc_layers_root)
try:
shutil.rmtree(gwc_layers_root)
print('Cleaned out old GeoServer GWC Layers Config: ' +
gwc_layers_root)
except Exception:
pass
if not os.path.exists(gwc_layers_root):
os.makedirs(gwc_layers_root)
def dump_geoserver_raster_data(self, config, settings, target_folder):
if (config.gs_data_dir):
if (config.gs_dump_raster_data):
# Dump '$config.gs_data_dir/data/geonode'
gs_data_root = os.path.join(config.gs_data_dir, 'data', 'geonode')
if not os.path.isabs(gs_data_root):
gs_data_root = os.path.join(settings.PROJECT_ROOT, '..', gs_data_root)
gs_data_folder = os.path.join(target_folder, 'gs_data_dir', 'data', 'geonode')
if not os.path.exists(gs_data_folder):
os.makedirs(gs_data_folder)
copy_tree(gs_data_root, gs_data_folder)
print "Dumped GeoServer Uploaded Data from '"+gs_data_root+"'."
def restore_geoserver_raster_data(self, config, settings, target_folder):
if (config.gs_data_dir):
if (config.gs_dump_raster_data):
gs_data_folder = os.path.join(target_folder, 'gs_data_dir', 'data', 'geonode')
if not os.path.exists(gs_data_folder):
print('Skipping geoserver raster data restore: ' +
'directory "{}" not found.'.format(gs_data_folder))
return
# Restore '$config.gs_data_dir/data/geonode'
gs_data_root = os.path.join(config.gs_data_dir, 'data', 'geonode')
if not os.path.isabs(gs_data_root):
gs_data_root = os.path.join(settings.PROJECT_ROOT, '..', gs_data_root)
try:
chmod_tree(gs_data_root)
except:
print 'Original GeoServer Data Dir "{}" must be writable by the current user. \
Do not forget to copy it first. It will be wiped-out by the Restore procedure!'.format(gs_data_root)
raise
try:
shutil.rmtree(gs_data_root)
print 'Cleaned out old GeoServer Data Dir: ' + gs_data_root
except:
pass
if not os.path.exists(gs_data_root):
os.makedirs(gs_data_root)
copy_tree(gs_data_folder, gs_data_root)
chmod_tree(gs_data_root)
print "GeoServer Uploaded Data Restored to '"+gs_data_root+"'."
# Cleanup '$config.gs_data_dir/gwc-layers'
gwc_layers_root = os.path.join(config.gs_data_dir, 'gwc-layers')
if not os.path.isabs(gwc_layers_root):
gwc_layers_root = os.path.join(settings.PROJECT_ROOT, '..', gwc_layers_root)
try:
shutil.rmtree(gwc_layers_root)
print 'Cleaned out old GeoServer GWC Layers Config: ' + gwc_layers_root
except:
pass
if not os.path.exists(gwc_layers_root):
os.makedirs(gwc_layers_root)
def restore_full(archive):
"""Full Restore of GeoNode DB"""
try:
# Create Target Folder
restore_folder = 'restore'
if not os.path.exists(restore_folder):
os.makedirs(restore_folder)
# Extract ZIP Archive to Target Folder
target_folder = extract_archive(archive, restore_folder)
# Prepare Target DB
try:
call_command('migrate', interactive=False, load_initial_data=False)
call_command('flush', interactive=False, load_initial_data=False)
helpers.patch_db()
except:
traceback.print_exc()
# Restore Fixtures
for app_name, dump_name in zip(helpers.app_names, helpers.dump_names):
fixture_file = os.path.join(target_folder, dump_name + '.json')
print "Deserializing " + fixture_file
try:
call_command('loaddata', fixture_file, app_label=app_name)
except:
#traceback.print_exc()
print "WARNING: No valid fixture data found for '" + dump_name + "'."
#helpers.load_fixture(app_name, fixture_file)
# Restore Media Root
media_root = settings.MEDIA_ROOT
media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)
try:
shutil.rmtree(media_root)
except:
pass
if not os.path.exists(media_root):
os.makedirs(media_root)
copy_tree(media_folder, media_root)
chmod_tree(media_root)
print "Media Files Restored into '" + media_root + "'."
# Restore Static Root
static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
try:
shutil.rmtree(static_root)
except:
pass
if not os.path.exists(static_root):
os.makedirs(static_root)
copy_tree(static_folder, static_root)
chmod_tree(static_root)
print "Static Root Restored into '" + static_root + "'."
# Restore Static Folders
static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder,
helpers.STATICFILES_DIRS)
for static_files_folder in static_folders:
try:
shutil.rmtree(static_files_folder)
except:
pass
if not os.path.exists(static_files_folder):
os.makedirs(static_files_folder)
copy_tree(
os.path.join(
static_files_folders,
os.path.basename(os.path.normpath(static_files_folder))),
static_files_folder)
chmod_tree(static_files_folder)
print "Static Files Restored into '" + static_files_folder + "'."
# Restore Template Folders
template_folders = settings.TEMPLATE_DIRS
template_files_folders = os.path.join(target_folder,
helpers.TEMPLATE_DIRS)
for template_files_folder in template_folders:
try:
shutil.rmtree(template_files_folder)
except:
pass
if not os.path.exists(template_files_folder):
os.makedirs(template_files_folder)
copy_tree(
os.path.join(
template_files_folders,
os.path.basename(os.path.normpath(template_files_folder))),
template_files_folder)
chmod_tree(template_files_folder)
print "Template Files Restored into '" + template_files_folder + "'."
# Restore Locale Folders
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder,
helpers.LOCALE_PATHS)
for locale_files_folder in locale_folders:
try:
shutil.rmtree(locale_files_folder)
except:
pass
if not os.path.exists(locale_files_folder):
os.makedirs(locale_files_folder)
copy_tree(
os.path.join(
locale_files_folders,
os.path.basename(os.path.normpath(locale_files_folder))),
locale_files_folder)
chmod_tree(locale_files_folder)
print "Locale Files Restored into '" + locale_files_folder + "'."
# Cleanup DB
try:
helpers.cleanup_db()
except:
traceback.print_exc()
except Exception, err:
print str(err)
def handle(self, **options):
# ignore_errors = options.get('ignore_errors')
config = Config(options)
force_exec = options.get('force_exec')
backup_dir = options.get('backup_dir')
skip_geoserver = options.get('skip_geoserver')
if not backup_dir or len(backup_dir) == 0:
raise CommandError("Destination folder '--backup-dir' is mandatory")
print "Before proceeding with the Backup, please ensure that:"
print " 1. The backend (DB or whatever) is accessible and you have rights"
print " 2. The GeoServer is up and running and reachable from this machine"
message = 'You want to proceed?'
if force_exec or helpers.confirm(prompt=message, resp=False):
# Create Target Folder
dir_time_suffix = get_dir_time_suffix()
target_folder = os.path.join(backup_dir, dir_time_suffix)
if not os.path.exists(target_folder):
os.makedirs(target_folder)
# Temporary folder to store backup files. It will be deleted at the end.
os.chmod(target_folder, 0777)
if not skip_geoserver:
self.create_geoserver_backup(settings, target_folder)
self.dump_geoserver_raster_data(config, settings, target_folder)
self.dump_geoserver_vector_data(config, settings, target_folder)
print("Duming geoserver external resources")
self.dump_geoserver_externals(config, settings, target_folder)
else:
print("Skipping geoserver backup")
try:
# Deactivate GeoNode Signals
print "Deactivating GeoNode Signals..."
designals()
print "...done!"
# Dump Fixtures
for app_name, dump_name in zip(config.app_names, config.dump_names):
print "Dumping '"+app_name+"' into '"+dump_name+".json'."
# Point stdout at a file for dumping data to.
output = open(os.path.join(target_folder, dump_name+'.json'), 'w')
call_command('dumpdata', app_name, format='json', indent=2, natural=True, stdout=output)
output.close()
# Store Media Root
media_root = settings.MEDIA_ROOT
media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)
if not os.path.exists(media_folder):
os.makedirs(media_folder)
copy_tree(media_root, media_folder)
print "Saved Media Files from '"+media_root+"'."
# Store Static Root
static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
if not os.path.exists(static_folder):
os.makedirs(static_folder)
copy_tree(static_root, static_folder)
print "Saved Static Root from '"+static_root+"'."
# Store Static Folders
static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS)
if not os.path.exists(static_files_folders):
os.makedirs(static_files_folders)
for static_files_folder in static_folders:
static_folder = os.path.join(static_files_folders,
os.path.basename(os.path.normpath(static_files_folder)))
if not os.path.exists(static_folder):
os.makedirs(static_folder)
copy_tree(static_files_folder, static_folder)
print "Saved Static Files from '"+static_files_folder+"'."
# Store Template Folders
template_folders = []
try:
template_folders = settings.TEMPLATE_DIRS
except:
try:
template_folders = settings.TEMPLATES[0]['DIRS']
except:
pass
template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)
if not os.path.exists(template_files_folders):
os.makedirs(template_files_folders)
for template_files_folder in template_folders:
template_folder = os.path.join(template_files_folders,
os.path.basename(os.path.normpath(template_files_folder)))
if not os.path.exists(template_folder):
os.makedirs(template_folder)
copy_tree(template_files_folder, template_folder)
print "Saved Template Files from '"+template_files_folder+"'."
# Store Locale Folders
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS)
if not os.path.exists(locale_files_folders):
os.makedirs(locale_files_folders)
for locale_files_folder in locale_folders:
locale_folder = os.path.join(locale_files_folders,
os.path.basename(os.path.normpath(locale_files_folder)))
if not os.path.exists(locale_folder):
os.makedirs(locale_folder)
copy_tree(locale_files_folder, locale_folder)
print "Saved Locale Files from '"+locale_files_folder+"'."
# Create Final ZIP Archive
zip_dir(target_folder, os.path.join(backup_dir, dir_time_suffix+'.zip'))
# Clean-up Temp Folder
try:
shutil.rmtree(target_folder)
except:
print "WARNING: Could not be possible to delete the temp folder: '" + str(target_folder) + "'"
print "Backup Finished. Archive generated."
return str(os.path.join(backup_dir, dir_time_suffix+'.zip'))
finally:
# Reactivate GeoNode Signals
print "Reactivating GeoNode Signals..."
resignals()
print "...done!"
def execute_restore(self, **options):
self.validate_backup_file_options(**options)
ignore_errors = options.get('ignore_errors')
force_exec = options.get('force_exec')
skip_geoserver = options.get('skip_geoserver')
skip_geoserver_info = options.get('skip_geoserver_info')
skip_geoserver_security = options.get('skip_geoserver_security')
backup_file = options.get('backup_file')
recovery_file = options.get('recovery_file')
backup_files_dir = options.get('backup_files_dir')
with_logs = options.get('with_logs')
notify = options.get('notify')
soft_reset = options.get('soft_reset')
# choose backup_file from backup_files_dir, if --backup-files-dir was provided
if backup_files_dir:
backup_file = self.parse_backup_files_dir(backup_files_dir)
else:
backup_files_dir = os.path.dirname(backup_file)
# calculate and validate backup archive hash
backup_md5 = self.validate_backup_file_hash(backup_file)
# check if the original backup file ini setting are available or not
backup_ini = self.check_backup_ini_settings(backup_file)
if backup_ini:
options['config'] = backup_ini
config = utils.Config(options)
# check if the backup has already been restored
if with_logs:
if RestoredBackup.objects.filter(archive_md5=backup_md5):
raise RuntimeError(
'Backup archive has already been restored. If you want to restore '
'this backup anyway, run the script without "-l" argument.'
)
# get a list of instance administrators' emails
admin_emails = []
if notify:
admins = get_user_model().objects.filter(is_superuser=True)
for user in admins:
if user.email:
admin_emails.append(user.email)
print("Before proceeding with the Restore, please ensure that:")
print(
" 1. The backend (DB or whatever) is accessible and you have rights"
)
print(
" 2. The GeoServer is up and running and reachable from this machine"
)
message = 'WARNING: The restore will overwrite ALL GeoNode data. You want to proceed?'
if force_exec or utils.confirm(prompt=message, resp=False):
# Create Target Folder
# restore_folder must be located in the directory Geoserver has access to (and it should
# not be Geoserver data dir)
# for dockerized project-template GeoNode projects, it should be located in /backup-restore,
# otherwise default tmp directory is chosen
temp_dir_path = backup_files_dir if os.path.exists(
backup_files_dir) else None
restore_folder = os.path.join(temp_dir_path,
f'tmp{str(uuid.uuid4())[:4]}')
try:
os.makedirs(restore_folder)
except Exception as e:
raise e
try:
# Extract ZIP Archive to Target Folder
target_folder = extract_archive(backup_file, restore_folder)
# Write Checks
media_root = settings.MEDIA_ROOT
media_folder = os.path.join(target_folder, utils.MEDIA_ROOT)
static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, utils.STATIC_ROOT)
static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder,
utils.STATICFILES_DIRS)
template_folders = []
try:
template_folders = settings.TEMPLATE_DIRS
except Exception:
try:
template_folders = settings.TEMPLATES[0]['DIRS']
except Exception:
pass
template_files_folders = os.path.join(target_folder,
utils.TEMPLATE_DIRS)
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder,
utils.LOCALE_PATHS)
try:
print(
f"[Sanity Check] Full Write Access to '{restore_folder}' ..."
)
chmod_tree(restore_folder)
print(
f"[Sanity Check] Full Write Access to '{media_root}' ..."
)
chmod_tree(media_root)
print(
f"[Sanity Check] Full Write Access to '{static_root}' ..."
)
chmod_tree(static_root)
for static_files_folder in static_folders:
print(
f"[Sanity Check] Full Write Access to '{static_files_folder}' ..."
)
chmod_tree(static_files_folder)
for template_files_folder in template_folders:
print(
f"[Sanity Check] Full Write Access to '{template_files_folder}' ..."
)
chmod_tree(template_files_folder)
for locale_files_folder in locale_folders:
print(
f"[Sanity Check] Full Write Access to '{locale_files_folder}' ..."
)
chmod_tree(locale_files_folder)
except Exception as exception:
if notify:
restore_notification.apply_async(
(admin_emails, backup_file, backup_md5,
str(exception)))
print(
"...Sanity Checks on Folder failed. Please make sure that the current user has full WRITE access to the above folders (and sub-folders or files)."
) # noqa
print("Reason:")
raise
if not skip_geoserver:
try:
print(
f"[Sanity Check] Full Write Access to '{target_folder}' ..."
)
chmod_tree(target_folder)
self.restore_geoserver_backup(config, settings,
target_folder,
skip_geoserver_info,
skip_geoserver_security,
ignore_errors,
soft_reset)
self.prepare_geoserver_gwc_config(config, settings)
self.restore_geoserver_raster_data(
config, settings, target_folder)
self.restore_geoserver_vector_data(
config, settings, target_folder, soft_reset)
print("Restoring geoserver external resources")
self.restore_geoserver_externals(
config, settings, target_folder)
except Exception as exception:
if recovery_file:
with tempfile.TemporaryDirectory(
dir=temp_dir_path) as restore_folder:
recovery_folder = extract_archive(
recovery_file, restore_folder)
self.restore_geoserver_backup(
config, settings, recovery_folder,
skip_geoserver_info,
skip_geoserver_security, ignore_errors,
soft_reset)
self.restore_geoserver_raster_data(
config, settings, recovery_folder)
self.restore_geoserver_vector_data(
config, settings, recovery_folder,
soft_reset)
self.restore_geoserver_externals(
config, settings, recovery_folder)
if notify:
restore_notification.apply_async(
(admin_emails, backup_file, backup_md5,
str(exception)))
raise exception
else:
print("Skipping geoserver backup restore")
# Prepare Target DB
try:
call_command('makemigrations', interactive=False)
call_command('migrate',
interactive=False,
load_initial_data=False)
db_name = settings.DATABASES['default']['NAME']
db_user = settings.DATABASES['default']['USER']
db_port = settings.DATABASES['default']['PORT']
db_host = settings.DATABASES['default']['HOST']
db_passwd = settings.DATABASES['default']['PASSWORD']
utils.patch_db(db_name, db_user, db_port, db_host,
db_passwd, settings.MONITORING_ENABLED)
except Exception:
traceback.print_exc()
try:
# Deactivate GeoNode Signals
with DisableDjangoSignals():
# Flush DB
try:
db_name = settings.DATABASES['default']['NAME']
db_user = settings.DATABASES['default']['USER']
db_port = settings.DATABASES['default']['PORT']
db_host = settings.DATABASES['default']['HOST']
db_passwd = settings.DATABASES['default'][
'PASSWORD']
utils.flush_db(db_name, db_user, db_port, db_host,
db_passwd)
except Exception:
try:
call_command('flush', interactive=False)
except Exception:
traceback.print_exc()
raise
# Restore Fixtures
abortlater = False
for app_name, dump_name in zip(config.app_names,
config.dump_names):
fixture_file = os.path.join(
target_folder, f"{dump_name}.json")
print(f"Deserializing '{fixture_file}'")
try:
call_command('loaddata',
fixture_file,
app_label=app_name)
except IntegrityError:
traceback.print_exc()
logger.warning(
f"WARNING: The fixture '{dump_name}' fails on integrity check and import is aborted after all fixtures have been checked."
) # noqa
abortlater = True
except Exception as e:
traceback.print_exc()
logger.warning(
f"WARNING: No valid fixture data found for '{dump_name}'."
)
# helpers.load_fixture(app_name, fixture_file)
raise e
if abortlater:
raise IntegrityError()
# Restore Media Root
if config.gs_data_dt_filter[0] is None:
shutil.rmtree(media_root, ignore_errors=True)
if not os.path.exists(media_root):
os.makedirs(media_root)
copy_tree(media_folder, media_root)
chmod_tree(media_root)
print(f"Media Files Restored into '{media_root}'.")
# Restore Static Root
if config.gs_data_dt_filter[0] is None:
shutil.rmtree(static_root, ignore_errors=True)
if not os.path.exists(static_root):
os.makedirs(static_root)
copy_tree(static_folder, static_root)
chmod_tree(static_root)
print(f"Static Root Restored into '{static_root}'.")
# Restore Static Folders
for static_files_folder in static_folders:
# skip restoration of static files of apps not located under LOCAL_ROOT path
# (check to prevent overriding files from site-packages
# in project-template based GeoNode projects)
if getattr(settings, 'LOCAL_ROOT', None) and \
not static_files_folder.startswith(settings.LOCAL_ROOT):
print(
f"Skipping static directory: {static_files_folder}. "
f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}."
)
continue
if config.gs_data_dt_filter[0] is None:
shutil.rmtree(static_files_folder,
ignore_errors=True)
if not os.path.exists(static_files_folder):
os.makedirs(static_files_folder)
copy_tree(
os.path.join(
static_files_folders,
os.path.basename(
os.path.normpath(
static_files_folder))),
static_files_folder)
chmod_tree(static_files_folder)
print(
f"Static Files Restored into '{static_files_folder}'."
)
# Restore Template Folders
for template_files_folder in template_folders:
# skip restoration of template files of apps not located under LOCAL_ROOT path
# (check to prevent overriding files from site-packages
# in project-template based GeoNode projects)
if getattr(settings, 'LOCAL_ROOT', None) and \
not template_files_folder.startswith(settings.LOCAL_ROOT):
print(
f"Skipping template directory: {template_files_folder}. "
f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}."
)
continue
if config.gs_data_dt_filter[0] is None:
shutil.rmtree(template_files_folder,
ignore_errors=True)
if not os.path.exists(template_files_folder):
os.makedirs(template_files_folder)
copy_tree(
os.path.join(
template_files_folders,
os.path.basename(
os.path.normpath(
template_files_folder))),
template_files_folder)
chmod_tree(template_files_folder)
print(
f"Template Files Restored into '{template_files_folder}'."
)
# Restore Locale Folders
for locale_files_folder in locale_folders:
# skip restoration of locale files of apps not located under LOCAL_ROOT path
# (check to prevent overriding files from site-packages
# in project-template based GeoNode projects)
if getattr(settings, 'LOCAL_ROOT', None) and \
not locale_files_folder.startswith(settings.LOCAL_ROOT):
print(
f"Skipping locale directory: {locale_files_folder}. "
f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}."
)
continue
if config.gs_data_dt_filter[0] is None:
shutil.rmtree(locale_files_folder,
ignore_errors=True)
if not os.path.exists(locale_files_folder):
os.makedirs(locale_files_folder)
copy_tree(
os.path.join(
locale_files_folders,
os.path.basename(
os.path.normpath(
locale_files_folder))),
locale_files_folder)
chmod_tree(locale_files_folder)
print(
f"Locale Files Restored into '{locale_files_folder}'."
)
call_command('collectstatic', interactive=False)
# Cleanup DB
try:
db_name = settings.DATABASES['default']['NAME']
db_user = settings.DATABASES['default']['USER']
db_port = settings.DATABASES['default']['PORT']
db_host = settings.DATABASES['default']['HOST']
db_passwd = settings.DATABASES['default'][
'PASSWORD']
utils.cleanup_db(db_name, db_user, db_port,
db_host, db_passwd)
except Exception:
traceback.print_exc()
# store backup info
restored_backup = RestoredBackup(
name=backup_file.rsplit('/', 1)[-1],
archive_md5=backup_md5,
creation_date=datetime.fromtimestamp(
os.path.getmtime(backup_file)))
restored_backup.save()
except Exception as exception:
if notify:
restore_notification.apply_async(
(admin_emails, backup_file, backup_md5,
str(exception)))
finally:
call_command('makemigrations', interactive=False)
call_command('migrate', interactive=False, fake=True)
call_command('sync_geonode_datasets',
updatepermissions=True,
ignore_errors=True)
if notify:
restore_notification.apply_async(
(admin_emails, backup_file, backup_md5))
print(
"HINT: If you migrated from another site, do not forget to run the command 'migrate_baseurl' to fix Links"
) # noqa
print(
" e.g.: DJANGO_SETTINGS_MODULE=my_geonode.settings python manage.py migrate_baseurl "
"--source-address=my-host-dev.geonode.org --target-address=my-host-prod.geonode.org"
)
print("Restore finished.")
finally:
shutil.rmtree(restore_folder)
def backup_full():
"""Full Backup of GeoNode DB"""
try:
# Create Target Folder
dir_time_suffix = get_dir_time_suffix()
target_folder = os.path.join('backup', dir_time_suffix)
if not os.path.exists(target_folder):
os.makedirs(target_folder)
# Dump Fixtures
for app_name, dump_name in zip(helpers.app_names, helpers.dump_names):
print "Dumping '" + app_name + "' into '" + dump_name + ".json'."
output = open(os.path.join(target_folder, dump_name + '.json'),
'w') # Point stdout at a file for dumping data to.
call_command('dumpdata',
app_name,
format='json',
indent=2,
natural=True,
stdout=output)
output.close()
# Store Media Root
media_root = settings.MEDIA_ROOT
media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)
if not os.path.exists(media_folder):
os.makedirs(media_folder)
copy_tree(media_root, media_folder)
print "Saved Media Files from '" + media_root + "'."
# Store Static Root
static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
if not os.path.exists(static_folder):
os.makedirs(static_folder)
copy_tree(static_root, static_folder)
print "Saved Static Root from '" + static_root + "'."
# Store Static Folders
static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder,
helpers.STATICFILES_DIRS)
if not os.path.exists(static_files_folders):
os.makedirs(static_files_folders)
for static_files_folder in static_folders:
static_folder = os.path.join(
static_files_folders,
os.path.basename(os.path.normpath(static_files_folder)))
if not os.path.exists(static_folder):
os.makedirs(static_folder)
copy_tree(static_files_folder, static_folder)
print "Saved Static Files from '" + static_files_folder + "'."
# Store Template Folders
template_folders = settings.TEMPLATE_DIRS
template_files_folders = os.path.join(target_folder,
helpers.TEMPLATE_DIRS)
if not os.path.exists(template_files_folders):
os.makedirs(template_files_folders)
for template_files_folder in template_folders:
template_folder = os.path.join(
template_files_folders,
os.path.basename(os.path.normpath(template_files_folder)))
if not os.path.exists(template_folder):
os.makedirs(template_folder)
copy_tree(template_files_folder, template_folder)
print "Saved Template Files from '" + template_files_folder + "'."
# Store Locale Folders
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder,
helpers.LOCALE_PATHS)
if not os.path.exists(locale_files_folders):
os.makedirs(locale_files_folders)
for locale_files_folder in locale_folders:
locale_folder = os.path.join(
locale_files_folders,
os.path.basename(os.path.normpath(locale_files_folder)))
if not os.path.exists(locale_folder):
os.makedirs(locale_folder)
copy_tree(locale_files_folder, locale_folder)
print "Saved Locale Files from '" + locale_files_folder + "'."
# Create Final ZIP Archive
zip_dir(target_folder, os.path.join('backup',
dir_time_suffix + '.zip'))
# Cleanup Temp Folder
shutil.rmtree(target_folder)
print "Backup Finished. Archive generated '" + os.path.join(
'backup', dir_time_suffix + '.zip') + "'."
except Exception, err:
pass
def handle(self, **options):
# ignore_errors = options.get('ignore_errors')
config = Config(options)
force_exec = options.get('force_exec')
backup_file = options.get('backup_file')
skip_geoserver = options.get('skip_geoserver')
backup_dir = options.get('backup_dir')
if not any([backup_file, backup_dir]):
raise CommandError("Mandatory option (--backup-file|--backup-dir)")
if all([backup_file, backup_dir]):
raise CommandError("Exclusive option (--backup-file|--backup-dir)")
if backup_file and not os.path.isfile(backup_file):
raise CommandError("Provided '--backup-file' is not a file")
if backup_dir and not os.path.isdir(backup_dir):
raise CommandError("Provided '--backup-dir' is not a directory")
print "Before proceeding with the Restore, please ensure that:"
print " 1. The backend (DB or whatever) is accessible and you have rights"
print " 2. The GeoServer is up and running and reachable from this machine"
message = 'WARNING: The restore will overwrite ALL GeoNode data. You want to proceed?'
if force_exec or helpers.confirm(prompt=message, resp=False):
target_folder = backup_dir
if backup_file:
# Create Target Folder
restore_folder = os.path.join(tempfile.gettempdir(), 'restore')
if not os.path.exists(restore_folder):
os.makedirs(restore_folder)
# Extract ZIP Archive to Target Folder
target_folder = extract_archive(backup_file, restore_folder)
# Write Checks
media_root = settings.MEDIA_ROOT
media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)
static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS)
template_folders = settings.TEMPLATE_DIRS
template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS)
try:
print("[Sanity Check] Full Write Access to '{}' ...".format(media_root))
chmod_tree(media_root)
print("[Sanity Check] Full Write Access to '{}' ...".format(static_root))
chmod_tree(static_root)
for static_files_folder in static_folders:
print("[Sanity Check] Full Write Access to '{}' ...".format(static_files_folder))
chmod_tree(static_files_folder)
for template_files_folder in template_folders:
print("[Sanity Check] Full Write Access to '{}' ...".format(template_files_folder))
chmod_tree(template_files_folder)
for locale_files_folder in locale_folders:
print("[Sanity Check] Full Write Access to '{}' ...".format(locale_files_folder))
chmod_tree(locale_files_folder)
except:
print("...Sanity Checks on Folder failed. Please make sure that the current user has full WRITE access to the above folders (and sub-folders or files).")
print("Reason:")
raise
if not skip_geoserver:
self.restore_geoserver_backup(settings, target_folder)
self.restore_geoserver_raster_data(config, settings, target_folder)
self.restore_geoserver_vector_data(config, settings, target_folder)
print("Restoring geoserver external resources")
self.restore_geoserver_externals(config, settings, target_folder)
else:
print("Skipping geoserver backup restore")
# Prepare Target DB
try:
call_command('migrate', interactive=False, load_initial_data=False)
db_name = settings.DATABASES['default']['NAME']
db_user = settings.DATABASES['default']['USER']
db_port = settings.DATABASES['default']['PORT']
db_host = settings.DATABASES['default']['HOST']
db_passwd = settings.DATABASES['default']['PASSWORD']
helpers.patch_db(db_name, db_user, db_port, db_host, db_passwd, settings.MONITORING_ENABLED)
except:
traceback.print_exc()
try:
# Deactivate GeoNode Signals
print "Deactivating GeoNode Signals..."
designals()
print "...done!"
# Flush DB
try:
db_name = settings.DATABASES['default']['NAME']
db_user = settings.DATABASES['default']['USER']
db_port = settings.DATABASES['default']['PORT']
db_host = settings.DATABASES['default']['HOST']
db_passwd = settings.DATABASES['default']['PASSWORD']
helpers.flush_db(db_name, db_user, db_port, db_host, db_passwd)
except:
try:
call_command('flush', interactive=False, load_initial_data=False)
except:
traceback.print_exc()
raise
# Restore Fixtures
for app_name, dump_name in zip(config.app_names, config.dump_names):
fixture_file = os.path.join(target_folder, dump_name+'.json')
print "Deserializing "+fixture_file
try:
call_command('loaddata', fixture_file, app_label=app_name)
except:
traceback.print_exc()
print "WARNING: No valid fixture data found for '"+dump_name+"'."
# helpers.load_fixture(app_name, fixture_file)
raise
# Restore Media Root
try:
shutil.rmtree(media_root)
except:
pass
if not os.path.exists(media_root):
os.makedirs(media_root)
copy_tree(media_folder, media_root)
chmod_tree(media_root)
print "Media Files Restored into '"+media_root+"'."
# Restore Static Root
try:
shutil.rmtree(static_root)
except:
pass
if not os.path.exists(static_root):
os.makedirs(static_root)
copy_tree(static_folder, static_root)
chmod_tree(static_root)
print "Static Root Restored into '"+static_root+"'."
# Restore Static Root
try:
shutil.rmtree(static_root)
except:
pass
if not os.path.exists(static_root):
os.makedirs(static_root)
copy_tree(static_folder, static_root)
chmod_tree(static_root)
print "Static Root Restored into '"+static_root+"'."
# Restore Static Folders
for static_files_folder in static_folders:
try:
shutil.rmtree(static_files_folder)
except:
pass
if not os.path.exists(static_files_folder):
os.makedirs(static_files_folder)
copy_tree(os.path.join(static_files_folders,
os.path.basename(os.path.normpath(static_files_folder))),
static_files_folder)
chmod_tree(static_files_folder)
print "Static Files Restored into '"+static_files_folder+"'."
# Restore Template Folders
for template_files_folder in template_folders:
try:
shutil.rmtree(template_files_folder)
except:
pass
if not os.path.exists(template_files_folder):
os.makedirs(template_files_folder)
copy_tree(os.path.join(template_files_folders,
os.path.basename(os.path.normpath(template_files_folder))),
template_files_folder)
chmod_tree(template_files_folder)
print "Template Files Restored into '"+template_files_folder+"'."
# Restore Locale Folders
for locale_files_folder in locale_folders:
try:
shutil.rmtree(locale_files_folder)
except:
pass
if not os.path.exists(locale_files_folder):
os.makedirs(locale_files_folder)
copy_tree(os.path.join(locale_files_folders,
os.path.basename(os.path.normpath(locale_files_folder))),
locale_files_folder)
chmod_tree(locale_files_folder)
print "Locale Files Restored into '"+locale_files_folder+"'."
call_command('collectstatic', interactive=False)
# Cleanup DB
try:
db_name = settings.DATABASES['default']['NAME']
db_user = settings.DATABASES['default']['USER']
db_port = settings.DATABASES['default']['PORT']
db_host = settings.DATABASES['default']['HOST']
db_passwd = settings.DATABASES['default']['PASSWORD']
helpers.cleanup_db(db_name, db_user, db_port, db_host, db_passwd)
except:
traceback.print_exc()
return str(target_folder)
finally:
# Reactivate GeoNode Signals
print "Reactivating GeoNode Signals..."
resignals()
print "...done!"
call_command('migrate', interactive=False, load_initial_data=False, fake=True)
print "HINT: If you migrated from another site, do not forget to run the command 'migrate_baseurl' to fix Links"
print " e.g.: DJANGO_SETTINGS_MODULE=my_geonode.settings python manage.py migrate_baseurl --source-address=my-host-dev.geonode.org --target-address=my-host-prod.geonode.org"
print "Restore finished. Please find restored files and dumps into:"
def execute_backup(self, **options):
ignore_errors = options.get('ignore_errors')
config = utils.Config(options)
force_exec = options.get('force_exec')
backup_dir = options.get('backup_dir')
skip_geoserver = options.get('skip_geoserver')
if not backup_dir or len(backup_dir) == 0:
raise CommandError(
"Destination folder '--backup-dir' is mandatory")
print("Before proceeding with the Backup, please ensure that:")
print(
" 1. The backend (DB or whatever) is accessible and you have rights"
)
print(
" 2. The GeoServer is up and running and reachable from this machine"
)
message = 'You want to proceed?'
if force_exec or utils.confirm(prompt=message, resp=False):
# Create Target Folder
dir_time_suffix = get_dir_time_suffix()
target_folder = os.path.join(backup_dir, dir_time_suffix)
if not os.path.exists(target_folder):
os.makedirs(target_folder)
# Temporary folder to store backup files. It will be deleted at the end.
os.chmod(target_folder, 0o777)
if not skip_geoserver:
self.create_geoserver_backup(config, settings, target_folder,
ignore_errors)
self.dump_geoserver_raster_data(config, settings,
target_folder)
self.dump_geoserver_vector_data(config, settings,
target_folder)
logger.info("Dumping geoserver external resources")
self.dump_geoserver_externals(config, settings, target_folder)
else:
print("Skipping geoserver backup")
# Deactivate GeoNode Signals
with DisableDjangoSignals():
# Dump Fixtures
for app_name, dump_name in zip(config.app_names,
config.dump_names):
# prevent dumping BackupRestore application
if app_name == 'br':
continue
logger.info(
f"Dumping '{app_name}' into '{dump_name}.json'.")
# Point stdout at a file for dumping data to.
with open(os.path.join(target_folder, f'{dump_name}.json'),
'w') as output:
call_command('dumpdata',
app_name,
format='json',
indent=2,
stdout=output)
# Store Media Root
media_root = settings.MEDIA_ROOT
media_folder = os.path.join(target_folder, utils.MEDIA_ROOT)
if not os.path.exists(media_folder):
os.makedirs(media_folder)
copy_tree(
media_root,
media_folder,
ignore=utils.ignore_time(config.gs_data_dt_filter[0],
config.gs_data_dt_filter[1]))
print(f"Saved Media Files from '{media_root}'.")
# Store Static Root
static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, utils.STATIC_ROOT)
if not os.path.exists(static_folder):
os.makedirs(static_folder)
copy_tree(
static_root,
static_folder,
ignore=utils.ignore_time(config.gs_data_dt_filter[0],
config.gs_data_dt_filter[1]))
print(f"Saved Static Root from '{static_root}'.")
# Store Static Folders
static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder,
utils.STATICFILES_DIRS)
if not os.path.exists(static_files_folders):
os.makedirs(static_files_folders)
for static_files_folder in static_folders:
# skip dumping of static files of apps not located under LOCAL_ROOT path
# (check to prevent saving files from site-packages in project-template based GeoNode projects)
if getattr(settings, 'LOCAL_ROOT', None) and \
not static_files_folder.startswith(settings.LOCAL_ROOT):
print(
f"Skipping static directory: {static_files_folder}. "
f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}."
)
continue
static_folder = os.path.join(
static_files_folders,
os.path.basename(
os.path.normpath(static_files_folder)))
if not os.path.exists(static_folder):
os.makedirs(static_folder)
copy_tree(static_files_folder,
static_folder,
ignore=utils.ignore_time(
config.gs_data_dt_filter[0],
config.gs_data_dt_filter[1]))
print(f"Saved Static Files from '{static_files_folder}'.")
# Store Template Folders
template_folders = []
try:
template_folders = settings.TEMPLATE_DIRS
except Exception:
try:
template_folders = settings.TEMPLATES[0]['DIRS']
except Exception:
pass
template_files_folders = os.path.join(target_folder,
utils.TEMPLATE_DIRS)
if not os.path.exists(template_files_folders):
os.makedirs(template_files_folders)
for template_files_folder in template_folders:
# skip dumping of template files of apps not located under LOCAL_ROOT path
# (check to prevent saving files from site-packages in project-template based GeoNode projects)
if getattr(settings, 'LOCAL_ROOT', None) and \
not template_files_folder.startswith(settings.LOCAL_ROOT):
print(
f"Skipping template directory: {template_files_folder}. "
f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}."
)
continue
template_folder = os.path.join(
template_files_folders,
os.path.basename(
os.path.normpath(template_files_folder)))
if not os.path.exists(template_folder):
os.makedirs(template_folder)
copy_tree(template_files_folder,
template_folder,
ignore=utils.ignore_time(
config.gs_data_dt_filter[0],
config.gs_data_dt_filter[1]))
print(
f"Saved Template Files from '{template_files_folder}'."
)
# Store Locale Folders
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder,
utils.LOCALE_PATHS)
if not os.path.exists(locale_files_folders):
os.makedirs(locale_files_folders)
for locale_files_folder in locale_folders:
# skip dumping of locale files of apps not located under LOCAL_ROOT path
# (check to prevent saving files from site-packages in project-template based GeoNode projects)
if getattr(settings, 'LOCAL_ROOT', None) and \
not locale_files_folder.startswith(settings.LOCAL_ROOT):
logger.info(
f"Skipping locale directory: {locale_files_folder}. "
f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}."
)
continue
locale_folder = os.path.join(
locale_files_folders,
os.path.basename(
os.path.normpath(locale_files_folder)))
if not os.path.exists(locale_folder):
os.makedirs(locale_folder)
copy_tree(locale_files_folder,
locale_folder,
ignore=utils.ignore_time(
config.gs_data_dt_filter[0],
config.gs_data_dt_filter[1]))
logger.info(
f"Saved Locale Files from '{locale_files_folder}'.")
# Create Final ZIP Archive
backup_archive = os.path.join(backup_dir,
f'{dir_time_suffix}.zip')
zip_dir(target_folder, backup_archive)
# Generate a md5 hash of a backup archive and save it
backup_md5_file = os.path.join(backup_dir,
f'{dir_time_suffix}.md5')
zip_archive_md5 = utils.md5_file_hash(backup_archive)
with open(backup_md5_file, 'w') as md5_file:
md5_file.write(zip_archive_md5)
# Generate the ini file with the current settings used by the backup command
backup_ini_file = os.path.join(backup_dir,
f'{dir_time_suffix}.ini')
with open(backup_ini_file, 'w') as configfile:
config.config_parser.write(configfile)
# Clean-up Temp Folder
try:
shutil.rmtree(target_folder)
except Exception:
logger.warning(
f"WARNING: Could not be possible to delete the temp folder: '{target_folder}'"
)
print("Backup Finished. Archive generated.")
return str(os.path.join(backup_dir, f'{dir_time_suffix}.zip'))
def handle(self, **options):
# ignore_errors = options.get('ignore_errors')
config = Config(options)
force_exec = options.get('force_exec')
backup_file = options.get('backup_file')
skip_geoserver = options.get('skip_geoserver')
backup_dir = options.get('backup_dir')
if not any([backup_file, backup_dir]):
raise CommandError("Mandatory option (--backup-file|--backup-dir)")
if all([backup_file, backup_dir]):
raise CommandError("Exclusive option (--backup-file|--backup-dir)")
if backup_file and not os.path.isfile(backup_file):
raise CommandError("Provided '--backup-file' is not a file")
if backup_dir and not os.path.isdir(backup_dir):
raise CommandError("Provided '--backup-dir' is not a directory")
print "Before proceeding with the Restore, please ensure that:"
print " 1. The backend (DB or whatever) is accessible and you have rights"
print " 2. The GeoServer is up and running and reachable from this machine"
message = 'WARNING: The restore will overwrite ALL GeoNode data. You want to proceed?'
if force_exec or helpers.confirm(prompt=message, resp=False):
target_folder = backup_dir
if backup_file:
# Create Target Folder
restore_folder = os.path.join(tempfile.gettempdir(), 'restore')
if not os.path.exists(restore_folder):
os.makedirs(restore_folder)
# Extract ZIP Archive to Target Folder
target_folder = extract_archive(backup_file, restore_folder)
# Write Checks
media_root = settings.MEDIA_ROOT
media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)
static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS)
template_folders = []
try:
template_folders = settings.TEMPLATE_DIRS
except:
try:
template_folders = settings.TEMPLATES[0]['DIRS']
except:
pass
template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS)
try:
print("[Sanity Check] Full Write Access to '{}' ...".format(media_root))
chmod_tree(media_root)
print("[Sanity Check] Full Write Access to '{}' ...".format(static_root))
chmod_tree(static_root)
for static_files_folder in static_folders:
print("[Sanity Check] Full Write Access to '{}' ...".format(static_files_folder))
chmod_tree(static_files_folder)
for template_files_folder in template_folders:
print("[Sanity Check] Full Write Access to '{}' ...".format(template_files_folder))
chmod_tree(template_files_folder)
for locale_files_folder in locale_folders:
print("[Sanity Check] Full Write Access to '{}' ...".format(locale_files_folder))
chmod_tree(locale_files_folder)
except:
print("...Sanity Checks on Folder failed. Please make sure that the current user has full WRITE access to the above folders (and sub-folders or files).")
print("Reason:")
raise
if not skip_geoserver:
self.restore_geoserver_backup(settings, target_folder)
self.restore_geoserver_raster_data(config, settings, target_folder)
self.restore_geoserver_vector_data(config, settings, target_folder)
print("Restoring geoserver external resources")
self.restore_geoserver_externals(config, settings, target_folder)
else:
print("Skipping geoserver backup restore")
# Prepare Target DB
try:
call_command('migrate', interactive=False, load_initial_data=False)
db_name = settings.DATABASES['default']['NAME']
db_user = settings.DATABASES['default']['USER']
db_port = settings.DATABASES['default']['PORT']
db_host = settings.DATABASES['default']['HOST']
db_passwd = settings.DATABASES['default']['PASSWORD']
helpers.patch_db(db_name, db_user, db_port, db_host, db_passwd, settings.MONITORING_ENABLED)
except:
traceback.print_exc()
try:
# Deactivate GeoNode Signals
print "Deactivating GeoNode Signals..."
designals()
print "...done!"
# Flush DB
try:
db_name = settings.DATABASES['default']['NAME']
db_user = settings.DATABASES['default']['USER']
db_port = settings.DATABASES['default']['PORT']
db_host = settings.DATABASES['default']['HOST']
db_passwd = settings.DATABASES['default']['PASSWORD']
helpers.flush_db(db_name, db_user, db_port, db_host, db_passwd)
except:
try:
call_command('flush', interactive=False, load_initial_data=False)
except:
traceback.print_exc()
raise
# Restore Fixtures
for app_name, dump_name in zip(config.app_names, config.dump_names):
fixture_file = os.path.join(target_folder, dump_name+'.json')
print "Deserializing "+fixture_file
try:
call_command('loaddata', fixture_file, app_label=app_name)
except:
traceback.print_exc()
print "WARNING: No valid fixture data found for '"+dump_name+"'."
# helpers.load_fixture(app_name, fixture_file)
raise
# Restore Media Root
try:
shutil.rmtree(media_root)
except:
pass
if not os.path.exists(media_root):
os.makedirs(media_root)
copy_tree(media_folder, media_root)
chmod_tree(media_root)
print "Media Files Restored into '"+media_root+"'."
# Restore Static Root
try:
shutil.rmtree(static_root)
except:
pass
if not os.path.exists(static_root):
os.makedirs(static_root)
copy_tree(static_folder, static_root)
chmod_tree(static_root)
print "Static Root Restored into '"+static_root+"'."
# Restore Static Root
try:
shutil.rmtree(static_root)
except:
pass
if not os.path.exists(static_root):
os.makedirs(static_root)
copy_tree(static_folder, static_root)
chmod_tree(static_root)
print "Static Root Restored into '"+static_root+"'."
# Restore Static Folders
for static_files_folder in static_folders:
try:
shutil.rmtree(static_files_folder)
except:
pass
if not os.path.exists(static_files_folder):
os.makedirs(static_files_folder)
copy_tree(os.path.join(static_files_folders,
os.path.basename(os.path.normpath(static_files_folder))),
static_files_folder)
chmod_tree(static_files_folder)
print "Static Files Restored into '"+static_files_folder+"'."
# Restore Template Folders
for template_files_folder in template_folders:
try:
shutil.rmtree(template_files_folder)
except:
pass
if not os.path.exists(template_files_folder):
os.makedirs(template_files_folder)
copy_tree(os.path.join(template_files_folders,
os.path.basename(os.path.normpath(template_files_folder))),
template_files_folder)
chmod_tree(template_files_folder)
print "Template Files Restored into '"+template_files_folder+"'."
# Restore Locale Folders
for locale_files_folder in locale_folders:
try:
shutil.rmtree(locale_files_folder)
except:
pass
if not os.path.exists(locale_files_folder):
os.makedirs(locale_files_folder)
copy_tree(os.path.join(locale_files_folders,
os.path.basename(os.path.normpath(locale_files_folder))),
locale_files_folder)
chmod_tree(locale_files_folder)
print "Locale Files Restored into '"+locale_files_folder+"'."
call_command('collectstatic', interactive=False)
# Cleanup DB
try:
db_name = settings.DATABASES['default']['NAME']
db_user = settings.DATABASES['default']['USER']
db_port = settings.DATABASES['default']['PORT']
db_host = settings.DATABASES['default']['HOST']
db_passwd = settings.DATABASES['default']['PASSWORD']
helpers.cleanup_db(db_name, db_user, db_port, db_host, db_passwd)
except:
traceback.print_exc()
return str(target_folder)
finally:
# Reactivate GeoNode Signals
print "Reactivating GeoNode Signals..."
resignals()
print "...done!"
call_command('migrate', interactive=False, load_initial_data=False, fake=True)
print "HINT: If you migrated from another site, do not forget to run the command 'migrate_baseurl' to fix Links"
print " e.g.: DJANGO_SETTINGS_MODULE=my_geonode.settings python manage.py migrate_baseurl --source-address=my-host-dev.geonode.org --target-address=my-host-prod.geonode.org"
print "Restore finished. Please find restored files and dumps into:"
def backup_full():
"""Full Backup of GeoNode DB"""
try:
# Create Target Folder
dir_time_suffix = get_dir_time_suffix()
target_folder = os.path.join('backup', dir_time_suffix)
if not os.path.exists(target_folder):
os.makedirs(target_folder)
# Dump Fixtures
for app_name, dump_name in zip(helpers.app_names, helpers.dump_names):
print "Dumping '"+app_name+"' into '"+dump_name+".json'."
output = open(os.path.join(target_folder,dump_name+'.json'),'w') # Point stdout at a file for dumping data to.
call_command('dumpdata',app_name,format='json',indent=2,natural=True,stdout=output)
output.close()
# Store Media Root
media_root = settings.MEDIA_ROOT
media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)
if not os.path.exists(media_folder):
os.makedirs(media_folder)
copy_tree(media_root, media_folder)
print "Saved Media Files from '"+media_root+"'."
# Store Static Root
static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
if not os.path.exists(static_folder):
os.makedirs(static_folder)
copy_tree(static_root, static_folder)
print "Saved Static Root from '"+static_root+"'."
# Store Static Folders
static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS)
if not os.path.exists(static_files_folders):
os.makedirs(static_files_folders)
for static_files_folder in static_folders:
static_folder = os.path.join(static_files_folders, os.path.basename(os.path.normpath(static_files_folder)))
if not os.path.exists(static_folder):
os.makedirs(static_folder)
copy_tree(static_files_folder, static_folder)
print "Saved Static Files from '"+static_files_folder+"'."
# Store Template Folders
template_folders = []
try:
template_folders = settings.TEMPLATE_DIRS
except:
try:
template_folders = settings.TEMPLATES[0]['DIRS']
except:
pass
template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)
if not os.path.exists(template_files_folders):
os.makedirs(template_files_folders)
for template_files_folder in template_folders:
template_folder = os.path.join(template_files_folders, os.path.basename(os.path.normpath(template_files_folder)))
if not os.path.exists(template_folder):
os.makedirs(template_folder)
copy_tree(template_files_folder, template_folder)
print "Saved Template Files from '"+template_files_folder+"'."
# Store Locale Folders
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS)
if not os.path.exists(locale_files_folders):
os.makedirs(locale_files_folders)
for locale_files_folder in locale_folders:
locale_folder = os.path.join(locale_files_folders, os.path.basename(os.path.normpath(locale_files_folder)))
if not os.path.exists(locale_folder):
os.makedirs(locale_folder)
copy_tree(locale_files_folder, locale_folder)
print "Saved Locale Files from '"+locale_files_folder+"'."
# Create Final ZIP Archive
zip_dir(target_folder, os.path.join('backup', dir_time_suffix+'.zip'))
# Cleanup Temp Folder
shutil.rmtree(target_folder)
print "Backup Finished. Archive generated '"+os.path.join('backup', dir_time_suffix+'.zip')+"'."
except Exception, err:
pass
def restore_full(archive):
"""Full Restore of GeoNode DB"""
try:
# Create Target Folder
restore_folder = 'restore'
if not os.path.exists(restore_folder):
os.makedirs(restore_folder)
# Extract ZIP Archive to Target Folder
target_folder = extract_archive(archive, restore_folder)
# Prepare Target DB
try:
call_command('migrate', interactive=False, load_initial_data=False)
call_command('flush', interactive=False, load_initial_data=False)
helpers.patch_db()
except:
traceback.print_exc()
# Restore Fixtures
for app_name, dump_name in zip(helpers.app_names, helpers.dump_names):
fixture_file = os.path.join(target_folder, dump_name+'.json')
print "Deserializing "+fixture_file
try:
call_command('loaddata', fixture_file, app_label=app_name)
except:
#traceback.print_exc()
print "WARNING: No valid fixture data found for '"+dump_name+"'."
#helpers.load_fixture(app_name, fixture_file)
# Restore Media Root
media_root = settings.MEDIA_ROOT
media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)
try:
shutil.rmtree(media_root)
except:
pass
if not os.path.exists(media_root):
os.makedirs(media_root)
copy_tree(media_folder, media_root)
chmod_tree(media_root)
print "Media Files Restored into '"+media_root+"'."
# Restore Static Root
static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
try:
shutil.rmtree(static_root)
except:
pass
if not os.path.exists(static_root):
os.makedirs(static_root)
copy_tree(static_folder, static_root)
chmod_tree(static_root)
print "Static Root Restored into '"+static_root+"'."
# Restore Static Folders
static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS)
for static_files_folder in static_folders:
try:
shutil.rmtree(static_files_folder)
except:
pass
if not os.path.exists(static_files_folder):
os.makedirs(static_files_folder)
copy_tree(os.path.join(static_files_folders, os.path.basename(os.path.normpath(static_files_folder))), static_files_folder)
chmod_tree(static_files_folder)
print "Static Files Restored into '"+static_files_folder+"'."
# Restore Template Folders
template_folders = []
try:
template_folders = settings.TEMPLATE_DIRS
except:
try:
template_folders = settings.TEMPLATES[0]['DIRS']
except:
pass
template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)
for template_files_folder in template_folders:
try:
shutil.rmtree(template_files_folder)
except:
pass
if not os.path.exists(template_files_folder):
os.makedirs(template_files_folder)
copy_tree(os.path.join(template_files_folders, os.path.basename(os.path.normpath(template_files_folder))), template_files_folder)
chmod_tree(template_files_folder)
print "Template Files Restored into '"+template_files_folder+"'."
# Restore Locale Folders
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS)
for locale_files_folder in locale_folders:
try:
shutil.rmtree(locale_files_folder)
except:
pass
if not os.path.exists(locale_files_folder):
os.makedirs(locale_files_folder)
copy_tree(os.path.join(locale_files_folders, os.path.basename(os.path.normpath(locale_files_folder))), locale_files_folder)
chmod_tree(locale_files_folder)
print "Locale Files Restored into '"+locale_files_folder+"'."
# Cleanup DB
try:
helpers.cleanup_db()
except:
traceback.print_exc()
except Exception, err:
print str(err)