def runner(cls, api, parsed_args):
cls.log_info('Starting backup: vManage URL: "%s" > Local workdir: "%s"', api.base_url, parsed_args.workdir)
# Backup workdir must be empty for a new backup
saved_workdir = cls.clean_workdir(parsed_args.workdir)
if saved_workdir:
cls.log_info('Previous backup under "%s" was saved as "%s"', parsed_args.workdir, saved_workdir)
target_info = ServerInfo(server_version=api.server_version)
if target_info.save(parsed_args.workdir):
cls.log_info('Saved vManage server information')
if CATALOG_TAG_ALL in parsed_args.tags:
# Items without index files to be included with tag 'all'
edge_certs = EdgeCertificate.get(api)
if edge_certs is None:
cls.log_error('Failed backup WAN edge certificates')
elif edge_certs.save(parsed_args.workdir):
cls.log_info('Saved WAN edge certificates')
for _, info, index_cls, item_cls in catalog_entries(*parsed_args.tags):
item_index = index_cls.get(api)
if item_index is None:
cls.log_debug('Skipped %s, item not supported by this vManage', info)
continue
if item_index.save(parsed_args.workdir):
cls.log_info('Saved %s index', info)
matched_item_iter = (
(item_id, item_name) for item_id, item_name in item_index
if parsed_args.regex is None or regex_search(parsed_args.regex, item_name)
)
for item_id, item_name in matched_item_iter:
item = item_cls.get(api, item_id)
if item is None:
cls.log_error('Failed backup %s %s', info, item_name)
continue
if item.save(parsed_args.workdir, item_index.need_extended_name, item_name, item_id):
cls.log_info('Done %s %s', info, item_name)
# Special case for DeviceTemplateAttached and DeviceTemplateValues
if isinstance(item, DeviceTemplate):
devices_attached = DeviceTemplateAttached.get(api, item_id)
if devices_attached is None:
cls.log_error('Failed backup %s %s attached devices', info, item_name)
continue
if devices_attached.save(parsed_args.workdir, item_index.need_extended_name, item_name, item_id):
cls.log_info('Done %s %s attached devices', info, item_name)
else:
cls.log_debug('Skipped %s %s attached devices, none found', info, item_name)
continue
try:
uuid_list = [uuid for uuid, _ in devices_attached]
values = DeviceTemplateValues(api.post(DeviceTemplateValues.api_params(item_id, uuid_list),
DeviceTemplateValues.api_path.post))
if values.save(parsed_args.workdir, item_index.need_extended_name, item_name, item_id):
cls.log_info('Done %s %s values', info, item_name)
except RestAPIException as ex:
cls.log_error('Failed backup %s %s values: %s', info, item_name, ex)
def runner(cls, api, parsed_args):
def load_items(index, item_cls):
item_iter = ((item_id,
item_cls.load(parsed_args.workdir,
index.need_extended_name, item_name,
item_id))
for item_id, item_name in index)
return ((item_id, item_obj) for item_id, item_obj in item_iter
if item_obj is not None)
cls.log_info(
'Starting restore%s: Local workdir: "%s" > vManage URL: "%s"',
', DRY-RUN mode' if parsed_args.dryrun else '',
parsed_args.workdir, api.base_url)
local_info = ServerInfo.load(parsed_args.workdir)
# Server info file may not be present (e.g. backup from older Sastre releases)
if local_info is not None and is_version_newer(
api.server_version, local_info.server_version):
cls.log_warning(
'Target vManage release (%s) is older than the release used in backup (%s). '
+
'Items may fail to be restored due to incompatibilities across releases.',
api.server_version, local_info.server_version)
cls.log_info('Loading existing items from target vManage')
target_all_items_map = {
hash(type(index)):
{item_name: item_id
for item_id, item_name in index}
for _, _, index, item_cls in cls.index_iter(
api, catalog_entries(CATALOG_TAG_ALL))
}
cls.log_info('Identifying items to be pushed')
id_mapping = {
} # {<old_id>: <new_id>}, used to replace old (saved) item ids with new (target) ids
restore_list = [
] # [ (<info>, <index_cls>, [(<item_id>, <item>, <id_on_target>), ...]), ...]
dependency_set = set() # {<item_id>, ...}
match_set = set() # {<item_id>, ...}
for tag in ordered_tags(parsed_args.tag):
cls.log_info('Inspecting %s items', tag)
tag_iter = ((info, index, load_items(index, item_cls))
for _, info, index, item_cls in cls.index_iter(
parsed_args.workdir, catalog_entries(tag)))
for info, index, loaded_items_iter in tag_iter:
target_item_map = target_all_items_map.get(hash(type(index)))
if target_item_map is None:
# Logging at warning level because the backup files did have this item
cls.log_warning(
'Will skip %s, item not supported by target vManage',
info)
continue
restore_item_list = []
for item_id, item in loaded_items_iter:
target_id = target_item_map.get(item.name)
if target_id is not None:
# Item already exists on target vManage, record item id from target
if item_id != target_id:
id_mapping[item_id] = target_id
if not parsed_args.force:
# Existing item on target vManage will be used, i.e. will not overwrite it
cls.log_debug(
'Will skip %s %s, item already on target vManage',
info, item.name)
continue
if item.is_readonly:
cls.log_debug('Will skip read-only %s %s', info,
item.name)
continue
item_matches = ((parsed_args.tag == CATALOG_TAG_ALL
or parsed_args.tag == tag) and
(parsed_args.regex is None or regex_search(
parsed_args.regex, item.name)))
if item_matches:
match_set.add(item_id)
if item_matches or item_id in dependency_set:
# A target_id that is not None signals a push operation, as opposed to post.
# target_id will be None unless --force is specified and item name is on target
restore_item_list.append((item_id, item, target_id))
dependency_set.update(item.id_references_set)
if len(restore_item_list) > 0:
restore_list.append((info, index, restore_item_list))
log_prefix = 'DRY-RUN: ' if parsed_args.dryrun else ''
if len(restore_list) > 0:
cls.log_info('%sPushing items to vManage', log_prefix)
# Items were added to restore_list following ordered_tags() order (i.e. higher level items before lower
# level items). The reverse order needs to be followed on restore.
for info, index, restore_item_list in reversed(restore_list):
pushed_item_dict = {}
for item_id, item, target_id in restore_item_list:
op_info = 'Create' if target_id is None else 'Update'
reason = ' (dependency)' if item_id in dependency_set - match_set else ''
try:
if target_id is None:
# Create new item
if parsed_args.dryrun:
cls.log_info('%s%s %s %s%s', log_prefix,
op_info, info, item.name, reason)
continue
# Not using item id returned from post because post can return empty (e.g. local policies)
api.post(item.post_data(id_mapping),
item.api_path.post)
pushed_item_dict[item.name] = item_id
else:
# Update existing item
update_data = item.put_data(id_mapping)
if item.get_raise(api,
target_id).is_equal(update_data):
cls.log_debug('%s%s skipped (no diffs) %s %s',
log_prefix, op_info, info,
item.name)
continue
if parsed_args.dryrun:
cls.log_info('%s%s %s %s%s', log_prefix,
op_info, info, item.name, reason)
continue
put_eval = UpdateEval(
api.put(update_data, item.api_path.put,
target_id))
if put_eval.need_reattach:
if put_eval.is_master:
cls.log_info(
'Updating %s %s requires reattach',
info, item.name)
action_list = cls.attach_template(
api, parsed_args.workdir,
index.need_extended_name,
[(item.name, item_id, target_id)])
else:
cls.log_info(
'Updating %s %s requires reattach of affected templates',
info, item.name)
target_templates = {
item_id: item_name
for item_id, item_name in
DeviceTemplateIndex.get_raise(api)
}
templates_iter = (
(target_templates[tgt_id], tgt_id)
for tgt_id in
put_eval.templates_affected_iter())
action_list = cls.reattach_template(
api, templates_iter)
cls.wait_actions(api,
action_list,
'reattaching templates',
raise_on_failure=True)
elif put_eval.need_reactivate:
cls.log_info(
'Updating %s %s requires vSmart policy reactivate',
info, item.name)
action_list = cls.activate_policy(
api,
*PolicyVsmartIndex.get_raise(
api).active_policy,
is_edited=True)
cls.wait_actions(api,
action_list,
'reactivating vSmart policy',
raise_on_failure=True)
except (RestAPIException, WaitActionsException) as ex:
cls.log_error('Failed %s %s %s%s: %s', op_info, info,
item.name, reason, ex)
else:
cls.log_info('Done: %s %s %s%s', op_info, info,
item.name, reason)
# Read new ids from target and update id_mapping
try:
new_target_item_map = {
item_name: item_id
for item_id, item_name in index.get_raise(api)
}
for item_name, old_item_id in pushed_item_dict.items():
id_mapping[old_item_id] = new_target_item_map[
item_name]
except RestAPIException as ex:
cls.log_critical('Failed retrieving %s: %s', info, ex)
break
else:
cls.log_info('%sNo items to push', log_prefix)
if parsed_args.attach:
try:
target_templates = {
item_name: item_id
for item_id, item_name in DeviceTemplateIndex.get_raise(
api)
}
target_policies = {
item_name: item_id
for item_id, item_name in PolicyVsmartIndex.get_raise(api)
}
saved_template_index = DeviceTemplateIndex.load(
parsed_args.workdir, raise_not_found=True)
attach_common_args = (api, parsed_args.workdir,
saved_template_index.need_extended_name)
# Attach WAN Edge templates
edge_templates_iter = (
(saved_name, saved_id, target_templates.get(saved_name))
for saved_id, saved_name in saved_template_index.
filtered_iter(DeviceTemplateIndex.is_not_vsmart))
wan_edge_set = {
uuid
for uuid, _ in EdgeInventory.get_raise(api)
}
action_list = cls.attach_template(*attach_common_args,
edge_templates_iter,
wan_edge_set)
if len(action_list) == 0:
cls.log_info('No WAN Edge attachments needed')
else:
cls.wait_actions(api, action_list,
'attaching WAN Edge templates')
# Attach vSmart template
vsmart_templates_iter = (
(saved_name, saved_id, target_templates.get(saved_name))
for saved_id, saved_name in saved_template_index.
filtered_iter(DeviceTemplateIndex.is_vsmart))
vsmart_set = {
uuid
for uuid, _ in ControlInventory.get_raise(
api).filtered_iter(ControlInventory.is_vsmart)
}
action_list = cls.attach_template(*attach_common_args,
vsmart_templates_iter,
vsmart_set)
if len(action_list) == 0:
cls.log_info('No vSmart attachments needed')
else:
cls.wait_actions(api, action_list,
'attaching vSmart template')
# Activate vSmart policy
_, policy_name = PolicyVsmartIndex.load(
parsed_args.workdir, raise_not_found=True).active_policy
action_list = cls.activate_policy(
api, target_policies.get(policy_name), policy_name)
if len(action_list) == 0:
cls.log_info('No vSmart policy to activate')
else:
cls.wait_actions(api, action_list,
'activating vSmart policy')
except (RestAPIException, FileNotFoundError) as ex:
cls.log_critical('Attach failed: %s', ex)
def runner(self, parsed_args, api, task_output=None):
self.log_info('Starting backup: vManage URL: "%s" -> Local workdir: "%s"', api.base_url, parsed_args.workdir)
# Backup workdir must be empty for a new backup
saved_workdir = clean_dir(parsed_args.workdir, max_saved=0 if parsed_args.no_rollover else 99)
if saved_workdir:
self.log_info('Previous backup under "%s" was saved as "%s"', parsed_args.workdir, saved_workdir)
target_info = ServerInfo(server_version=api.server_version)
if target_info.save(parsed_args.workdir):
self.log_info('Saved vManage server information')
# Backup items not registered to the catalog, but to be included when tag is 'all'
if CATALOG_TAG_ALL in parsed_args.tags:
edge_certs = EdgeCertificate.get(api)
if edge_certs is None:
self.log_error('Failed backup WAN edge certificates')
elif edge_certs.save(parsed_args.workdir):
self.log_info('Saved WAN edge certificates')
for inventory, info in ((EdgeInventory.get(api), 'WAN edge'), (ControlInventory.get(api), 'controller')):
if inventory is None:
self.log_error('Failed retrieving %s inventory', info)
continue
for uuid, _, hostname, _ in inventory.extended_iter():
if hostname is None:
self.log_debug('Skipping %s, no hostname', uuid)
continue
for item, config_type in ((DeviceConfig.get(api, DeviceConfig.api_params(uuid)), 'CFS'),
(DeviceConfigRFS.get(api, DeviceConfigRFS.api_params(uuid)), 'RFS')):
if item is None:
self.log_error('Failed backup %s device configuration %s', config_type, hostname)
continue
if item.save(parsed_args.workdir, item_name=hostname, item_id=uuid):
self.log_info('Done %s device configuration %s', config_type, hostname)
# Backup items registered to the catalog
for _, info, index_cls, item_cls in catalog_iter(*parsed_args.tags, version=api.server_version):
item_index = index_cls.get(api)
if item_index is None:
self.log_debug('Skipped %s, item not supported by this vManage', info)
continue
if item_index.save(parsed_args.workdir):
self.log_info('Saved %s index', info)
matched_item_iter = (
(item_id, item_name) for item_id, item_name in item_index
if parsed_args.regex is None or regex_search(parsed_args.regex, item_name)
)
for item_id, item_name in matched_item_iter:
item = item_cls.get(api, item_id)
if item is None:
self.log_error('Failed backup %s %s', info, item_name)
continue
if item.save(parsed_args.workdir, item_index.need_extended_name, item_name, item_id):
self.log_info('Done %s %s', info, item_name)
# Special case for DeviceTemplateAttached and DeviceTemplateValues
if isinstance(item, DeviceTemplate):
devices_attached = DeviceTemplateAttached.get(api, item_id)
if devices_attached is None:
self.log_error('Failed backup %s %s attached devices', info, item_name)
continue
if devices_attached.save(parsed_args.workdir, item_index.need_extended_name, item_name, item_id):
self.log_info('Done %s %s attached devices', info, item_name)
else:
self.log_debug('Skipped %s %s attached devices, none found', info, item_name)
continue
try:
uuid_list = [uuid for uuid, _ in devices_attached]
values = DeviceTemplateValues(api.post(DeviceTemplateValues.api_params(item_id, uuid_list),
DeviceTemplateValues.api_path.post))
if values.save(parsed_args.workdir, item_index.need_extended_name, item_name, item_id):
self.log_info('Done %s %s values', info, item_name)
except RestAPIException as ex:
self.log_error('Failed backup %s %s values: %s', info, item_name, ex)
def runner(self, parsed_args, api=None, task_output=None):
source_info = f'Local workdir: "{parsed_args.workdir}"' if api is None else f'vManage URL: "{api.base_url}"'
self.log_info('Starting migrate: %s %s -> %s Local output dir: "%s"', source_info, parsed_args.from_version,
parsed_args.to_version, parsed_args.output)
# Output directory must be empty for a new migration
saved_output = clean_dir(parsed_args.output, max_saved=0 if parsed_args.no_rollover else 99)
if saved_output:
self.log_info('Previous migration under "%s" was saved as "%s"', parsed_args.output, saved_output)
if api is None:
backend = parsed_args.workdir
local_info = ServerInfo.load(backend)
server_version = local_info.server_version if local_info is not None else None
else:
backend = api
server_version = backend.server_version
try:
# Load migration processors
loaded_processors = {
FeatureTemplate: FeatureProcessor.load(from_version=parsed_args.from_version,
to_version=parsed_args.to_version),
DeviceTemplate: DeviceProcessor.load(from_version=parsed_args.from_version,
to_version=parsed_args.to_version)
}
self.log_info('Loaded template migration recipes')
server_info = ServerInfo(server_version=parsed_args.to_version)
if server_info.save(parsed_args.output):
self.log_info('Saved vManage server information')
id_mapping = {} # {<old_id>: <new_id>}
for tag in ordered_tags(CATALOG_TAG_ALL, reverse=True):
self.log_info('Inspecting %s items', tag)
for _, info, index_cls, item_cls in catalog_iter(tag, version=server_version):
item_index = self.index_get(index_cls, backend)
if item_index is None:
self.log_debug('Skipped %s, none found', info)
continue
name_set = {item_name for item_id, item_name in item_index}
is_bad_name = False
export_list = []
id_hint_map = {item_name: item_id for item_id, item_name in item_index}
for item_id, item_name in item_index:
item = self.item_get(item_cls, backend, item_id, item_name, item_index.need_extended_name)
if item is None:
self.log_error('Failed loading %s %s', info, item_name)
continue
try:
item_processor = loaded_processors.get(item_cls)
if item_processor is None:
raise StopProcessorException()
self.log_debug('Evaluating %s %s', info, item_name)
if not item_processor.is_in_scope(item, migrate_all=(parsed_args.scope == 'all')):
self.log_debug('Skipping %s, migration not necessary', item_name)
raise StopProcessorException()
new_name, is_valid = item.get_new_name(parsed_args.name)
if not is_valid:
self.log_error('New %s name is not valid: %s', info, new_name)
is_bad_name = True
raise StopProcessorException()
if new_name in name_set:
self.log_error('New %s name collision: %s -> %s', info, item_name, new_name)
is_bad_name = True
raise StopProcessorException()
name_set.add(new_name)
new_id = str(uuid4())
new_payload, trace_log = item_processor.eval(item, new_name, new_id)
for trace in trace_log:
self.log_debug('Processor: %s', trace)
if item.is_equal(new_payload):
self.log_debug('Skipping %s, no changes', item_name)
raise StopProcessorException()
new_item = item_cls(update_ids(id_mapping, new_payload))
id_mapping[item_id] = new_id
id_hint_map[new_name] = new_id
if item_processor.replace_original():
self.log_debug('Migrated replaces original: %s -> %s', item_name, new_name)
item = new_item
else:
self.log_debug('Migrated adds to original: %s + %s', item_name, new_name)
export_list.append(new_item)
except StopProcessorException:
pass
export_list.append(item)
if is_bad_name:
raise TaskException(f'One or more new {info} names are not valid')
if not export_list:
self.log_info('No %s migrated', info)
continue
if issubclass(item_cls, FeatureTemplate):
for factory_default in (factory_cedge_aaa, factory_cedge_global):
if any(factory_default.name == elem.name for elem in export_list):
self.log_debug('Using existing factory %s %s', info, factory_default.name)
# Updating because device processor always use the built-in IDs
id_mapping[factory_default.uuid] = id_hint_map[factory_default.name]
else:
export_list.append(factory_default)
id_hint_map[factory_default.name] = factory_default.uuid
self.log_debug('Added factory %s %s', info, factory_default.name)
new_item_index = index_cls.create(export_list, id_hint_map)
if new_item_index.save(parsed_args.output):
self.log_info('Saved %s index', info)
for new_item in export_list:
if new_item.save(parsed_args.output, new_item_index.need_extended_name, new_item.name,
id_hint_map[new_item.name]):
self.log_info('Saved %s %s', info, new_item.name)
except (ProcessorException, TaskException) as ex:
self.log_critical('Migration aborted: %s', ex)