def _update_config(tinydb_log_file, __file__, new_netor_home_directory):
"""
Execute the actual updates in the files. Salt master, minion and proxy.
:param tinydb_log_file: the filename to send the logging message after the operation is completed
:param __file__: script name who is sending the message to log
:param new_netor_home_directory: it is the actual new Neto home directory to be updated on files
:return: nothing
"""
_create_update_master_minion_proxy(new_netor_home_directory, 'master')
_create_update_master_minion_proxy(new_netor_home_directory, 'minion')
_create_update_master_minion_proxy(new_netor_home_directory, 'proxy')
print('\nNetor home directory replaced in salt master, minion and proxy.')
print("\nAdd or modified if necessary " + new_netor_home_directory + "bin to your .profile")
print(" vi $HOME/.profile")
print(" PATH=\"$PATH:" + new_netor_home_directory + "bin\n")
print("\nAdd or modified if necessary " + new_netor_home_directory + " to /etc/environment")
print(" sudo vi /etc/environment")
print(" NETOR=\"$PATH:" + new_netor_home_directory)
print("\nLogoff session or restart system, and login again.")
print("\nATTENTION: If you are using Salt restart the daemons with \"netor-salt-restart\"\n")
netorlogging.log_msg(tinydb_log_file, __file__,
"Netconf executed. Neto.config and static vars in scripts updated. ")
def export_csv(self, tinydb_log_file, db_path_name, export_path_name):
"""
Export full DB table 'devices' content to CSV
:param tinydb_log_file: log file to send the message
:param db_path_name: full path name of the DB to list
:param export_path_name: full path name of .CVS file to export the DB
:return:
"""
db_devices = self.table_devices.all()
file = open(export_path_name, "w")
file.write(
"customer,site,dev_name,dev_ip,os,userid,passwd,salt_proxy_required\n"
)
print()
for item in db_devices:
print("%s,%s,%s,%s,%s,%s,%s,%s" %
(item['customer'], item['site'], item['dev_name'],
item['dev_ip'], item['os'], item['userid'], item['passwd'],
item['salt_proxy_required']))
file.write("%s,%s,%s,%s,%s,%s,%s,%s\n" %
(item['customer'], item['site'], item['dev_name'],
item['dev_ip'], item['os'], item['userid'],
item['passwd'], item['salt_proxy_required']))
file.close()
netorlogging.log_msg(
tinydb_log_file, __file__, "Full DB exported: " + db_path_name +
" to filename" + export_path_name)
print("\nFull DB exported: " + db_path_name + " to filename " +
export_path_name)
def _switchdb():
"""
Updates the full path name of the DB in configuration and scripts files.
As an example it can be used to have multiple database files on the same netor home directory, in order
to support different group of devices, like different customers, environments, group of devices, or even
for copying DB files to different folders or machines for sharing them.
:return: nothing
"""
netorconf.check_netor_config(_NETOR_HOME_DIRECTORY)
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
netor_config_path_name = (_NETOR_HOME_DIRECTORY + "netor/netor.config")
config.read(netor_config_path_name)
current_db_path_name = config['TinyDB']['db_path_name']
tinydb_log_file = config['TinyDB']['tinydb_log_file']
print('\nAvailable Databases:\n')
for item in glob.glob(_NETOR_HOME_DIRECTORY + 'netor/tinydb/data/*.json'):
print(item)
print()
db_name = input("Enter DB full path name, or \"name\" to create a new DB as \"name.json\", "
"or hit enter to use current [" + current_db_path_name + "]: ").lower()
if db_name == '':
print("\nUsing current DB")
db_name = current_db_path_name
elif os.path.isfile(db_name):
print("\nReusing existing DB")
db_name = db_name
elif db_name.isalnum():
print("\nCreating new DB")
db_name = _NETOR_HOME_DIRECTORY + "netor/tinydb/data/" + db_name + ".json"
print("\nPlease create the first customer in order to create DB with (dbcustomer)")
else:
print('Invalid DB file directory')
exit(0)
_update_config(db_name)
netorlogging.log_msg(tinydb_log_file, __file__,
"Switch DB command executed. Parameters in configuration files and scripts updated. ")
print("\nSwitch DB command executed. Parameters in configuration files and scripts updated. ")
def list(self, tinydb_log_file, db_path_name):
"""
List full DB content.
:param tinydb_log_file: log file to send the message
:param db_path_name: full path name of the DB to list
:return:
"""
print("\nLIST DATABASE")
db_customers = sorted(self.table_customers.all(),
key=lambda x: (x['customer']))
db_sites = sorted(self.table_sites.all(),
key=lambda x: (x['customer'], x['site']))
db_devices = sorted(self.table_devices.all(),
key=lambda x:
(x['customer'], x['site'], x['dev_name']))
print()
print("List Customers")
for item in db_customers:
print(item['customer'].replace('_', ' '))
print()
print("List Sites")
print('\n%-23s %-23s' % ('Customer Name', 'Site Name'))
for item in db_sites:
print('%-23s %-23s' % (item['customer'].replace(
'_', ' '), item['site'].replace('_', ' ')))
print()
print("List Devices")
print('\n%-22s %-22s %-20s %-17s %-11s %-20s %-20s %-16s' %
('Customer Name', 'Site Name', 'Device Name', 'Device IP',
'Device OS', 'User Name', 'Password', 'Salt Proxy Req'))
for item in db_devices:
print(
'%-22s %-22s %-20s %-17s %-11s %-20s %-20s %-16s' %
(item['customer'].replace('_', ' '), item['site'].replace(
'_', ' '), item['dev_name'], item['dev_ip'], item['os'],
item['userid'], item['passwd'], item['salt_proxy_required']))
netorlogging.log_msg(tinydb_log_file, __file__,
"Full DB listed: " + db_path_name)
print("\nFull DB listed: " + db_path_name)
def ansible_push_inventory(tinydb_log_file, ansible_hosts_path_name,
ansible_backup_hosts_path_name,
devices_to_push):
"""
Backup of the ``ansible/hosts`` inventory file to ``ansible/backup`` directory.
Generates a new ``ansible/hosts`` file based on the DB information (actual push of data).
:param tinydb_log_file: log file to send the message after the operations are being completed
:param ansible_hosts_path_name: full path name of the ansible hosts file
:param ansible_backup_hosts_path_name: full path name of the ansible hosts file directory
:param devices_to_push: list of devices to push
:return: nothing
"""
overwrite = input("\nAnsible: hosts file will be overwritten (y/n): ")
if overwrite.lower() == "y":
print(
"\nBacking up ansible/hosts inventory file to ansible/backup")
source = ansible_hosts_path_name
destination = ansible_backup_hosts_path_name + "_" + datetime.datetime.now(
).strftime('%Y-%m-%d-%H-%M-%S')
copyfile(source, destination)
print("Generating new ansible/hosts inventory file")
try:
file = open(source, "w+")
except PermissionError:
print("\nFile 'hosts' not owned. Permission denied.\n")
sys.exit(1)
file.write("\n#Time of push DB: " + str(datetime.datetime.now()) +
"\n")
c_prev, s_prev = "", ""
for item_groups in devices_to_push:
if item_groups['customer'] != c_prev:
file.write("\n[" + item_groups['customer'] +
":children]\n")
c_prev = item_groups['customer']
if item_groups['site'] != s_prev:
file.write(item_groups['customer'] + "_" +
item_groups['site'] + "\n")
s_prev = item_groups['site']
else:
file.write(item_groups['customer'] + "_" +
item_groups['site'] + "\n")
s_prev = item_groups['site']
elif item_groups['site'] != s_prev:
file.write(item_groups['customer'] + "_" +
item_groups['site'] + "\n")
s_prev = item_groups['site']
c_prev, s_prev = "", ""
for item_device in devices_to_push:
if c_prev != item_device['customer']:
file.write("\n[" + item_device['customer'] + "_" +
item_device['site'] + "]\n")
c_prev = item_device['customer']
if s_prev != item_device['site']:
file.write(
"{}_{}_{} ansible_host={} ansible_network_os={} ansible_user={} ansible_password={} "
"username={} password={} platform={}\n".format(
item_device['customer'], item_device['site'],
item_device['dev_name'], item_device['dev_ip'],
item_device['os'], item_device['userid'],
item_device['passwd'], item_device['userid'],
item_device['passwd'], item_device['os']))
s_prev = item_device['site']
else:
file.write(
"{}_{}_{} ansible_host={} ansible_network_os={} ansible_user={} ansible_password={} "
"username={} password={} platform={}\n".format(
item_device['customer'], item_device['site'],
item_device['dev_name'], item_device['dev_ip'],
item_device['os'], item_device['userid'],
item_device['passwd'], item_device['userid'],
item_device['passwd'], item_device['os']))
s_prev = item_device['site']
elif s_prev == item_device['site']:
file.write(
"{}_{}_{} ansible_host={} ansible_network_os={} ansible_user={} ansible_password={} "
"username={} password={} platform={}\n".format(
item_device['customer'], item_device['site'],
item_device['dev_name'], item_device['dev_ip'],
item_device['os'], item_device['userid'],
item_device['passwd'], item_device['userid'],
item_device['passwd'], item_device['os']))
s_prev = item_device['site']
elif s_prev != item_device['site']:
file.write("\n[" + item_device['customer'] + "_" +
item_device['site'] + "]\n")
file.write(
"{}_{}_{} ansible_host={} ansible_network_os={} ansible_user={} ansible_password={} "
"username={} password={} platform={}\n".format(
item_device['customer'], item_device['site'],
item_device['dev_name'], item_device['dev_ip'],
item_device['os'], item_device['userid'],
item_device['passwd'], item_device['userid'],
item_device['passwd'], item_device['os']))
file.close()
netorlogging.log_msg(
tinydb_log_file, __file__,
"Push DB executed. Ansible configuration files updated.")
print("\nPush DB executed. Ansible configuration files updated.\n")
return
else:
return
def salt_push_inventory(tinydb_log_file, salt_minion_path_name,
salt_backup_directory, salt_pillar_directory,
salt_backup_pillar_directory, salt_top_path_name,
salt_states_directory, devices_to_push):
"""
Backup ``salt/config/minion`` file to ``salt/config/backup`` directory.
Generates a new minion file based on the actual minion file, but replacing the first lines
**(to be improved in order to review the content/update of the lines)**.
Generates the new ``top.sls`` file with the devices listed in the DB, and all the states only in the
\\'*' section.
**(To be improved to actually add the corresponding states to specific devices)**.
Backup ``salt/config/pillar/\\*.sls`` files to ``/salt/config/backup/pillar directory``.
Generates new ``pillar/*device_name*.sls`` files based on the content of the DB file, if the device has to be
managed via ``salt-proxy`` according the DB ``salt_proxy_required`` field. With the following details:
::
proxy:
proxytype: napalm
driver: ios
host: dev_ip
username: userid
password: passwd
optional_args:
use_keys: True
auto_rollback_on_error: True
:param tinydb_log_file: log file to send the message after the operations are being completed
:param salt_minion_path_name: minion full path name
:param salt_backup_directory: minion backup directory
:param salt_pillar_directory: pillar .sls files directory
:param salt_backup_pillar_directory: backup pillar \\*.sls directory
:param salt_top_path_name: full path name of ''top.sls'' file
:param salt_states_directory: add all the states to the general ``\\'*'`` section of the ``top.sls`` file
:param devices_to_push: list of devices to push to Ansible and Salt
:return: nothing
"""
overwrite = input(
"\nSalt: minion config file will be updated, and top.sls and [dev_name].sls files will be "
"overwritten (y/n): ")
if overwrite.lower() == "y":
print(
"\nBacking up salt/config/minion configuration file to salt/config/backup"
)
salt_minion_backup_path_name = salt_backup_directory + "minion_" + \
datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
copyfile(salt_minion_path_name, salt_minion_backup_path_name)
print("Creating new temporary minion configuration file")
new_file_name = salt_minion_path_name + ".pre"
new_file = open(new_file_name, 'w+')
new_file.write("beacons:\n")
new_file.write(" salt_proxy:\n")
new_file.write(" - proxies:\n")
for doc in devices_to_push:
if doc['salt_proxy_required'] == "y":
new_file.write(" " + doc['customer'] + "_" +
doc['site'] + "_" + doc['dev_name'] +
": {}\n")
new_file.write(" - interval: 60\n")
new_file.write("\n")
salt_minion_former_start_config = False
try:
with open(salt_minion_path_name, 'r') as file:
for line in file:
if "Primary configuration settings" in line:
salt_minion_former_start_config = True
if salt_minion_former_start_config:
new_file.writelines(line)
except (OSError, IOError):
print("Original minion file not found, skipping.")
new_file.close()
print("Replacing minion configuration file")
move(new_file_name, salt_minion_path_name)
print(
"Backing up salt/config/pillar sls inventory files to salt/config/backup"
)
for file_name in os.listdir(salt_pillar_directory):
file_is_directory = salt_pillar_directory + file_name
if not os.path.isdir(file_is_directory):
source_file_name = salt_pillar_directory + file_name
destination_file_name = salt_backup_pillar_directory + file_name + "." + \
datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
copyfile(source_file_name, destination_file_name)
print("Removing salt/config/pillar sls inventory files")
for file_name in os.listdir(salt_pillar_directory):
file_is_not_directory = salt_pillar_directory + file_name
if not os.path.isdir(file_is_not_directory):
os.remove(file_is_not_directory)
print("Generating new top.sls inventory file")
new_top_file = open(salt_top_path_name, "w+")
new_top_file.write("base:\n")
new_top_file.write(" '*':\n")
for state_file_name in os.listdir(salt_states_directory):
dev_name = state_file_name.split(".sls")
new_top_file.write(" - " + str(dev_name[0]) + "\n")
for item_device in devices_to_push:
new_top_file.write("\n '" + item_device['customer'] + "_" +
item_device['site'] + "_" +
item_device['dev_name'] + "':\n")
new_top_file.write(" - " + item_device['customer'] + "_" +
item_device['site'] + "_" +
item_device['dev_name'] + "\n")
new_top_file.close()
print("Generating new sls devices files")
for item_device in devices_to_push:
new_sls_device_file_name = salt_pillar_directory + item_device['customer'] + "_" + \
item_device['site'] + "_" + item_device['dev_name'] + ".sls"
new_sls_device_file = open(new_sls_device_file_name, "w+")
new_sls_device_file.write("proxy:\n")
new_sls_device_file.write(" proxytype: napalm\n")
new_sls_device_file.write(" driver: " + item_device['os'] +
"\n")
new_sls_device_file.write(" host: " + item_device['dev_ip'] +
"\n")
new_sls_device_file.write(" username: "******"\n")
new_sls_device_file.write(" password: "******"\n")
new_sls_device_file.write(" optional_args:\n")
new_sls_device_file.write(" use_keys: True\n")
new_sls_device_file.write(" auto_rollback_on_error: True\n")
new_sls_device_file.write("role: network\n")
new_sls_device_file.close()
netorlogging.log_msg(
tinydb_log_file, __file__,
"Push DB executed. Salt configuration files updated.")
print("\nPush DB executed. Salt configuration files updated.")
else:
return
def _redirect():
"""
Worker redirect to all operations on the DB tables.
Operates the DB on table specified, for list, add, modify and delete registers.
It uses the local static variables of the ``worker.py`` script as ``NETOR_HOME_DIRECTORY`` and ``DB_PATH_NAME``,
unless a full path name to a TinyDB database is specified.
It supports the following parameters:
1. specifying operations:
- l (list)
- a (add)
- m (modify)
- d (delete)
2. table to operate with
- customer
- sites
- devices
3. specifying DB full path name
Example:
``../tinydb/scripts/worker.py -l customers /full/path/name/database.json``
Logging to file ./log/tinydb.log
:return: nothing
"""
netorconf.check_netor_config(_NETOR_HOME_DIRECTORY)
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
config.read((_NETOR_HOME_DIRECTORY + "netor/netor.config"))
tinydb_log_file = config['TinyDB']['tinydb_log_file']
if (len(sys.argv) == 0) or len(sys.argv) > 4:
print("\nInvalid parameters. Admin -operation tablename dbfile\n")
return
else:
option = sys.argv[1]
table = sys.argv[2]
if len(sys.argv) == 4:
if os.path.isfile((sys.argv[3])):
print("Using specified DB file: " + sys.argv[3])
db_path_name = sys.argv[3]
else:
print("DB path or file not found")
return
elif len(sys.argv) == 3:
db_path_name = _DB_PATH_NAME
print("\nUsing script default DB File: " + db_path_name + "\n")
else:
print("\nInvalid parameters. Admin -operation tablename\n")
return
if table == "customers":
x = customers.Customers(db_path_name)
elif table == "sites":
x = sites.Sites(db_path_name)
elif table == "devices":
x = devices.Devices(db_path_name)
else:
print("Invalid table name")
return
if option == "-l":
item = x.list()
if item:
netorlogging.log_msg(
tinydb_log_file, __file__,
"DB \"" + db_path_name.split("/")[-1] + "\" table \"" + table +
"\" listed")
elif option == "-a":
item = x.add()
if item:
netorlogging.log_msg(
tinydb_log_file, __file__,
"DB \"" + db_path_name.split("/")[-1] + "\" table \"" + table +
"\" added \"" + item + "\"")
elif option == "-m":
item = x.modify()
if item:
netorlogging.log_msg(
tinydb_log_file, __file__,
"DB \"" + db_path_name.split("/")[-1] + "\" table \"" + table +
"\" modified: \"" + item[0] + "\" to \"" + item[1] + "\"")
elif option == "-d":
item = x.delete()
if item:
netorlogging.log_msg(
tinydb_log_file, __file__,
"DB \"" + db_path_name.split("/")[-1] + "\" table \"" + table +
"\" deleted \"" + item + "\"")
else:
print("Invalid class operation")
def start_process():
"""
Starts the process of import the CSV to DB, asking for the path names to the files.
:return: Nothing
"""
netorconf.check_netor_config(_NETOR_HOME_DIRECTORY)
netorconf.check_netor_config(_NETOR_HOME_DIRECTORY)
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
config.read((_NETOR_HOME_DIRECTORY + "netor/netor.config"))
tinydb_log_file = config['TinyDB']['tinydb_log_file']
print('\nAvailable Databases:\n')
for item in glob.glob(_NETOR_HOME_DIRECTORY + 'netor/tinydb/data/*.json'):
print(item)
select_db = input(
'\nSelect DB to upload the CSV. Copy full path to the DB, or Hit ENTER to use Current ['
+ _DB_PATH_NAME + '] or \"new\" for a new DB: ').lower()
if select_db == '':
destination_db_to_import = _DB_PATH_NAME
elif select_db == 'new':
new_db = input('Enter new DB name. Example \"newdb.json\": ')
if not ('.json' in new_db or not new_db.isalnum()):
print('\nInvalid file name')
exit(1)
destination_db_to_import = _NETOR_HOME_DIRECTORY + 'netor/tinydb/data/' + new_db
print(
'\nRemember to \"netor-db-switch\" in order to start using the data uploaded to the DB.'
)
elif select_db in glob.glob(_NETOR_HOME_DIRECTORY +
'netor/tinydb/data/*.json'):
destination_db_to_import = select_db
print(
'\nRemember to \"netor-db-switch\" in order to start using the data uploaded to the DB.'
)
else:
print('\nInvalid file name')
destination_db_to_import = ''
exit(1)
csv_file_to_import = input('Enter full path name to .CSV file to import (example: /full/path/name/file.csv): '). \
lower()
print()
if not os.path.isfile(csv_file_to_import):
print('\nFile not found')
exit(1)
else:
total_lines_number_processed, lines_imported = _import_csv(
destination_db_to_import, csv_file_to_import)
print('\nFinished importing the file.')
print('%i lines processed' % total_lines_number_processed)
print('%i lines imported' % lines_imported)
netorlogging.log_msg(
tinydb_log_file, __file__, 'CSV imported. ' +
str(total_lines_number_processed) + ' lines processed, and ' +
str(lines_imported) + ' lines imported.')
return
