def update_vlans(netid):
"""Sets vlans from a json file"""
fname = "vlans_generated_{}".format(netid)
vlans_to_deploy = json.reader(fname)
obj = Vlans()
obj.update_vlans_list(vlans_to_deploy)
def bulk_deploy_networks_for_all_orgs(agent):
org_group = settings["CLI"]["networks-org"]
store_list = settings["CLI"]["networks-store-list"]
serials_list = settings.get("CLI").get("networks-serials")
l.runlogs_logger.info("deploy networks <starting>")
org_list = json.reader(org_group, "templates")
orglist = json.make_pretty(org_list)
l.logger.info("org list: {}".format(orglist))
l.logger.info("store list: {}".format(store_list))
l.logger.info("serials list: {}".format(serials_list))
for org in org_list:
org_name = org["org_name"]
auto_globals.select_org(org_name)
l.runlogs_logger.info("selected org: {}".format(org_name))
l.runlogs_logger.info("using clone source: {}".format(settings["CLI"]["networks-clone-source"]))
l.runlogs_logger.info("using serials : {}".format(settings.get("CLI").get("networks-serials")))
# Now get the netid for the clone_source
store_name = settings["CLI"]["networks-clone-source"]
auto_globals.select_store(store_name)
auto_globals.load_store(agent, store_name)
config.set_clone_id(settings["netid"])
bulk.perform_bulk_deploy_networks(agent, deploy, deploy_serials, store_list)
l.runlogs_logger.info("deploy networks <finished>")
def set_each(self, netid, fw_rules, store_number):
success = False
str = None
try:
fwrules = json.reader("l3fwrules_deploy_{}".format(store_number))
singleRule = []
for rule in fwrules:
singleRule.append(rule)
success, str = meraki.updatemxl3fwrules(
config.api_key, netid, singleRule)
log.logger.debug(rule["comment"])
if not success:
log.logger.error("failed rule comment:{} {}".format(
rule["comment"], str))
log.runlogs_logger.error("failed rule comment:{} {}".format(
rule["comment"], str))
gv.fake_assert()
except (meraki.EmailFormatError, meraki.OrgPermissionError,
meraki.ListError) as err:
log.logger.error("Meraki error: {}".format(err.default))
log.runlogs_logger.error("Meraki error: {}".format(err.default))
exit(-1)
except Exception as err:
log.logger.error("exception failure netid:{} {}".format(
netid, str))
log.runlogs_logger.error("exception failure netid:{}".format(str))
gv.fake_assert()
return success, str
def setupSiteToSiteVpn():
netid = settings["netid"]
vlan_handler.createVlanFiles()
netx = json.reader(settings["CONFIG"]["netx-file"])
# Retrieves hubnetwork for the right org_name
hubnetworksAllOrgs = settings["CONFIG"]["hubnetworks"]
org_name = settings["org-name"]
hubnetworks = None
for item in hubnetworksAllOrgs:
if item["org_name"] == org_name:
hubnetworks = item["id"]
break
assert (hubnetworks)
# Generate vpn hubnetworks and vpn-subnets
defaultroute = settings["CONFIG"]["defaultroute"]
upper = "{}/22".format(netx['upper'])
lower = "{}/22".format(netx['lower'])
subnets = []
usevpn = []
subnets.append(lower)
usevpn.append(True) # enable the subnet above
subnets.append(upper)
usevpn.append(True) # enable the subnet above
# Upload to meraki
import utils.auto_logger as l
l.logger.debug("vpn update with hubnetworks: {}".format(hubnetworks))
l.logger.debug("vpn update with subnets: {}".format(subnets))
vpn.updatevpnsettings(netid, hubnetworks, defaultroute, subnets, usevpn)
l.logger.info("vpn updated.")
def __init__(self, storeNumber):
self.vlanFunnelTable = vlan_handler.createVlanTable(storeNumber)
self.firewallOutputFile = config.firewallConverted
fname = "firewall_template"
self.fwRules = json.reader(fname, "templates")
if self.fwRules is None:
l.logger.error("unable to load rules from firewall_template")
self.storeNumber = storeNumber
def getCreatedNetworkId(networkName):
fname = "network_{}".format(networkName)
network = Json.reader(fname)
if network is None:
logger.error("unable to load rules from firewall_template")
runlogs_logger.error("unable to load rules from firewall_template")
gv.fake_assert()
return network["id"]
def list(self, fname):
try:
vlans = json.reader(fname)
for vl in vlans:
l.logger.info("vlan: {}".format(vl))
except Exception as err:
l.logger.error("failed.")
l.logger.error("fname: {}".format(fname))
traceback.print_tb(err.__traceback__)
def create_update_vlans_list(self, netid, update_flag=True):
fname = "vlans_generated_{}".format(netid)
vlans_to_deploy = json.reader(fname)
self.get_vlans(netid)
vlans_deployed = []
for item in self.vlans:
id = int(item['id'])
vlans_deployed.append(id)
deploy_count = 0
for vlan in vlans_to_deploy:
id = int(vlan['id'])
if id not in vlans_deployed:
try:
apikey = config.api_key
networkid = netid
name = vlan['name']
vlanid = id
subnet = vlan['subnet']
mxip = vlan['applianceIp']
success, str = meraki.addvlan(apikey,
networkid,
vlanid,
name,
subnet,
mxip,
suppressprint=True)
l.logger.info("created vlan {}".format(id))
l.runlogs_logger.info("created vlan {} netid {}".format(
id, networkid))
self.update_single_vlan(vlan, update_all=True)
deploy_count += 1
except (meraki.EmailFormatError, meraki.OrgPermissionError,
meraki.ListError) as err:
l.logger.error("Meraki error: {} netid {}".format(
err.default, vlanid))
l.runlogs_logger.error("Meraki error: {}".format(
err.default))
exit(-1)
except Exception as err:
l.logger.error("{}".format(err.args))
l.runlogs_logger.error("{}".format(err.args))
gv.fake_assert()
if deploy_count == 0:
l.logger.info("vlans already exist - no vlans added")
l.runlogs_logger.info("vlans already exist - no vlans added")
else:
l.logger.info("added a total of {} vlans".format(deploy_count))
l.runlogs_logger.info(
"added a total of {} vlans".format(deploy_count))
def set(self, netid):
success=False
str=None
try:
fname = config.firewallConverted
fwrules = json.reader(fname)
success, str = meraki.updatemxl3fwrules(config.apikey, netid, fwrules)
if not success:
l.logger.error("failed netid:{} {}".format(netid, str))
except Exception as err:
l.logger.error("exception failure netid:{}".format(netid))
traceback.print_tb(err.__traceback__)
return success, str
def bulk_get(agent):
l.runlogs_logger.info("bulk get started")
org_group = settings["CLI"]["l3fwrules-org"]
store_list = settings["CLI"]["l3fwrules-store-list"]
org_list = json.reader(org_group,"templates")
for org in org_list:
org_name = org["org_name"]
l.runlogs_logger.info("selected org: {}".format(org_name))
auto_globals.select_org(org_name)
bulk.perform_bulk_get_firewall(agent, get, org_name, store_list)
l.runlogs_logger.info("bulk get finished")
def _set(org_name, fw_rules=None):
org_id = settings["org-id"]
assert (fw_rules)
fname = fw_rules
_vpn_rules = Json.reader(fname, "templates")
vpn_rules = []
# Remove default rules
for entry in _vpn_rules:
if entry.get("comment") == "Default rule":
continue
vpn_rules.append(entry)
obj = VpnFirewall()
obj.update_settings(org_id, vpn_rules)
def convert_master_template_to_jinja():
from utils.auto_json import reader, make_pretty
from copy import deepcopy
master_tpl = reader("vlan_template_master_final", configDir="config")
# str = make_pretty(master_tpl)
# print(str)
jinja_tpl = []
item = {}
for vlan in master_tpl:
id = vlan["Vlan"]
item["id"] = id
item["networkId"] = "{{networkid}}"
item["name"] = vlan["Description"]
subnet = vlan["Subnet"]
last_cctect = subnet.split(".")
last_octect = last_cctect[3]
last_octect = last_cctect[3].split("/")
last_octect = int(last_octect[0])
str = "{{vlan[{}]['subnet']}}.{}".format(id, last_octect + 1)
item["applianceIp"] = str
subnet = vlan["Subnet"]
str = "{{vlan[{}]['subnet']}}".format(id)
item["subnet"] = str
item["dnsNameservers"] = vlan["dnsNameservers"]
item["fixedIpAssignments"] = {}
item["reservedIpRanges"] = []
if vlan["reservedIpRanges1-start"] is not "":
it = {}
it["comment"] = vlan["reservedIpRanges1-comment"]
it["end"] = vlan["reservedIpRanges1-end"]
it["start"] = vlan["reservedIpRanges1-start"]
item["reservedIpRanges"].append(deepcopy(it))
if vlan["reservedIpRanges2-start"] is not "":
it = {}
it["comment"] = vlan["reservedIpRanges2-comment"]
it["end"] = vlan["reservedIpRanges2-end"]
it["start"] = vlan["reservedIpRanges2-start"]
item["reservedIpRanges"].append(deepcopy(it))
jinja_tpl.append(deepcopy(item))
str = make_pretty(jinja_tpl)
print(str)
def __init__(self, _fw_rules=None):
self.vlanFunnelTable = vlan_handler.createVlanTable()
fname = _fw_rules
self.firewallOutputFile = "l3fwrules_deploy_{}".format(settings["store-number"])
self.firewallOutputFileNetx = "l3fwrules_netx"
#self.header_netx_csv = ["policy", "protocol", "srcCidr", "srcPort", "destCidr", "destPort", "comment", "syslogEnabled"]
self.header_netx_csv = ["comment", "policy", "protocol", "srcPort", "srcCidr", "destPort", "destCidr", "syslogEnabled"]
self.fwRules = json.reader(fname, "templates")
self.funnelNetx = json.reader("vlans-funnel_netx")
assert(self.funnelNetx)
# Transform template to csv for external use
Csv.transform_to_csv(fname, header=self.header_netx_csv, path="templates")
# Transform template to format for external use
self.transform_rules_from_vlan_to_netx()
if self.fwRules is None:
l.logger.error("unable to load rules from firewall_template : {}". format(fname))
l.logger.info("using firewall golden rule : {}".format(fname))
def createContext():
context = {'networkid': "L_650207196201623673", 'vlan': {}}
fname = "funnel_vlans_table"
vlans = json.reader(fname)
for key, value in vlans.items():
vlanId = int(key)
subnet = value
octets = subnet.split(".")
octets = "{}.{}.{}".format(octets[0], octets[1], octets[2])
context["vlan"][vlanId] = {}
context["vlan"][vlanId]['octets'] = octets
context["vlan"][vlanId]['subnet'] = subnet
return context
def setEach(self, netid):
success=False
str=None
try:
fname = config.firewallConverted
fwrules = json.reader(fname)
singleRule = []
for rule in fwrules:
singleRule.append(rule)
success, str = meraki.updatemxl3fwrules(config.apikey, netid, singleRule)
l.logger.debug(rule["comment"])
if not success:
l.logger.error("failed rule comment:{} {}".format(rule["comment"], str))
except Exception as err:
l.logger.error("exception failure netid:{}".format(netid))
traceback.print_tb(err.__traceback__)
return success, str
def createUpdateVlanList(self, netid):
fname = "vlans_generated_{}".format(netid)
apikey = config.apikey
vlans = json.reader(fname)
for vl in vlans:
try:
networkid = vl['networkId']
id = vl['id']
name = vl['name']
subnet = vl['subnet']
applianceIp = vl['applianceIp']
if id in config.vlanUpdateOnly:
success = True
else:
success, _err = meraki.addvlan(apikey, networkid, id, name,
subnet, applianceIp)
if success:
fixedipassignments = vl['fixedIpAssignments']
reservedipranges = vl['reservedIpRanges']
vpnnatsubnet = None
dnsnameservers = vl['dnsNameservers']
success = True
_err = ""
if fixedipassignments or \
reservedipranges or \
vpnnatsubnet or \
dnsnameservers:
success, _err = meraki.updatevlan(
apikey, networkid, id, name, subnet, applianceIp,
fixedipassignments, reservedipranges, vpnnatsubnet,
dnsnameservers)
if success:
l.logger.debug("success")
l.logger.debug("vlan: {}".format(vl))
else:
l.logger.error("failed.")
l.logger.error("vlan: {}".format(vl))
return False
except Exception as err:
l.logger.error("exception")
traceback.print_tb(err.__traceback__)
return True
def set(self, netid, fw_rules, store_number):
success = False
str = None
try:
fwrules = json.reader("l3fwrules_deploy_{}".format(store_number))
success, str = meraki.updatemxl3fwrules(config.api_key, netid,
fwrules)
log.logger.info("success netid {} {}".format(netid, str))
if not success:
log.logger.error("failed netid {} {}".format(netid, str))
log.runlogs_logger.error("failed {} {}".format(netid, str))
gv.fake_assert()
except Exception as err:
log.logger.error("exception failure netid {}\n{}".format(
netid, str))
log.runlogs_logger.error("exception failure netid {}\n{}".format(
netid, str))
gv.fake_assert()
return success, str
def createContext(self):
self.context = {
'networkid': None,
'vlan' : {}
}
self.context['networkid']=self.netid
fname = "{}_table".format(config.vlanFunnelFile)
vlans = json.reader(fname)
for key, value in vlans.items():
vlanId = int(key)
subnet=value
octets=subnet.split(".")
octets="{}.{}.{}".format(octets[0], octets[1], octets[2])
self.context["vlan"][vlanId] = {}
self.context["vlan"][vlanId]['octets'] = octets
self.context["vlan"][vlanId]['subnet'] = subnet
l.logger.debug(self.context)
def add(netid, storeNumber):
networkName = "Store_{}".format(storeNumber)
fname, lower, upper = vlan_handler.createVlanFiles(netid, storeNumber)
l.logger.debug(fname)
# Get the ip from the vlan_generated file
vlans = json.reader(fname.split(".")[0])
for vlan in vlans:
ip = None
if vlan['id'] == int(config.staticRouteNextHop):
ip = vlan["applianceIp"]
break
subnet = "{}/22".format(lower) # lower
name = "lower summary subnet"
static_route.addStaticRoute(netid, name, subnet, ip)
subnet = "{}/22".format(upper) # lower
name = "upper summary subnet"
static_route.addStaticRoute(netid, name, subnet, ip)
def bulk_update(agent):
l.runlogs_logger.info("bulk update started")
if agent is "cli-deploy-networks":
org_group = settings["CLI"]["networks-org"]
fw_rules = None
store_list = settings["CLI"]["networks-store-list"]
else:
org_group = settings["CLI"]["l3fwrules-org"]
fw_rules = settings["CLI"]["l3fwrules-version"]
store_list = settings["CLI"]["l3fwrules-store-list"]
org_list = json.reader(org_group,"templates")
for org in org_list:
org_name = org["org_name"]
l.runlogs_logger.info("selected org: {}".format(org_name))
l.runlogs_logger.info("using l3fwrules : {}".format(fw_rules))
auto_globals.select_org(org_name)
fw_rules = "{}".format(fw_rules)
bulk.perform_bulk_update_firewall(agent, deploy, fw_rules, store_list)
l.runlogs_logger.info("bulk update finished")
def create_context(self):
self.context = {'networkid': None, 'vlan': {}}
self.context['networkid'] = self.netid
fname = "{}_table".format(settings["CONFIG"]["funnel-file"])
vlans = json.reader(fname)
for key, value in vlans.items():
vlanId = int(key)
subnet = value
octets = subnet.split(".")
octets = "{}.{}.{}".format(octets[0], octets[1], octets[2])
"""
Sample usage inside the template.
(note vlan is only a name to lock the reference with the template,
could not be anything)
'applianceIp': "{{vlan[19]['octets']}}.1",
"""
self.context["vlan"][vlanId] = {}
self.context["vlan"][vlanId]['octets'] = octets
self.context["vlan"][vlanId]['subnet'] = subnet
l.logger.debug(self.context)
def bulk_update(agent, vlans_only=False):
l.runlogs_logger.info("bulk update started")
org_group, store_list, vlans_list = ENTER_CONTEXT(agent)
org_list = json.reader(org_group, "templates")
assert(agent in ["cli-deploy-vlans-add", "cli-deploy-vlans-delete", "cli-deploy-sites", "cli-deploy-l3fwrules"])
for org in org_list:
org_name = org["org_name"]
auto_globals.select_org(org_name)
l.runlogs_logger.info("selected org: {}".format(org_name))
if agent == "cli-deploy-vlans-delete":
bulk.perform_bulk_update_store(agent, org_name, store_list, deploy_vlans_delete, vlans_list)
elif agent == "cli-deploy-vlans-add":
bulk.perform_bulk_update_store(agent, org_name, store_list, deploy_vlans_add)
else:
if agent in ["cli-deploy-sites", "cli-deploy-l3fwrules"]:
fw_rules = settings["CLI"]["sites-l3fwrules-version"]
l.runlogs_logger.info("selected l3fwrules : {}".format(fw_rules))
bulk.perform_bulk_update_store(agent, org_name, store_list, deploy)
l.runlogs_logger.info("bulk update finished")
def setupSiteToSiteVpn(netid, storeNumber):
# Load vlan table
store = "mx{}a".format(storeNumber)
networkName = "Store_{}".format(storeNumber)
vlan_handler.createVlanFiles(netid, storeNumber)
netx = json.reader(config.netxFile)
# Generate vpn hubnetworks and vpn-subnets
hubnetworks = config.hubnetworks
defaultroute = config.defaultroute
upper = "{}/22".format(netx['upper'])
lower = "{}/22".format(netx['lower'])
subnets = []
usevpn = []
subnets.append(lower)
usevpn.append(True) # enable the subnet above
subnets.append(upper)
usevpn.append(True) # enable the subnet above
# Upload to meraki
vpn.updatevpnsettings(netid, hubnetworks, defaultroute, subnets, usevpn)
url = "http://{}:{}@phxproxyvip.safeway.com:8080".format(
proxyUser, proxyPassword)
os.environ['HTTPS_PROXY'] = url
def getOrgId():
orgName = config[0]["org-name"]
orgid = None
for org in orgs:
if org["name"] == orgName:
orgid = org["id"]
break
return orgid
config = json.reader("safeway-config", "config")
personal = json.reader("../../safeway-personal")
orgs = json.reader("../config/safeway-orgs")
apikey = personal[0]["apikey"]
firewall = config[0]["firewall"]
staticRouteNextHop = firewall['staticRouteNextHop']
firewallConverted = firewall["output_file"]
orgid = getOrgId()
useProxy = personal[0]["useProxy"]
#if useProxy:
# proxyUser = personal[0]['proxy_user']
# proxyPassword = personal[0]['proxy_password']