def migrate_ips(self, block=None): """3. Migrate m.ip_addresses -> q.quark_ip_addresses This migration is complicated. I believe q.subnets will need to be populated during this step as well. m.ip_addresses is scattered all over the place and it is not a 1:1 relationship between m -> q. Some more thought will be needed for this one. First we need to use m.ip_addresses to find the relationship between the ip_block and the m.interfaces. After figuring out that it will then be possible to create a q.subnet connected to the network. """ addresses = self.melange_session.query(melange.IpAddresses)\ .filter_by(ip_block_id=block.id).all() for address in addresses: init_id(self.json_data, 'ips', address.id) """Populate interface_network cache""" interface = address.interface_id if interface is not None and\ interface not in self.interface_network: self.interface_network[interface] = \ trim_br(block.network_id) if interface in self.interface_network and\ self.interface_network[interface] != \ trim_br(block.network_id): self.log.error("Found interface with different " "network id: {0} != {1}" .format(self.interface_network[interface], trim_br(block.network_id))) deallocated = False deallocated_at = None # If marked for deallocation # put it into the quark ip table as deallocated if address.marked_for_deallocation == 1: deallocated = True deallocated_at = address.deallocated_at ip_address = netaddr.IPAddress(address.address) q_ip = quarkmodels.IPAddress(id=address.id, created_at=address.created_at, used_by_tenant_id= address.used_by_tenant_id, network_id= trim_br(block.network_id), subnet_id=block.id, version=ip_address.version, address_readable=address.address, deallocated_at=deallocated_at, _deallocated=deallocated, address=int(ip_address.ipv6())) # Populate interface_ip cache if interface not in self.interface_ip: self.interface_ip[interface] = set() self.interface_ip[interface].add(q_ip) self.add_to_session(q_ip, 'ips', q_ip.id)
def migrate_ips(self, block=None): """3. Migrate m.ip_addresses -> q.quark_ip_addresses This migration is complicated. I believe q.subnets will need to be populated during this step as well. m.ip_addresses is scattered all over the place and it is not a 1:1 relationship between m -> q. Some more thought will be needed for this one. First we need to use m.ip_addresses to find the relationship between the ip_block and the m.interfaces. After figuring out that it will then be possible to create a q.subnet connected to the network. """ addresses = self.melange_session.query(melange.IpAddresses)\ .filter_by(ip_block_id=block.id).all() for address in addresses: init_id(self.json_data, 'ips', address.id) """Populate interface_network cache""" interface = address.interface_id if interface is not None and\ interface not in self.interface_network: self.interface_network[interface] = \ trim_br(block.network_id) if interface in self.interface_network and\ self.interface_network[interface] != \ trim_br(block.network_id): self.log.error("Found interface with different " "network id: {0} != {1}".format( self.interface_network[interface], trim_br(block.network_id))) deallocated = False deallocated_at = None # If marked for deallocation # put it into the quark ip table as deallocated if address.marked_for_deallocation == 1: deallocated = True deallocated_at = address.deallocated_at ip_address = netaddr.IPAddress(address.address) q_ip = quarkmodels.IPAddress( id=address.id, created_at=address.created_at, used_by_tenant_id=address.used_by_tenant_id, network_id=trim_br(block.network_id), subnet_id=block.id, version=ip_address.version, address_readable=address.address, deallocated_at=deallocated_at, _deallocated=deallocated, address=int(ip_address.ipv6())) # Populate interface_ip cache if interface not in self.interface_ip: self.interface_ip[interface] = set() self.interface_ip[interface].add(q_ip) self.add_to_session(q_ip, 'ips', q_ip.id)
def migrate_networks(self): """1. Migrate the m.ip_blocks -> q.quark_networks Migration of ip_blocks to networks requires one take into consideration that blocks can have 'children' blocks. A scan of the melange tables shows though that this feature hasn't been used. An ip_block has a cidr which maps to a corresponding subnet in quark. """ blocks = self.melange_session.query(melange.IpBlocks).all() networks = dict() """Create the networks using the network_id. It is assumed that a network can only belong to one tenant""" for block in blocks: init_id(self.json_data, 'networks', trim_br(block.network_id)) if trim_br(block.network_id) not in networks: networks[trim_br(block.network_id)] = { "tenant_id": block.tenant_id, "name": block.network_name, "max_allocation": block.max_allocation, "created_at": block.created_at } elif trim_br(block.network_id) in networks: if networks[trim_br( block.network_id )]["created_at"] > block.created_at: # noqa networks[trim_br( block.network_id )]["created_at"] = block.created_at # noqa elif networks[trim_br( block.network_id)]["tenant_id"] != block.tenant_id: r = "Found different tenant on network:{0} != {1}"\ .format(networks[trim_br( block.network_id)]["tenant_id"], block.tenant_id) self.log.critical(r) set_reason(self.json_data, 'networks', trim_br(block.network_id), r) raise Exception for net in networks: cache_net = networks[net] q_network = quarkmodels.Network( id=net, tenant_id=cache_net["tenant_id"], name=cache_net["name"], max_allocation=cache_net["max_allocation"]) self.add_to_session(q_network, 'networks', net) blocks_without_policy = 0 for block in blocks: init_id(self.json_data, 'subnets', block.id) q_subnet = quarkmodels.Subnet(id=block.id, network_id=trim_br(block.network_id), tenant_id=block.tenant_id, cidr=block.cidr, do_not_use=block.omg_do_not_use, created_at=block.created_at) self.add_to_session(q_subnet, 'subnets', q_subnet.id) q_dns1 = quarkmodels.DNSNameserver( tenant_id=block.tenant_id, created_at=block.created_at, ip=int(netaddr.IPAddress(block.dns1)), # noqa subnet_id=q_subnet.id) q_dns2 = quarkmodels.DNSNameserver( tenant_id=block.tenant_id, created_at=block.created_at, ip=int(netaddr.IPAddress(block.dns2)), # noqa subnet_id=q_subnet.id) self.new_to_session(q_dns1) self.new_to_session(q_dns2) self.migrate_ips(block=block) self.migrate_routes(block=block) # caching policy_ids for use in migrate_policies if block.policy_id: if block.policy_id not in self.policy_ids.keys(): self.policy_ids[block.policy_id] = {} self.policy_ids[block.policy_id][block.id] =\ trim_br(block.network_id) else: self.log.warning("Found block without a policy: {0}".format( block.id)) blocks_without_policy += 1 # have to add new routes as well: new_gates = 0 for block in blocks: if block.gateway: self.migrate_new_routes(block) new_gates += 1 self.log.info( "Cached {0} policy_ids. {1} blocks found without policy.".format( len(self.policy_ids), blocks_without_policy)) self.log.info("{0} brand new gateways created.".format(new_gates))
def migrate_networks(self): """1. Migrate the m.ip_blocks -> q.quark_networks Migration of ip_blocks to networks requires one take into consideration that blocks can have 'children' blocks. A scan of the melange tables shows though that this feature hasn't been used. An ip_block has a cidr which maps to a corresponding subnet in quark. """ blocks = self.melange_session.query(melange.IpBlocks).all() networks = dict() """Create the networks using the network_id. It is assumed that a network can only belong to one tenant""" for block in blocks: init_id(self.json_data, 'networks', trim_br(block.network_id)) if trim_br(block.network_id) not in networks: networks[trim_br(block.network_id)] = { "tenant_id": block.tenant_id, "name": block.network_name, "max_allocation": block.max_allocation, "created_at": block.created_at} elif trim_br(block.network_id) in networks: if networks[trim_br(block.network_id)]["created_at"] > block.created_at: # noqa networks[trim_br(block.network_id)]["created_at"] = block.created_at # noqa elif networks[trim_br( block.network_id)]["tenant_id"] != block.tenant_id: r = "Found different tenant on network:{0} != {1}"\ .format(networks[trim_br( block.network_id)]["tenant_id"], block.tenant_id) self.log.critical(r) set_reason(self.json_data, 'networks', trim_br(block.network_id), r) raise Exception for net in networks: cache_net = networks[net] q_network = quarkmodels.Network(id=net, tenant_id=cache_net["tenant_id"], name=cache_net["name"], max_allocation= cache_net["max_allocation"]) self.add_to_session(q_network, 'networks', net) blocks_without_policy = 0 for block in blocks: init_id(self.json_data, 'subnets', block.id) q_subnet = quarkmodels.Subnet(id=block.id, network_id= trim_br(block.network_id), tenant_id=block.tenant_id, cidr=block.cidr, do_not_use=block.omg_do_not_use, created_at=block.created_at) self.add_to_session(q_subnet, 'subnets', q_subnet.id) q_dns1 = quarkmodels.DNSNameserver(tenant_id=block.tenant_id, created_at=block.created_at, ip= int(netaddr.IPAddress(block.dns1)), # noqa subnet_id=q_subnet.id) q_dns2 = quarkmodels.DNSNameserver(tenant_id=block.tenant_id, created_at=block.created_at, ip= int(netaddr.IPAddress(block.dns2)), # noqa subnet_id=q_subnet.id) self.new_to_session(q_dns1) self.new_to_session(q_dns2) self.migrate_ips(block=block) self.migrate_routes(block=block) # caching policy_ids for use in migrate_policies if block.policy_id: if block.policy_id not in self.policy_ids.keys(): self.policy_ids[block.policy_id] = {} self.policy_ids[block.policy_id][block.id] =\ trim_br(block.network_id) else: self.log.warning("Found block without a policy: {0}" .format(block.id)) blocks_without_policy += 1 # have to add new routes as well: new_gates = 0 for block in blocks: if block.gateway: self.migrate_new_routes(block) new_gates += 1 self.log.info("Cached {0} policy_ids. {1} blocks found without policy." .format(len(self.policy_ids), blocks_without_policy)) self.log.info("{0} brand new gateways created.".format(new_gates))