def handle_open_template_msg(self, conn, ot):
try:
template = DBService.run_and_wait(db.TopologyTemplate.objects.get(name=ot.template_name))
except db.TopologyTemplate.DoesNotExist:
self.terminate_connection(conn, "template '%s' does not exist" % ot.template_name)
return
# find an IP block to allocate IPs from for this user
blocks = DBService.run_and_wait(db.IPBlock.objects.filter(org=conn.vns_user_profile.org))
if not blocks:
self.terminate_connection(conn, "your organization (%s) has no available IP blocks" % conn.vns_user_profile.org)
return
ip_block_from = blocks[0]
if ot.get_src_filters() == VNSOpenTemplate.NO_SRC_FILTERS:
src_filters = []
else:
src_filters = ot.get_src_filters()
err_msg, topo, alloc, tree = AddressAllocation.instantiate_template(conn.vns_user_profile.user,
template,
ip_block_from,
src_filters,
True, True)
if err_msg:
self.terminate_connection(conn, err_msg)
else:
s2intfnum = '2' if ot.template_name == '1-router 2-server' else '1'
rtable_msg = VNSRtable(ot.vrhost, VNSSimulator.build_rtable(topo, s2intfnum))
conn.send(rtable_msg)
logging.debug('Sent client routing table message: %s' % rtable_msg)
self.handle_connect_to_topo(conn, topo.id, ot.vrhost)
def finish_marathon(cls):
"""
заканчивает марафон для всех
"""
db = DBService()
users = db.update('user', {'win': 1}, 'win = 0')
db.close()
def build_rtable(topo, s2intfnum):
# TODO: write this function for real; just a quick hack for now
s1 = DBService.run_and_wait(lambda:db.IPAssignment.objects.get(topology=topo, port__node=db.Node.objects.get(template=topo.template, name='Server1')))
s2 = DBService.run_and_wait(lambda:db.IPAssignment.objects.get(topology=topo, port__node=db.Node.objects.get(template=topo.template, name='Server2')))
return '\n'.join(['0.0.0.0 172.24.74.17 0.0.0.0 eth0',
'%s %s 255.255.255.254 eth1' % (s1.ip, s1.ip),
'%s %s 255.255.255.254 eth%s' % (s2.ip, s2.ip, s2intfnum)])
def terminate_connection(self, conn, why, notify_client=True, log_it=True, lvl=logging.INFO):
"""Terminates the client connection conn. This event will be logged
unless log_it is False. If notify_client is True, then the client will
be sent a VNSClose message with an explanation."""
# terminate the client
if conn.connected:
if notify_client:
for m in VNSClose.get_banners_and_close(why):
conn.send(m)
conn.transport.loseConnection()
if log_it:
logging.log(lvl, 'terminating client (%s): %s' % (conn, why))
# cleanup client and topology info
tid = self.clients.get(conn)
if tid is not None:
del self.clients[conn]
topo = self.topologies[tid]
topo.client_disconnected(conn)
if not topo.is_active():
if topo.has_gateway():
self.resolver.unregister_topology(topo)
with self.topologies_lock:
del self.topologies[tid]
self.topologies_changed = True
DBService.run_and_wait(topo.get_stats().finalize)
if topo.is_temporary():
AddressAllocation.free_topology(tid)
else:
AddressAllocation.deallocate_from_topology(topo.t)
for ti_conn in topo.interactors:
self.terminate_ti_connection(ti_conn, 'GOODBYE: Topology %d has been shutdown' % tid)
def instantiate_template(org, owner, template, ip_block_from, src_filters, temporary,
use_recent_alloc_logic=True, public=False,
use_first_available=False):
"""Instantiates a new Topology object. The IP addresses will come from
ip_block_from. The topology will be assigned the specified source filters.
If use_first_available is True, then the first available block will be used.
Otherwise, a random available block will be used. The latter is generally
better suited to temporary allocations (ones that only last a short time).
A tuple is returned -- if the first element is not None, then an error has
occurred and nothing was instantiated (the first element is an error
message). Otherwise, elements 2-4 are the Topology, IPBlockAllocation, and
PortTreeNode root node objects."""
t = db.Topology()
t.org = org
t.owner = owner
t.template = template
t.enabled = True
t.public = public
t.temporary = temporary
t.ip_block = ip_block_from
DBService.run_and_wait(t.save)
logging.info("Instantiated a new topology for %s from '%s'" % (owner, t.template.name))
for sf_ip, sf_mask in src_filters:
tsif = db.TopologySourceIPFilter()
tsif.topology = t
tsif.ip = sf_ip
tsif.mask = sf_mask
DBService.run_and_wait(tsif.save)
logging.info('IP source filter for new topology %d: %s' % (t.id, tsif))
return (None, t, None, None)
def get_members_id(cls):
"""
:return list: - айди участников
"""
db = DBService()
res = db.select('user', 'tg_id', 'win = 0')
db.close()
return res.fetchall()
def get_task_link(cls, task):
db = DBService()
link = db.select(
'marathon',
query='SELECT t{}_link FROM marathon ORDER BY id DESC LIMIT 0, 1'.
format(task))
db.close()
for i in link:
return i['t{}_link'.format(task)]
def get_user(user_id):
DBService.DBConnect("UserData.db", "User")
try:
selectResult = DBService.select_By_ID(user_id)
except:
abort(404)
finally:
DBService.DBClose()
return jsonify({'users': selectResult})
def delete_user(user_id):
DBService.DBConnect("UserData.db", "User")
try:
DBService.delete(user_id)
except:
abort(404)
finally:
DBService.DBClose()
return jsonify({'result': True})
def is_open_reward(cls):
db = DBService()
rew = db.select(
'marathon',
query='SELECT reward FROM marathon ORDER BY id DESC LIMIT 0, 1')
db.close()
for i in rew:
if i['reward'] == 1:
return True
else:
return False
def get_link(cls):
"""
:return str: - ссылка на марафон
"""
db = DBService()
link = db.select('marathon', 'link', 'status = 0')
db.close()
for i in link:
k = i['link']
return k
def t3_is_finished(cls, user_id):
db = DBService()
tasks = db.select(
'user',
query='SELECT tasks_status FROM user WHERE tg_id={}'.format(
user_id))
db.close()
for i in tasks:
if i['tasks_status'][2] == '0':
return False
else:
return True
def get_balance(cls, tg_id):
"""
:param tg_id: - айди пользователя
:return int: - баланс
"""
db = DBService()
bal = db.select('user', 'points', 'tg_id = {}'.format(tg_id))
db.close()
b = bal.fetchall()
return b[0]['points']
def get_t_available(cls):
"""
:return int: - номер задания
"""
db = DBService()
res = db.select(
'marathon',
query="SELECT t_available FROM marathon ORDER BY id DESC LIMIT 0, 1"
)
db.close()
stat = res.fetchall()
return stat[0]['t_available']
def get_t6_status(cls, tg_id):
"""
-1 - нет первогого голоса, иначе хранит первый голос
:param tg_id: - айди пользователя
:return int: - статус 6 задания
"""
db = DBService()
stat = db.select('user', 't6_status', 'tg_id = {}'.format(tg_id))
db.close()
for i in stat:
res = i['t6_status']
return res
def update_user(user_id):
if not request.json or not 'name' in request.json or not 'birthday' in request.json:
abort(404)
DBService.DBConnect("UserData.db", "User")
try:
DBService.update(user_id,
[request.json['name'], request.json['birthday']])
except:
abort(404)
finally:
DBService.DBClose()
return jsonify({'result': True})
def is_win(cls, tg_id):
"""
:param tg_id: - айди пользователя
:return True: - если уже участвовал
:return False:- ещё не участвовал
"""
db = DBService()
res = db.select('user', 'win', 'tg_id = {}'.format(tg_id))
stat = res.fetchall()
if str(stat[0]['win']) == '0':
return False
else:
return True
def deallocate_from_topology(topo):
"""Removes any IP allocations and assignments from a topology."""
# Check that the topology is not active
try:
_ = DBService.run_and_wait(lambda:db.UsageStats.objects.get(topo_uuid=topo.uuid, active=True))
except db.UsageStats.DoesNotExist:
pass
else:
raise IPError("Cannot deallocate addresses while topology is in use")
iba = DBService.run_and_wait(lambda:db.IPBlockAllocation.objects.get(topology=topo))
iba.delete()
assignments = DBService.run_and_wait(lambda:db.IPAssignment.objects.filter(topology=topo))
for ia in assignments:
ia.delete()
def start_topology(self, tid, client_ip, user):
"""Handles starting up the specified topology id. Returns a 2-tuple.
The first element is None and the second is a string if an error occurs;
otherwise the first element is the topology."""
try:
topo = DBService.run_and_wait(lambda: Topology(tid, self.raw_socket, client_ip, user))
topo.interactors = [] # list of TI connections to this topo
except TopologyCreationException as e:
return (None, str(e))
except db.Topology.DoesNotExist:
return (None, 'topology %d does not exist' % tid)
except db.IPAssignment.DoesNotExist:
return (None, 'topology %d is missing an IP assignment' % tid)
except db.IPBlockAllocation.DoesNotExist:
return (None, 'topology %d is not allocated any IPs' % tid)
except AddressAllocation.IPError:
return (None, 'not enough IPs to allocate for topology %d' % tid)
except:
msg = 'topology instantiation unexpectedly failed'
log_exception(logging.ERROR, msg)
return (None, msg)
if topo.has_gateway():
self.resolver.register_topology(topo)
with self.topologies_lock:
self.topologies[tid] = topo
self.topologies_changed = True
return (topo, None)
def add_user():
if not request.json or not 'name' in request.json or not 'birthday' in request.json:
abort(404)
DBService.DBClose()
DBService.DBConnect("UserData.db", "User")
try:
id = DBService.insert([request.json['name'], request.json['birthday']])
selectResult = DBService.select_By_ID(id)
except:
abort(404)
finally:
DBService.DBClose()
return jsonify({'users': selectResult})
def free_topology(tid):
"""Deletes the topology associated with tid, as well as any IPAssignment,
MACAssignment, TopologySourceIPFilter, TopologyUserFilter, and
IPBlockAllocation objects belonging to it."""
def run():
try:
topo = db.Topology.objects.get(pk=tid)
db.TopologySourceIPFilter.objects.filter(topology=topo).delete()
db.TopologyUserFilter.objects.filter(topology=topo).delete()
db.IPAssignment.objects.filter(topology=topo).delete()
db.MACAssignment.objects.filter(topology=topo).delete()
db.IPBlockAllocation.objects.filter(topology=topo).delete()
topo.delete()
logging.info('freed topology %d' % tid)
except db.Topology.DoesNotExist:
logging.warning('asked to free non-existent topology %d' % tid)
DBService.run_background(run)
def send_motd_to_client(self, conn):
"""Sends a message to a newly connected client, if such a a message is set."""
# see if there is any admin message to be sent to a client upon connecting
try:
msg_for_client = DBService.run_and_wait(lambda: db.SystemInfo.objects.get(name='motd')).value
logging.info('sending message to clients: %s' % msg_for_client)
for m in VNSBanner.get_banners(msg_for_client):
conn.send(m)
except db.SystemInfo.DoesNotExist:
pass
def is_active(cls):
"""
:return True: - если марафон идёт
:return False:- ещё не начался
"""
db = DBService()
stat = db.select(
'marathon',
query="SELECT status FROM marathon ORDER BY id DESC LIMIT 0, 1")
db.close()
if str(stat.rowcount) == '0':
return False
stat = stat.fetchall()
if stat[0]['status'] == 1:
return False
elif stat[0]['status'] == 0:
return True
def periodic_callback(self):
# save statistics values
for topo in self.topologies.values():
stats = topo.get_stats()
stats_deferred = DBService.run_in_db_thread(stats.save_if_changed)
stats_deferred.addCallback(lambda c: self.__stop_topo_if_necessary(topo, c))
# see if there is any admin message to be sent to all clients
try:
bts = DBService.run_and_wait(db.SystemInfo.objects.get(name='banner_to_send'))
msg_for_clients = bts.value
bts.delete()
logging.info('sending message to clients: %s' % msg_for_clients)
for conn in self.clients.keys():
for m in VNSBanner.get_banners(msg_for_clients):
conn.send(m)
except db.SystemInfo.DoesNotExist:
pass
# note in the db that the reactor thread is still running
def __update_reactor_alive_time():
try:
latest = db.SystemInfo.objects.get(name='last_alive_time')
except db.SystemInfo.DoesNotExist:
latest = db.SystemInfo()
latest.name = 'last_alive_time'
latest.value = str(int(time()))
latest.save()
DBService.run_in_db_thread(__update_reactor_alive_time)
# see if there are any topologies to be deleted
jtd = DBService.run_and_wait(db.JournalTopologyDelete.objects.all)
for je in jtd:
try:
topo = self.topologies[je.topology.id]
except KeyError:
# this topology is not in use, we can just delete it from
# the db - this also deletes the journal entry
DBService.run_in_db_thread(je.topology.delete)
else:
# the topology is in use; stop it and then delete it
self.stop_topology(topo, "topology has been deleted")
DBService.run_in_db_thread(je.topology.delete)
reactor.callLater(30, self.periodic_callback)
def is_exist(cls, tg_id):
"""
:param tg_id: - айди пользователя
:return True: - уже существует
:return False:- не существует
"""
db = DBService()
stat = db.select(
'user',
query='SELECT EXISTS (SELECT * FROM user WHERE tg_id = {})'.format(
tg_id))
db.close()
if str(stat.fetchall()[0][
'EXISTS (SELECT * FROM user WHERE tg_id = {})'.format(
tg_id)]) == '0':
return False
else:
return True
def cleanup_and_exit(self):
"""Cleanly terminate connected clients and then forcibly terminate the program."""
# see if the admin put a reason for the shutdown in the database
try:
why = DBService.run_and_wait(lambda:db.SystemInfo.objects.get(name='shutdown_reason')).value
except db.SystemInfo.DoesNotExist:
why = 'the simulator is shutting down'
logging.info('VNS simulator shutting down: %s' % why)
for conn in self.clients.keys():
self.terminate_connection(conn, why)
os._exit(0) # force the termination (otherwise the pcap thread keeps going)
def finish(cls):
"""
заканчивает марафон
"""
db = DBService()
db.update('marathon', {'status': 1}, 'status = 0')
db.close()
def open_task(cls, task):
"""
открывает задание в бд
:param task: - номер задания
"""
db = DBService()
db.update('marathon', {'t_available': task}, 'status = 0')
db.close()
def set_task_link(cls, task, link):
"""
:param task: - номер задания
:param link: - ссылка
"""
db = DBService()
db.update('marathon', {'t{}_link'.format(task): link}, 'status = 0')
db.close()
def create(cls, tg_id):
"""
добавляет пользователя в бд
:param tg_id: - айди пользователя
"""
db = DBService()
db.insert('user', {'tg_id': tg_id})
db.close()
def set_t6_status(cls, tg_id, stat):
"""
:param tg_id: - айди пользователя
:param stat: - статус
"""
db = DBService()
db.update('user', {'t6_status': stat}, 'tg_id = {}'.format(tg_id))
db.close()
def __allocs_filter(alloc, other_ip_mask_list):
"""Returns True if this any of the IP source filters associated with the
topology using alloc overlaps with the IP source filters in other_ip_mask_list."""
if not other_ip_mask_list:
return True # empty list => 0/0 => overlaps with everything
alloc_src_filters = DBService.run_and_wait(lambda:db.TopologySourceIPFilter.objects.filter(topology=alloc.topology))
if not alloc_src_filters:
return True # empty list => 0/0 => overlaps with everything
for asf in alloc_src_filters:
if is_any_overlapping(__str_ip_to_int(asf.ip), asf.mask, other_ip_mask_list):
return True
return False
def is_task_finished(cls, tg_id, task):
"""
:param tg_id: - айди пользователя
:param task: - номер задания
:return True: - задание выполнено
:return False:- задание не выполнено
"""
db = DBService()
stat = db.select('user', 'tasks_status', 'tg_id = {}'.format(tg_id))
db.close()
stat = stat.fetchall()
if task != 3:
if stat[0]['tasks_status'][task - 1] == '0':
return False
else:
return True
else:
if int(stat[0]['tasks_status'][task - 1]) < 5:
return False
else:
return True
def delete_user(cls, tg_id):
"""
param tg_id: телеграм айди юзера
function deletes user from database
"""
db = DBService()
db.select('user',
query='DELETE FROM user WHERE tg_id = {}'.format(tg_id))
db.close()
def __realloc_if_available_work(ra, ip_block_from):
# Start a database query for the IP block allocations which we'll need later
closest_pre_alloc_wait = DBService.run_background(lambda:db.IPBlockAllocation.objects.filter(start_addr__lte=ra.start_addr).order_by('-start_addr')[0])
print closest_pre_alloc_wait
# the recent allocation must be from the block we're trying to allocate from
start_addr = struct.unpack('>I', inet_aton(ra.start_addr))[0]
start_addr_from = struct.unpack('>I', inet_aton(ip_block_from.subnet))[0]
if not is_overlapping(start_addr, ra.mask, start_addr_from, ip_block_from.mask):
return None
# the recent allocation must not be in use
try:
# does the closest active allocation BEFORE the recent alloc overlap it?
closest_pre_alloc = closest_pre_alloc_wait()
sa_pre = struct.unpack('>I', inet_aton(closest_pre_alloc.start_addr))[0]
if is_overlapping(start_addr, ra.mask, sa_pre, closest_pre_alloc.mask):
return None
# does the closest active allocation AFTER to the recent alloc overlap it?
closest_post_alloc = DBService.run_and_wait(lambda:db.IPBlockAllocation.objects.filter(start_addr__gte=ra.start_addr).order_by('start_addr')[0])
sa_post = struct.unpack('>I', inet_aton(closest_post_alloc.start_addr))[0]
if is_overlapping(start_addr, ra.mask, sa_post, closest_post_alloc.mask):
return None
except IndexError:
pass
# it isn't in use => allocate it
new_alloc = db.IPBlockAllocation()
new_alloc.block_from = ip_block_from
new_alloc.topology = None
new_alloc.start_addr = ra.start_addr
new_alloc.mask = ra.mask
DBService.run_and_wait(new_alloc.save)
logging.info('RE-allocated new block of addresses: %s' % new_alloc)
return new_alloc
def start(cls, link):
"""
начинает новый марафон
"""
db = DBService()
db.insert(
'marathon',
{'link': link},
)
db.close()
def __init__(self):
# Initialise the DB thread
DBService.start()
# close out any hanging stats records (shouldn't be any unless the
# server was shutdown abnormally with no chance to cleanup)
DBService.run_and_wait(lambda: db.UsageStats.objects.filter(active=True).update(active=False))
# free any hanging temporary topologies
def __free_hanging_temp_topos(topos):
for t in topos:
AddressAllocation.free_topology(t.id)
DBService.run_and_wait(lambda:__free_hanging_temp_topos(db.Topology.objects.filter(temporary=True)), priority=0)
self.topologies = {} # maps active topology ID to its Topology object
self.resolver = TopologyResolver() # maps MAC/IP addresses to a Topology
self.clients = {} # maps active conn to the topology ID it is conn to
self.server = create_vns_server(VNS_DEFAULT_PORT,
self.handle_recv_msg,
self.handle_new_client,
self.handle_client_disconnected)
self.ti_clients = {} # maps active TI conns to the topology ID it is conn to
self.ti_server = create_ti_server(TI_DEFAULT_PORT,
self.handle_recv_ti_msg,
self.handle_new_client,
self.handle_ti_client_disconnected)
if BORDER_DEV_NAME:
self.__start_raw_socket(BORDER_DEV_NAME)
# run pcap in another thread (it will run forever)
reactor.callInThread(self.__run_pcap, BORDER_DEV_NAME)
else:
self.raw_socket = None
# lock used to prevent self.topologies from being *changed* by the main
# twisted thread while the topology queue service thread is reading it
self.topologies_lock = Lock()
# communicates from the main twisted thread to the topology queue
# service thread that the topologies dictionary has changed
self.topologies_changed = False
# The topology queue service thread will wait on this condition for a
# a chosen/dequeued job to be finish (so it can pick the next one).
self.service_condition = Condition()
# Is set when a job is enqueued. Is cleared when the queues are empty.
# The topology queue service thread will clear this event if it makes a
# a pass over all the queues and they are empty. If it makes a pass
# and this event is cleared, then it will wait on this event.
self.job_available_event = Event()
# run the topology queue service thread
reactor.callInThread(self.__run_topology_queue_service_thread)
self.periodic_callback()
def __realloc_if_available(owner, template, ip_block_from):
"""Checks to see if owner has previously allocated the specified template
from ip_block_from. If so, then previously allocated block is checked to
see if it is available. If so, then it is allocated and returned.
Otherwise, None is returned. Any record of a recent allocation is deleted."""
recent_allocs = DBService.run_and_wait(lambda:db.RecentIPBlockAllocation.objects.filter(user=owner, template=template))
if recent_allocs:
ra = recent_allocs[0]
ret = __realloc_if_available_work(ra, ip_block_from)
if ret:
logging.info('Reallocated %s' % ra)
else:
logging.info('Unable to reallocate %s' % ra)
recent_allocs.delete()
return ret
else:
return None
def add_points(cls, tg_id, points):
"""
выполняет задание пользователю
:param tg_id: - айди пользователя
:param points:- инкремент баланса
"""
db = DBService()
db.select(
'user',
query="update user set points = points + {} where tg_id = {}".
format(points, tg_id))
db.close()
def finish_task(cls, tg_id, task):
"""
выполняет задание пользователю
:param tg_id: - айди пользователя
:param task: - номер задания
"""
db = DBService()
stat = db.select('user', 'tasks_status', 'tg_id = {}'.format(tg_id))
stat = stat.fetchall()
s = stat[0]['tasks_status']
if task != 3:
data = s[:task - 1] + '1' + s[task:]
else:
data = s[:task - 1] + str(int(s[task - 1]) + 1) + s[task:]
db.update('user', {'tasks_status': data}, 'tg_id = {}'.format(tg_id))
db.close()
def handle_auth_reply(self, conn, ar, terminate_connection):
if not conn.vns_auth_salt:
msg = 'unexpectedly received authentication reply from conn_user=%s ar_user=%s at %s'
terminate_connection(conn, msg % (conn.vns_user_profile, ar.username, conn))
return
try:
up = DBService.run_and_wait(lambda:db.UserProfile.objects.get(user__username=ar.username, retired=False))
except db.UserProfile.DoesNotExist:
logging.info('unrecognized username tried to login: %s' % ar.username)
terminate_connection(conn, "authentication failed")
return
expected = hashlib.sha1(conn.vns_auth_salt + str(up.get_sim_auth_key())).digest()
if ar.ssp != expected:
logging.info('user %s provided an incorrect password' % ar.username)
terminate_connection(conn, "authentication failed")
else:
conn.vns_auth_salt = None # only need one auth reply
conn.vns_authorized = True
conn.vns_user_profile = up
msg = 'authenticated %s as %s' % (conn, ar.username)
conn.send(VNSAuthStatus(True, msg))
def open_reward(cls):
db = DBService()
db.update('marathon', {'reward': 1}, 'status = 0')
db.close()
def get_allusers():
DBService.DBConnect("UserData.db", "User")
selectResult = DBService.select_All()
DBService.DBClose()
return jsonify({'users': selectResult})
response = requests.get(url)
return response.text
# removes urls on references to the concepts
def sanitizeConceptsObject(concepts):
for concept in concepts:
concept.pop('freebase', None)
concept.pop('dbpedia', None)
concept.pop('yago', None)
concept.pop('opencyc', None)
aiService = AIService.AIService()
springerService = SpringerRestService.SpringerService()
dbService = DBService.DBService()
# use this link for help on setting the right query : https://dev.springer.com/adding-constraints
# list of subjects that can be used in query
# Astronomy , Behavioral Sciences , Biomedical Sciences , Business & Management , Chemistry, Climate , Computer Science , Earth Sciences , Economics
# Education & Language , Energy , Engineering , Environmental Sciences , Food Science & Nutrition, Geography , Law , Life Sciences
# Materials , Mathematics , Medicine , Philosophy , Physics , Popular Science , Public Health , Social Sciences , Statistics , Water
# subject names which have a space in between should be enclosed in double quotes for example "Computer Science"
springerService.setQuery('subject:"Education & Language"')
springerService.startFromRecordNumber = 200
# fetch a page from springer service
page = springerService.getNextPage()
count = 0
skipped = 0
def allocate_to_topology(topo, ip_block_from, owner, use_first_available=False, use_recent_alloc_logic=True):
"""Allocates IP addresses to an existing topology. If the topology already
has IPs assigned, this does nothing.
@param topo The Topology to allocate IPs to
@param ip_block_from The IPBlock from which to allocate addresses
@param use_first_available If True, use the first available IP addresses
rather than a random block
@param owner The user who has caused this allocation to take place
@exception IPError If there are not enough IPs to allocate"""
# Check we don't have any IPs already
try:
_ = DBService.run_and_wait(lambda:db.IPBlockAllocation.objects.get(topology=topo))
except db.IPBlockAllocation.DoesNotExist:
pass
else:
# We already have an IP block allocation - do nothing
return
# Get the template this topology was created from, and the topology's source
# filters
template = topo.template
filters = DBService.run_and_wait(lambda:db.TopologySourceIPFilter.objects.filter(topology=topo))
src_filters = [(sif.ip, sif.mask) for sif in filters]
# build a depth-first "tree" of the topology from the port connected to the gateway
root = template.get_root_port()
if not root:
return ("template '%s' has no ports" % template.name,)
try:
tree = root.get_tree(True)
num_addrs = tree.compute_subnet_size()
except:
# topology graph has cycles - just make each link its own /31
tree = None
links = DBService.run_and_wait(lambda:db.Link.objects.filter(port1__node__template=template))
num_addrs = len(links) * 2
# try to give the user the allocation they most recently had
alloc = __realloc_if_available(owner, template, ip_block_from) if use_recent_alloc_logic else None
if not alloc:
# allocate a subblock of IPs for the new topology
allocs = allocate_ip_block(ip_block_from, 1, num_addrs, src_filters, use_first_available)
if not allocs:
raise IPError("Not enough IP addresses to allocate for topology %d" % topo.id)
alloc = allocs[0]
# Assign IP addresses
start_addr = struct.unpack('>I', inet_aton(alloc.start_addr))[0]
if tree:
assignments = tree.assign_addr(start_addr, alloc.size())
else:
assignments = []
for i,link in enumerate(links):
assignments.append((link.port1, start_addr+2*i, 31))
assignments.append((link.port2, start_addr+2*i+1, 31))
# Add the allocation to the database
alloc.topology = topo
alloc.save()
# Add the allocation to the list of recently allocated blocks
if use_recent_alloc_logic:
ra = db.RecentIPBlockAllocation()
ra.user = owner
ra.template = template
ra.start_addr = alloc.start_addr
ra.mask = alloc.mask
# Save and wait for it - we wait because we need the ID for the next bit
DBService.run_and_wait(ra.save)
# Add the assignments to the database
wait_for = []
for port, ip, mask_sz in assignments:
ipa = db.IPAssignment()
ipa.topology = topo
ipa.port = port
ipa.ip = inet_ntoa(struct.pack('>I', ip))
ipa.mask = mask_sz
waiter = DBService.run_background(ipa.save)
wait_for.append(waiter)
logging.info('IP assignment for topology %d: %s' % (topo.id, ipa))
# Wait for all the DB transactions to complete
for waiter in wait_for:
waiter()
def __allocate_ip_block(block_from, num_blocks_to_alloc, min_block_mask_bits, src_filters, use_first_available):
# find the allocations we need to workaround to avoid collisions ("allocations of concern")
db_allocs = DBService.run_and_wait(lambda:db.IPBlockAllocation.objects.filter(block_from=block_from))
allocations = [(__str_ip_to_int(a.start_addr), a.mask) for a in db_allocs]
#ip_mask_list = [(__str_ip_to_int(sf_ip), sf_mask) for sf_ip, sf_mask in src_filters]
#aoc = filter(lambda alloc : __allocs_filter(alloc, ip_mask_list), allocations)
aoc = allocations
aoc.sort()
# add fake start and end usages so we can allocate blocks on the edges too
block_from_start_addr = __str_ip_to_int(block_from.subnet)
aoc.insert(0, (block_from_start_addr-1, 32)) # "use" last addr before block_from
block_from_end_addr_plus_1 = block_from_start_addr + 2 ** (32 - block_from.mask)
aoc.append((block_from_end_addr_plus_1, 32)) # "use" first addr after block_from
# randomize the order in which we consider allocations => less likely to
# reallocate a block soon after it is deallocated => user who wants to reuse
# a particular IP block should be able to more often
if not use_first_available:
split_index = random.randint(0, len(aoc)-1)
aoc = aoc[split_index:] + aoc[:split_index + 1] # +1 => overlap so i,i+1 block is still checked
# iterate over each adjacent pair of used block until a free segment
# (between used blocks) is found
new_allocations = []
num_addrs_needed = 2 ** (32 - min_block_mask_bits)
mask = 0xFFFFFFFF << (32 - min_block_mask_bits)
for i in range(len(aoc) - 1):
# compute the first address after this used block at which the subnet
# can be allocated (i.e., align the start address to the subnet size)
start_addr, num_masked_bits = aoc[i]
sz = 2 ** (32 - num_masked_bits)
faa = start_addr + sz # first available address
aligned_faa = faa & mask
if aligned_faa < faa:
aligned_faa += num_addrs_needed # must start after the used block
start_addr2, _ = aoc[i + 1]
num_addrs_avail = start_addr2 - aligned_faa
logging.debug('considering space b/w %s and %s => %d addresses available' % (__aoc_to_str(aoc[i]), __aoc_to_str(aoc[i+1]), num_addrs_avail))
if num_addrs_avail >= num_addrs_needed:
# if we're not too worried about fragmentation, then choose from
# among the possible sub-blocks in this block
if use_first_available:
aligned_aa = aligned_faa
else:
max_offset = num_addrs_avail / num_addrs_needed
offset = random.randint(0, max_offset-1)
aligned_aa = aligned_faa + (num_addrs_needed * offset)
# create the allocation
new_alloc = db.IPBlockAllocation()
new_alloc.block_from = block_from
new_alloc.topology = None
new_alloc.start_addr = inet_ntoa(struct.pack('>I', aligned_aa))
new_alloc.mask = min_block_mask_bits
DBService.run_and_wait(new_alloc.save)
logging.info('Allocated new block of addresses: %s' % new_alloc)
new_allocations.append(new_alloc)
if len(new_allocations) == num_blocks_to_alloc:
return new_allocations # successfully allocated all requested blocks
# failed to make all of the requested allocations -- insufficient address space
logging.info('Not able to allocate %d blocks of %d addresses each: only got %d blocks' % (num_blocks_to_alloc, num_addrs_needed, len(new_allocations)))
return new_allocations
