def push_props_to_isl(tx, subject, *fields):
if not fields:
return
copy_fields = {
name: 'source.' + name for name in fields}
q = textwrap.dedent("""
MATCH (source:link_props)
WHERE source.src_switch = $src_switch
AND source.src_port = $src_port
AND source.dst_switch = $dst_switch
AND source.dst_port = $dst_port
MATCH
(:switch {name: source.src_switch})
-
[target:isl {
src_switch: source.src_switch,
src_port: source.src_port,
dst_switch: source.dst_switch,
dst_port: source.dst_port
}]
->
(:switch {name: source.dst_switch})
""") + db.format_set_fields(
db.escape_fields(copy_fields, raw_values=True),
field_prefix='target.')
p = _make_match(subject)
db.log_query('link props to ISL', q, p)
tx.run(q, p)
def create_if_missing(tx, timestamp, *links):
q = textwrap.dedent("""
MATCH (src:switch {name: $src_switch})
MATCH (dst:switch {name: $dst_switch})
MERGE (src) - [target:isl {
src_switch: $src_switch,
src_port: $src_port,
dst_switch: $dst_switch,
dst_port: $dst_port
}] -> (dst)
ON CREATE SET
target.status=$status, target.actual=$status,
target.latency=-1,
target.time_create=$timestamp,
target.time_modify=$timestamp
ON MATCH SET target.time_modify=$timestamp""")
for isl in sorted(links):
for target in (isl, isl.reversed()):
p = _make_match(target)
p['status'] = 'inactive'
p['timestamp'] = str(timestamp)
logger.info('Ensure ISL %s exists', target)
db.log_query('create ISL', q, p)
tx.run(q, p)
def sync_with_link_props(tx, isl, *fields):
copy_fields = {name: 'source.' + name for name in fields}
q = textwrap.dedent("""
MATCH (source:link_props)
WHERE source.src_switch = $src_switch
AND source.src_port = $src_port
AND source.dst_switch = $dst_switch
AND source.dst_port = $dst_port
MATCH
(:switch {name: $src_switch})
-
[target:isl {
src_switch: $src_switch,
src_port: $src_port,
dst_switch: $dst_switch,
dst_port: $dst_port
}]
->
(:switch {name: $dst_switch})
""") + db.format_set_fields(
db.escape_fields(copy_fields, raw_values=True), field_prefix='target.')
p = _make_match(isl)
db.log_query('propagate link props to ISL', q, p)
tx.run(q, p)
def get_old_flow(new_flow):
query = (
"MATCH (a:switch) - [r:flow {"
" flowid: $flowid "
"}] -> (b:switch) "
"WHERE r.cookie <> $cookie "
"RETURN r ")
params = {
'flowid': new_flow['flowid'],
'cookie': int(new_flow['cookie'])
}
db.log_query('Get old flow', query, params)
old_flows = graph.run(query, params).data()
if not old_flows:
message = 'Flow {} not found'.format(new_flow['flowid'])
logger.error(message)
# TODO (aovchinnikov): replace with specific exception.
raise Exception(message)
else:
logger.info('Flows were found: %s', old_flows)
for data in old_flows:
old_flow = hydrate_flow(data['r'])
logger.info('check cookies: %s ? %s',
new_flow['cookie'], old_flow['cookie'])
if is_same_direction(new_flow['cookie'], old_flow['cookie']):
logger.info('Flow was found: flow=%s', old_flow)
return dict(old_flow)
# FIXME(surabujin): use custom exception!!!
raise Exception(
'Requested flow {}(cookie={}) don\'t found corresponding flow (with '
'matching direction in Neo4j)'.format(
new_flow['flowid'], new_flow['cookie']))
def delete_flow_segments(flow, tx=None):
"""
Whenever adjusting flow segments, always update available bandwidth. Even when creating a flow
where we might remove anything old and then create the new .. it isn't guaranteed that the
old segments are the same as the new segements.. so update bandwidth to be save.
"""
flow_path = get_flow_path(flow)
flowid = flow['flowid']
parent_cookie = flow['cookie']
logger.debug('DELETE Flow Segments : flowid: %s parent_cookie: 0x%x [path: %s]', flowid, parent_cookie, flow_path)
delete_segment_query = (
"MATCH (:switch) - [fs:flow_segment { "
" flowid: $flowid, "
" parent_cookie: $parent_cookie "
"}] -> (:switch) "
"DELETE fs")
params = {
'flowid': flowid,
'parent_cookie': parent_cookie
}
db.log_query('Delete flow segments', delete_segment_query, params)
if tx:
tx.run(delete_segment_query, params)
else:
graph.run(delete_segment_query, params)
update_flow_segment_available_bw(flow, tx)
def del_props(tx, isl, props):
logger.info('ISL drop %s props request: %s', isl,
', '.join(repr(x) for x in props))
remove = ['target.{}'.format(db.escape(x)) for x in props]
if not remove:
return
remove.insert(0, '')
p = _make_match(isl)
q = textwrap.dedent("""
MATCH
(:switch {name: $src_switch})
-
[target:isl {
src_switch: $src_switch,
src_port: $src_port,
dst_switch: $dst_switch,
dst_port: $dst_port
}]
->
(:switch {name: $dst_switch})""") + '\nREMOVE '.join(remove)
db.log_query('ISL drop props', q, p)
stats = tx.run(q, p).stats()
if 'max_bandwidth' in props:
db_isl = fetch(tx, isl)
set_props(tx, isl,
{'max_bandwidth': db_isl.get('default_max_bandwidth', 0)})
return stats['contains_updates']
def drop_by_mask(tx, mask):
logger.info('Delete link props by mask %s', mask)
p = _make_match_by_mask(mask)
if not p:
raise exc.UnacceptableDataError(
mask, 'reject to drop all link props in DB')
where = ['target.{0}=${0}'.format(x) for x in p]
q = 'MATCH (target:link_props)\n'
if where:
q += 'WHERE ' + '\n AND '.join(where)
q += '\nRETURN target, id(target) as ref'
db.log_query('pre delete link props fetch', q, p)
refs = []
persistent = []
for db_record in tx.run(q, p):
persistent.append(model.LinkProps.new_from_db(db_record['target']))
refs.append(db_record['ref'])
q = 'MATCH (target:link_props)\n'
q += 'WHERE id(target) in [{}]'.format(', '.join(str(x) for x in refs))
q += '\nDELETE target'
db.log_query('delete link props', q, {})
tx.run(q)
return persistent
def fetch(tx, isl):
p = _make_match(isl)
q = textwrap.dedent("""
MATCH
(:switch {name: $src_switch})
-
[target:isl {
src_switch: $src_switch,
src_port: $src_port,
dst_switch: $dst_switch,
dst_port: $dst_port
}]
->
(:switch {name: $dst_switch})
RETURN target""")
db.log_query('ISL fetch', q, p)
cursor = tx.run(q, p)
try:
target = db.fetch_one(cursor)['target']
except exc.DBEmptyResponse:
raise exc.DBRecordNotFound(q, p)
return target
def remove_flow(flow, parent_tx=None):
"""
Deletes the flow and its flow segments. Start with flow segments (symmetrical mirror of store_flow).
Leverage a parent transaction if it exists, otherwise create / close the transaction within this function.
- flowid **AND** cookie are *the* primary keys for a flow:
- both the forward and the reverse flow use the same flowid
NB: store_flow is used for uni-direction .. whereas flow_id is used both directions .. need cookie to differentiate
"""
logger.info('Remove flow: %s', flow['flowid'])
tx = parent_tx if parent_tx else graph.begin()
delete_flow_segments(flow, tx)
query = (
"MATCH (:switch) - [f:flow {"
" flowid: $flowid, "
" cookie: $cookie "
"}] -> (:switch) "
"DELETE f")
params = {
'flowid': flow['flowid'],
'cookie': flow['cookie']
}
db.log_query('Remove flow', query, params)
tx.run(query, params).data()
if not parent_tx:
tx.commit()
def update_config():
q = 'MERGE (target:config {name: "config"})\n'
q += db.format_set_fields(db.escape_fields(
{x: '$' + x for x in features_status_app_to_transport_map.values()},
raw_values=True), field_prefix='target.')
p = {
y: features_status[x]
for x, y in features_status_app_to_transport_map.items()}
with graph.begin() as tx:
db.log_query('CONFIG update', q, p)
tx.run(q, p)
def read_config():
q = 'MATCH (target:config {name: "config"}) RETURN target LIMIT 2'
db.log_query('CONFIG read', q, None)
with graph.begin() as tx:
cursor = tx.run(q)
try:
config_node = db.fetch_one(cursor)['target']
for feature, name in features_status_app_to_transport_map.items():
features_status[feature] = config_node[name]
except exc.DBEmptyResponse:
logger.info(
'There is no persistent config in DB, fallback to'
' builtin defaults')
def activate_switch(self):
switch_id = self.payload['switch_id']
logger.info('Switch %s activation request: timestamp=%s',
switch_id, self.timestamp)
with graph.begin() as tx:
flow_utils.precreate_switches(tx, switch_id)
q = 'MATCH (target:switch {name: $dpid}) SET target.state="active"'
p = {'dpid': switch_id}
db.log_query('SWITCH activate', q, p)
tx.run(q, p)
def get_flow_by_id_and_cookie(flow_id, cookie):
query = ("MATCH ()-[r:flow]->() WHERE r.flowid=$flow_id "
"and r.cookie=$cookie RETURN r")
params = {'flow_id': flow_id, 'cookie': cookie}
db.log_query('Get flow by id and cookie', query, params)
result = graph.run(query, params).data()
if not result:
return
flow = hydrate_flow(result[0]['r'])
logger.debug('Found flow for id %s and cookie %s: %s', flow_id, cookie,
flow)
return flow
def precreate_switches(tx, *nodes):
switches = [x.lower() for x in nodes]
switches.sort()
for dpid in switches:
query = (
"MERGE (sw:switch {name: $dpid}) "
"ON CREATE SET sw.state='inactive' "
"ON MATCH SET sw.tx_override_workaround='dummy'")
params = {
'dpid': dpid
}
db.log_query('Precreate switches', query, params)
tx.run(query, params)
def get_flows_by_src_switch(switch_id):
query = ("MATCH (sw:switch)-[r:flow]->(:switch) "
"WHERE sw.name=$switch_id RETURN r")
params = {'switch_id': switch_id}
db.log_query('Get flows by src switch', query, params)
result = graph.run(query, params).data()
flows = []
for item in result:
flows.append(hydrate_flow(item['r']))
logger.debug('Found flows for switch %s: %s', switch_id, flows)
return flows
def get_flow_segment_by_src_switch_and_cookie(switch_id, cookie):
query = ("MATCH p = (sw:switch)-[fs:flow_segment]->(:switch) "
"WHERE sw.name=$switch_id AND fs.cookie=$cookie "
"RETURN fs")
params = {'switch_id': switch_id, 'cookie': cookie}
db.log_query('Get flow segment by src switch and cookie', query, params)
result = graph.run(query, params).data()
if not result:
return
segment = result[0]['fs']
logger.debug('Found segment for switch %s and cookie %s: %s', switch_id,
cookie, segment)
return segment
def set_props(tx, subject):
db_subject = fetch(tx, subject)
origin, update = db.locate_changes(db_subject, subject.props_db_view())
if update:
q = textwrap.dedent("""
MATCH (target:link_props)
WHERE id(target)=$target_id
""") + db.format_set_fields(
db.escape_fields(update), field_prefix='target.')
p = {'target_id': db.neo_id(db_subject)}
db.log_query('propagate link props to ISL', q, p)
tx.run(q, p)
return origin, update.keys()
def get_flow_segments_by_dst_switch(switch_id):
query = ("MATCH p = (:switch)-[fs:flow_segment]->(sw:switch) "
"WHERE sw.name=$switch_id "
"RETURN fs")
params = {'switch_id': switch_id}
db.log_query('Get flow segments by dst switch', query, params)
result = graph.run(query, params).data()
# group flow_segments by parent cookie, it is helpful for building
# transit switch rules
segments = []
for relationship in result:
segments.append(relationship['fs'])
logger.debug('Found segments for switch %s: %s', switch_id, segments)
return segments
def set_link_props(tx, isl, props):
target = fetch_link_props(tx, isl)
origin, update = _locate_changes(target, props)
if update:
q = textwrap.dedent("""
MATCH (target:link_props)
WHERE id(target)=$target_id
""") + db.format_set_fields(db.escape_fields(update),
field_prefix='target.')
p = {'target_id': db.neo_id(target)}
db.log_query('link_props set props', q, p)
tx.run(q, p)
sync_with_link_props(tx, isl, *update.keys())
return origin
def fetch(tx, subject):
p = _make_match(subject)
q = textwrap.dedent("""
MATCH (target:link_props {
src_switch: $src_switch,
src_port: $src_port,
dst_switch: $dst_switch,
dst_port: $dst_port})
RETURN target""")
db.log_query('link props update', q, p)
cursor = tx.run(q, p)
try:
db_object = db.fetch_one(cursor)['target']
except exc.DBEmptyResponse:
raise exc.DBRecordNotFound(q, p)
return db_object
def drop(tx, subject):
logger.info("Delete %s request", subject)
q = textwrap.dedent("""
MATCH (target:link_props)
WHERE target.src_switch=$src_switch,
AND target.src_port=$src_port,
AND target.dst_port=$dst_port,
AND target.dst_switch=$dst_switch
DELETE target
RETURN target""")
p = _make_match(subject)
db.log_query('delete link props', q, p)
cursor = tx.run(q, p)
try:
db_subject = db.fetch_one(cursor)['target']
except exc.DBEmptyResponse:
raise exc.DBRecordNotFound(q, p)
return model.LinkProps.new_from_db(db_subject)
def fetch_flow_segments(flowid, parent_cookie):
"""
:param flowid: the ID for the entire flow, typically consistent across updates, whereas the cookie may change
:param parent_cookie: the cookie for the flow as a whole; individual segments may vary
:return: array of segments
"""
fetch_query = (
"MATCH (:switch) - [fs:flow_segment { "
" flowid: $flowid,"
" parent_cookie: $parent_cookie "
"}] -> (:switch) "
"RETURN fs "
"ORDER BY fs.seq_id")
params = {
'flowid': flowid,
'parent_cookie': parent_cookie
}
db.log_query('Fetch flow segments', fetch_query, params)
# This query returns type py2neo.types.Relationship .. it has a dict method to return the properties
result = graph.run(fetch_query, params).data()
return [dict(x['fs']) for x in result]
def update_isl_bandwidth(src_switch, src_port, dst_switch, dst_port, tx=None):
"""
This will update the available_bandwidth for the isl that matches the src/dst information.
It does this by looking for all flow segments over the ISL, where ignore_bandwidth = false.
Because there may not be any segments, have to use "OPTIONAL MATCH"
"""
# print('Update ISL Bandwidth from %s:%d --> %s:%d' % (src_switch, src_port, dst_switch, dst_port))
available_bw_query = (
"MATCH "
" (src:switch {name: $src_switch}), "
" (dst:switch {name: $dst_switch}) "
"WITH src,dst "
"MATCH (src) - [i:isl { "
" src_port: $src_port, "
" dst_port: $dst_port "
"}] -> (dst) "
"WITH src,dst,i "
"OPTIONAL MATCH (src) - [fs:flow_segment { "
" src_port: $src_port, "
" dst_port: $dst_port, "
" ignore_bandwidth: false "
"}] -> (dst) "
"WITH sum(fs.bandwidth) AS used_bandwidth, i as i "
"SET i.available_bandwidth = i.max_bandwidth - used_bandwidth ")
logger.debug('Update ISL Bandwidth from %s:%d --> %s:%d' % (src_switch, src_port, dst_switch, dst_port))
params = {
'src_switch': src_switch,
'src_port': src_port,
'dst_switch': dst_switch,
'dst_port': dst_port,
}
db.log_query('Update ISL bandwidth', available_bw_query, params)
if tx:
tx.run(available_bw_query, params)
else:
graph.run(available_bw_query, params)
def create_switch(self):
switch_id = self.payload['switch_id']
logger.info('Switch %s creation request: timestamp=%s',
switch_id, self.timestamp)
with graph.begin() as tx:
flow_utils.precreate_switches(tx, switch_id)
p = {
'address': self.payload['address'],
'hostname': self.payload['hostname'],
'description': self.payload['description'],
'controller': self.payload['controller'],
'state': 'active'}
q = 'MATCH (target:switch {name: $dpid})\n' + db.format_set_fields(
db.escape_fields(
{x: '$' + x for x in p}, raw_values=True),
field_prefix='target.')
p['dpid'] = switch_id
db.log_query('SWITCH create', q, p)
tx.run(q, p)
def create_if_missing(tx, *batch):
q = textwrap.dedent("""
MERGE (target:link_props {
src_switch: $src_switch,
src_port: $src_port,
dst_port: $dst_port,
dst_switch: $dst_switch})
ON CREATE SET
target.time_create=$time_create,
target.time_modify=$time_modify
ON MATCH SET target.time_modify=$time_modify""")
for link in sorted(batch):
p = _make_match(link)
time_fields = [link.time_create, link.time_modify]
time_fields = [
str(x) if isinstance(x, model.TimeProperty) else x
for x in time_fields]
p['time_create'], p['time_modify'] = time_fields
logger.info('Ensure link property %s exists', link)
db.log_query('create link props', q, p)
tx.run(q, p)
def merge_flow_relationship(flow, tx):
q = (
"MERGE (src:switch {name: $src_switch}) "
" ON CREATE SET src.state = 'inactive' "
"MERGE (dst:switch {name: $dst_switch}) "
" ON CREATE SET dst.state = 'inactive' "
"MERGE (src)-[f:flow {"
" flowid: $flowid, "
" cookie: $cookie } ]->(dst)"
"SET f.src_switch = src.name, "
" f.src_port = $src_port, "
" f.src_vlan = $src_vlan, "
" f.dst_switch = dst.name, "
" f.dst_port = $dst_port, "
" f.dst_vlan = $dst_vlan, "
" f.meter_id = $meter_id, "
" f.bandwidth = $bandwidth, "
" f.ignore_bandwidth = $ignore_bandwidth, "
" f.periodic_pings = $periodic_pings, "
" f.transit_vlan = $transit_vlan, "
" f.description = $description, "
" f.last_updated = $last_updated, "
" f.flowpath = $flowpath"
)
p = model.dash_to_underscore(flow)
# FIXME(surabujin): do we really want to keep this time representation?
# FIXME(surabujin): format datetime as '1532609693'(don 't match with
# format used in PCE/resource cache)
p['last_updated'] = str(int(time.time()))
path = p['flowpath'].copy()
path.pop('clazz', None)
p['flowpath'] = json.dumps(path)
db.log_query('Save(update) flow', q, p)
tx.run(q, p)
def merge_flow_segments(_flow, tx=None):
"""
This function creates each segment relationship in a flow, and then it calls the function to
update bandwidth. This should always be down when creating/merging flow segments.
To create segments, we leverages the flow path .. and the flow path is a series of nodes, where
each 2 nodes are the endpoints of an ISL.
"""
flow = copy.deepcopy(_flow)
create_segment_query = (
"MERGE "
" (src:switch {name: $src_switch}) "
"ON CREATE SET src.state='inactive' "
"MERGE "
" (dst:switch {name: $dst_switch}) "
"ON CREATE SET dst.state='inactive' "
"MERGE (src) - [fs:flow_segment {"
" flowid: $flowid, "
" parent_cookie: $parent_cookie "
"}] -> (dst) "
"SET "
" fs.cookie=$cookie, "
" fs.src_switch=$src_switch, "
" fs.src_port=$src_port, "
" fs.dst_switch=$dst_switch, "
" fs.dst_port=$dst_port, "
" fs.seq_id=$seq_id, "
" fs.segment_latency=$segment_latency, "
" fs.bandwidth=$bandwidth, "
" fs.ignore_bandwidth=$ignore_bandwidth ")
flow_path = get_flow_path(flow)
flow_cookie = flow['cookie']
flow['parent_cookie'] = flow_cookie # primary key of parent is flowid & cookie
logger.debug('MERGE Flow Segments : %s [path: %s]', flow['flowid'], flow_path)
for i in range(0, len(flow_path), 2):
src = flow_path[i]
dst = flow_path[i+1]
# <== SRC
flow['src_switch'] = src['switch_id']
flow['src_port'] = src['port_no']
flow['seq_id'] = src['seq_id']
# Ignore latency if not provided
flow['segment_latency'] = src.get('segment_latency', 'NULL')
# ==> DEST
flow['dst_switch'] = dst['switch_id']
flow['dst_port'] = dst['port_no']
# Allow for per segment cookies .. see if it has one set .. otherwise use the cookie of the flow
# NB: use the "dst cookie" .. since for flow segments, the delete rule will use the dst switch
flow['cookie'] = dst.get('cookie', flow_cookie)
db.log_query('Merge flow segment', create_segment_query, flow)
# TODO: Preference for transaction around the entire delete
# TODO: Preference for batch command
if tx:
tx.run(create_segment_query, flow)
else:
graph.run(create_segment_query, flow)
update_flow_segment_available_bw(flow, tx)
def update_status(tx, isl, mtime=True):
logging.info("Sync status both sides of ISL %s to each other", isl)
q = textwrap.dedent("""
MATCH
(:switch {name: $src_switch})
-
[self:isl {
src_switch: $src_switch,
src_port: $src_port,
dst_switch: $dst_switch,
dst_port: $dst_port
}]
->
(:switch {name: $dst_switch})
MATCH
(:switch {name: $peer_src_switch})
-
[peer:isl {
src_switch: $peer_src_switch,
src_port: $peer_src_port,
dst_switch: $peer_dst_switch,
dst_port: $peer_dst_port
}]
->
(:switch {name: $peer_dst_switch})
WITH self, peer, CASE
WHEN self.actual = $status_up AND peer.actual = $status_up
THEN $status_up
WHEN self.actual = $status_moved OR peer.actual = $status_moved
THEN $status_moved
ELSE $status_down
END AS isl_status
SET self.status=isl_status
SET peer.status=isl_status""")
p = {
'status_up': 'active',
'status_moved': 'moved',
'status_down': 'inactive'
}
p.update(_make_match(isl))
p.update({'peer_' + k: v for k, v in _make_match(isl.reversed()).items()})
expected_update_properties_count = 2
if mtime:
if not isinstance(mtime, model.TimeProperty):
mtime = model.TimeProperty.now()
q += '\nSET self.time_modify=$mtime, peer.time_modify=$mtime'
p['mtime'] = str(mtime)
expected_update_properties_count = 4
db.log_query('ISL update status', q, p)
cursor = tx.run(q, p)
cursor.evaluate() # to fetch first record
stats = cursor.stats()
if stats['properties_set'] != expected_update_properties_count:
logger.error(
'Failed to sync ISL\'s %s records statuses. Looks like it is '
'unidirectional.', isl)
