def optimize_projection(self, fields):
"""
Propagate a SELECT clause through a FROM Node.
Args:
fields: A set of String instances (queried fields).
"""
if self.capabilities.projection or self.capabilities.fullquery:
self.query.select().select(fields)
if self.capabilities.projection:
# Push fields into the From node
return self
else:
# Provided fields is set to None if it corresponds to SELECT *
provided_fields = self.get_query().get_select()
# Test whether this From node can return every queried Fields.
if provided_fields and fields - provided_fields:
Log.warning(
"From::optimize_projection: some requested fields (%s) are not provided by {%s} From node. Available fields are: {%s}"
% (', '.join(list(fields - provided_fields)),
self.get_query().get_from(), ', '.join(
list(provided_fields))))
# If this From node returns more Fields than those explicitely queried
# (because the projection capability is not enabled), create an additional
# Projection Node above this From Node in order to guarantee that
# we only return queried fields
if not provided_fields or provided_fields - fields:
return Projection(self, fields)
#projection.query = self.query.copy().filter_by(filter) # XXX
return self
def parse_manifest(cls, rspec, rspec_version = 'GENI 3', slice_urn = None, start_time = None):
rspec = RSpec(rspec, version=rspec_version)
_resources = cls._get_resources(rspec)
_nodes = cls._get_nodes(rspec)
# XXX Not supported yet
#_channels = cls._get_channels(rspec)
#_links = cls._get_links(rspec)
_leases = cls._get_leases(rspec)
# XXX Until WiLab supports Leases
end_time = cls._get_expiration(rspec)
if start_time is None:
start_time = 1388530800
resources = list()
resources.extend(cls._process_resources(_resources))
resources.extend(cls._process_nodes(_nodes))
#resources.extend(cls._process_channels(_channels))
#resources.extend(cls._process_links(_links))
Log.warning("XXX Until WiLab supports Leases")
# XXX Generate Leases based on the Resources instead of Leases
leases = cls._process_leases(resources, slice_urn, start_time, end_time)
return {'resource': resources, 'lease': leases }
def right_callback(self, record):
"""
\brief Process records received from the right child
\param record A dictionary representing the received record
"""
if record.is_last():
self._on_right_done()
return
# Skip records missing information necessary to join
#DEPRECATED| if self.predicate.value not in record or not record[self.predicate.value]:
#Log.tmp("%s <= %s" %(set(self.predicate.get_value()) , set(record.keys())))
if not set([self.predicate.get_value()]) <= set(record.keys()) \
or Record.is_empty_record(record, set([self.predicate.get_value()])):
Log.warning("Missing LEFTJOIN predicate %s in right record %r: ignored" % \
(self.predicate, record))
return
# We expect to receive information about keys we asked, and only these,
# so we are confident the key exists in the map
# XXX Dangers of duplicates ?
key = Record.get_value(record, self.predicate.value)
left_records = self.left_map.get(key, None)
if left_records:
for left_record in self.left_map[key]:
left_record.update(record)
self.send(left_record)
del self.left_map[key]
def boot(self):
"""
Boot the Interface (prepare metadata, etc.).
"""
assert isinstance(self.platforms, list), "Invalid platforms"
for platform in self.platforms:
# Get platform configuration
platform_config = platform['config']
if platform_config:
platform_config = json.loads(platform_config)
platform_name = platform['platform']
args = [None, platform_name, None, platform_config, {}, None]
gateway = Gateway.get(platform['gateway_type'])(*args)
try:
announces = gateway.get_metadata()
except Exception, e:
# ROUTERV2
Log.warning("Cannot get metadata for platform %s: %s" % (platform_name, e))
# XXX Disable platform ?
announces = list()
self.metadata[platform_name] = list()
for announce in announces:
self.metadata[platform_name].append(announce)
def get_dialect_and_field_info(self, table):
t = self.config[table]
filename = t['filename']
with open(filename, 'rb') as f:
sample = f.read(1024)
dialect = csv.Sniffer().sniff(sample)
self.has_headers[table] = csv.Sniffer().has_header(sample)
HAS_FIELDS_OK, HAS_FIELDS_KO, HAS_FIELDS_ERR = range(1, 4)
HAS_TYPES_OK, HAS_TYPES_KO, HAS_TYPES_ERR = range(1, 4)
has_fields = HAS_FIELDS_KO
has_types = HAS_TYPES_KO
if isinstance(t, dict):
if 'fields' in t:
try:
field_names, field_types = [], []
for name, type in t['fields']:
field_names.append(name)
has_fields = HAS_FIELDS_OK
field_types.append(type)
has_types = HAS_TYPES_OK
except Exception, e:
Log.warning(
"Wrong format for fields in platform configuration")
has_fields = HAS_FIELDS_ERR
has_types = HAS_TYPES_ERR
def _process_lease(cls, lease):
# Keep only necessary information in leases
new_lease = dict()
authority = 'urn:publicid:IDN+wilab2.ilabt.iminds.be+authority+cm'
if (not 'component_manager_id' in lease) or (lease['component_manager_id'] != authority):
Log.warning("Authority is not WiLab - Ignore lease = ",lease)
#return None
new_lease['resource'] = lease.pop('component_id')
new_lease['lease_id'] = None
new_lease['slice'] = lease.pop('slice_urn')
new_lease['start_time'] = int(lease['start_time'])
new_lease['duration'] = int(lease['duration'])
if 'end_time' in lease:
new_lease['end_time'] = int(lease['end_time'])
if not 'end_time' in lease and set(['start_time', 'duration']) <= set(lease.keys()):
new_lease['end_time'] = lease['start_time'] + lease['duration'] * cls.get_grain()
elif not 'duration' in lease and set(lease.keys()) <= set(['start_time', 'end_time']):
new_lease['duration'] = (lease['end_time'] - lease['start_time']) / cls.get_grain()
# XXX GRANULARITY Hardcoded for the moment
if 'granularity' not in lease:
new_lease['granularity'] = cls.get_grain()
else:
new_lease['granularity'] = lease['granularity']
return new_lease
def xmlrpc_forward(self, request, query, annotations=None):
"""
"""
Log.info("Incoming XMLRPC request, query = %r, annotations = %r" %
(self.display_query(query), annotations))
if Options().disable_auth:
Log.info("Authentication disabled by configuration")
else:
if not annotations or not 'authentication' in annotations:
msg = "You need to specify an authentication token in annotations"
return dict(ResultValue.get_error(ResultValue.FORBIDDEN, msg))
# We expect to find an authentication token in the annotations
if annotations:
auth = annotations.get('authentication', None)
else:
auth = {}
auth['request'] = request
# Check login password
try:
user = Auth(auth, self.interface).check()
except Exception, e:
Log.warning(
"XMLRPCAPI::xmlrpc_forward: Authentication failed...: %s" %
str(e))
msg = "Authentication failed: %s" % e
return dict(ResultValue.get_error(ResultValue.FORBIDDEN, msg))
def xrn_hook(resource):
urn = resource.get('component_id')
if not urn:
Log.warning('No urn !!!')
return resource
resource['urn'] = urn
resource['hrn'] = urn_to_hrn(urn)[0]
return resource
def make_metadata(self):
"""
Prepare metadata (e.g. Tables encapsulated in Announces instances) related
to this Gateway. Metadata are in the generic case retrieved both by
inspecting the pgsql schema and the .h file related to this Gateway.
"""
# Import metadata from pgsql schema.
# By default, we only fetch tables and we ignore views.
########
announces_pgsql = self.make_metadata_from_names(self.get_table_names())
if not announces_pgsql:
Log.warning("Cannot find metadata for platform %s: %s" %
(self.platform, e))
else:
Log.info("Tables imported from pgsql schema: %s" %
[announce.get_table() for announce in announces_pgsql])
###
"""
class table {
string comment;
field fields[];
key keys[];
};
class field {
string comment;
bool is_const;
bool is_array;
string type;
};
class key {
table table; /**< BACKWARD_1N */
field fields[];
};
"""
# 1) router::boot:
# for each gateway:
# if gateway.type == postgresql:
# router.instantiate_gateway(gateway)
# PostgreSQLGateway::__init__():
# self.router.instantiate_gateway(platform = postgresql_metadata, config = config)
# 2) d = self.router.forward(Query.get('object').select([name, field.name, field.type, field.comment, key])
# see core/interface.py:180
# 3) announces_pgsql = Announce.from_dict(d)
# In this gateway inject field info in table info
#########
# Fetch metadata from .h files (if any)
announces_h = Announces.from_dot_h(self.get_platform(),
self.get_gateway_type())
Log.info("Tables imported from .h schema: %s" %
[announce.get_table() for announce in announces_h])
# Return the resulting announces
return self.merge_announces(
announces_pgsql, announces_h) if announces_h else announces_pgsql
def inject_at(self, query):
"""
Update From Nodes of the QueryPlan in order to take into account AT
clause involved in a user Query.
Args:
query: The Query issued by the user.
"""
Log.warning("HARDCODED: AT injection in FROM Nodes: %r" % self.froms)
for from_node in self.froms:
from_node.query.timestamp = query.get_timestamp()
def get_slicename(self, filters):
# XXX If slicename is not in WHERE of the Query it will cause an error
filters = {'value': filters['slice_hrn']}
fields = ['name']
plc_api = xmlrpclib.ServerProxy(API_URL, allow_none=True)
result = plc_api.GetSliceTags(self._get_auth(), filters, fields)
if not result:
Log.warning("No Slice name for this hrn ", filters)
return None
else:
return result[0]['name']
def get_location(cls, city):
location = None
try:
#from geopy.geocoders import Nominatim
#geolocator = Nominatim()
#from geopy.geocoders import GeoNames
#geolocator = GeoNames()
from geopy.geocoders import GoogleV3
geolocator = GoogleV3()
location = geolocator.geocode(city)
except Exception, e:
Log.warning("geopy.geocoders failed to get coordinates for city = ",city)
Log.warning(e)
def get_cache(self, annotations=None):
user = annotations.get('user')
user_id = user.get('user_id') if user else None
if not user_id:
# Use global cache
Log.warning("Use of global cache for query, annotations=%r" %
(annotations, ))
return self._cache
# Use per-user cache
if user_id not in self._cache_user:
self._cache_user[user_id] = Cache()
return self._cache_user[user_id]
def build_rspec(cls,
slice_hrn,
resources,
leases,
flowspace,
vms,
rspec_version=None):
Log.warning("NitosBroker Parser build")
rspec = []
cls.rspec_add_header(rspec)
lease_map = cls.rspec_add_leases(rspec, leases)
cls.rspec_add_resources(rspec, resources, lease_map)
cls.rspec_add_footer(rspec)
return "\n".join(rspec)
def left_callback(self, record):
"""
\brief Process records received by the left child
\param record A dictionary representing the received record
"""
if record.is_last():
# left_done. Injection is not the right way to do this.
# We need to insert a filter on the key in the right member
predicate = Predicate(self.predicate.get_value(), included,
self.left_map.keys())
if self.right.get_query().action == ACTION_CREATE:
# XXX If multiple insert, we need to match the right ID with the
# right inserted items
if len(self.left_map.keys()) > 1:
raise NotImplemented
# Pass the id as a param
keys = self.left_map.keys()
if not keys:
# No JOIN possible
self.left_done = True
self._on_right_done()
return
key = self.left_map.keys()[0]
query = self.right.get_query()
query.params[self.predicate.get_value()] = key
else: # pass the id as a filter which is the normal behaviour
self.right = self.right.optimize_selection(
Filter().filter_by(predicate))
self.right.set_callback(
self.right_callback) # already done in __init__ ?
self.left_done = True
self.right.start()
return
# Directly send records missing information necessary to join
# XXXX !!! XXX XXX XXX
if not Record.has_fields(record, self.predicate.get_field_names()):
Log.warning("Missing LEFTJOIN predicate %s in left record %r : forwarding" % \
(self.predicate, record))
self.send(record)
# Store the result in a hash for joining later
hash_key = Record.get_value(record, self.predicate.key)
if not hash_key in self.left_map:
self.left_map[hash_key] = []
self.left_map[hash_key].append(record)
def _process_resource(cls, resource):
"""
Postprocess resources read from the RSpec. This applies to nodes, channels and links.
In particular, it sets the urn, hrn, network_hrn, facility_name and testbed_name fields.
"""
urn = resource['component_id']
hrn, type = urn_to_hrn(resource['component_id'])
resource['urn'] = urn
resource['hrn'] = hrn
resource['network_hrn'] = Xrn(resource['component_id']).authority[0] # network ? XXX
# We also add 'facility' and 'testbed' fields
resource['facility_name'] = cls.get_resource_facility_name(urn)
resource['testbed_name'] = cls.get_resource_testbed_name(urn)
if 'exclusive' not in resource:
resource['exclusive'] = 'true'
elif resource['exclusive'] is None:
resource['exclusive'] = 'true'
else:
Log.warning("EXCLUSIVE = ",resource['exclusive'])
#if 'location' in node:
# if node['location']:
# node['latitude'] = node['location']['latitude']
# node['longitude'] = node['location']['longitude']
# del node['location']
#else:
# if the location is not provided, aproximate it from the city
t_urn = resource['urn'].split('+')
city = t_urn[3].split('.')[1]
if city == 'iii':
city = 'Institute for Information Industry, Taïwan 106'
resource['country'] = 'Taiwan'
else:
resource['country'] = 'France'
location = cls.get_location(city)
if location is not None:
resource['latitude'] = str(location.latitude)
resource['longitude'] = str(location.longitude)
return resource
def check(self):
# Method.type_check() should have checked that all of the
# mandatory fields were present.
assert self.auth.has_key('Username')
# Get record (must be enabled)
try:
query_users = Query.get('local:user').filter_by('email', '==', self.auth['Username'].lower())
user, = self.interface.execute_local_query(query_users)
except Exception, e:
import traceback
traceback.print_exc()
Log.warning("Authentication failed, delete expired sessions")
query_sessions = Query.delete('local:session').filter_by('expires', '<', int(time.time()))
try:
self.interface.execute_local_query(query_sessions)
except: pass
raise AuthenticationFailure, "No such account (PW): %s" % e
def filter(self, query, record, annotations, is_query=True):
# TMP CACHE DEBUG
#import pdb
#pdb.set_trace()
for rule in self.rules:
if not rule.match(query, annotations):
continue
target = Target.get(rule.target)
if not target:
Log.warning("Unknown target %s" % rule.target)
continue
# TODO: ROUTERV2
# Cache per user
# Adding interface in order to access router.get_cache(annotations)
decision, data = target(self._interface).process(
query, record, annotations, is_query)
if decision == TargetValue.ACCEPT:
return (self.ACCEPT, None)
elif decision == TargetValue.REWRITE:
return (self.REWRITE, data)
elif decision == TargetValue.CACHE_HIT:
return (self.CACHE_HIT, data)
elif decision == TargetValue.DENIED:
return (self.DENIED, None)
elif decision == TargetValue.ERROR:
return (self.ERROR, data)
elif decision == TargetValue.CONTINUE:
continue
## Let's create a cache entry
#if is_query:
# # We are dealing with queries
# cache = self._interface.get_cache(annotations)
# # XXX TEMP HACK
# try:
# cache.new_entry(query)
# except Exception,e:
# Log.warning(e)
# Default decision : ACCEPT
return (self.ACCEPT, None)
def optimize_selection(self, filter):
parent_filter, top_filter = Filter(), Filter()
for predicate in filter:
if predicate.get_field_names() <= self.parent.get_query(
).get_select():
parent_filter.add(predicate)
else:
Log.warning(
"SubQuery::optimize_selection() is only partially implemented : %r"
% predicate)
top_filter.add(predicate)
if parent_filter:
self.parent = self.parent.optimize_selection(parent_filter)
self.parent.set_callback(self.parent_callback)
if top_filter:
return Selection(self, top_filter)
return self
def merge_announces(self, announces_pgsql, announces_h):
# Merge metadata
s = PostgreSQLGateway.get_colliding_announces(announces_pgsql,
announces_h)
if s:
Log.warning(
"merge_announces: colliding announces for table(s): {%s}" %
", ".join(s))
announces = list()
announces.extend(announces_h)
table_names = [
announce.get_table().get_name() for announce in announces_h
]
for announce_pgsql in announces_pgsql:
table_name = announce_pgsql.get_table().get_name()
if table_name not in table_names:
announces.append(announces_pgsql)
table_names.append(table_name)
return announces
def _get_expiration(cls, rspec):
# get the expires tag in the header of the RSpec
# convert it to timestamp
# XXX Until WiLab supports Leases
# this will be used as lease['end_time']
try:
rspec_string = rspec.toxml()
import xml.etree.ElementTree as ET
rspec = ET.fromstring(rspec_string)
expiration = rspec.get("expires")
import time
from datetime import datetime
ret = int(time.mktime(datetime.strptime(expiration, "%Y-%m-%dT%H:%M:%SZ").timetuple()))
return ret
# XXX To be removed in Router-v2
except Exception, e:
import traceback
Log.warning("Exception in _get_expiration: %s" % e)
traceback.print_exc()
return None
def _process_leases(cls, leases, slice_urn, start_time, end_time):
ret = list()
try:
for lease in leases:
lease['slice_urn'] = slice_urn
lease['start_time'] = start_time
lease['end_time'] = end_time
# duration in seconds from now till end_time
duration = end_time - start_time
# duration in minutes
duration = duration / 60
lease['duration'] = int(duration)
new_lease = cls._process_lease(lease)
if not new_lease:
continue
ret.append(new_lease)
# XXX To be removed in Router-v2
except Exception, e:
import traceback
Log.warning("Exception in _process_leases: %s" % e)
traceback.print_exc()
def _process_link(cls, link):
authority = 'urn:publicid:IDN+wall2.ilabt.iminds.be+authority+cm'
if (not 'component_manager' in link) or (link['component_manager'] != authority):
Log.warning("Authority is not wall2 - Ignore link = ",link)
#return None
return super(WiLabtParser, cls)._process_link(link)
def parse(cls, rspec, rspec_version=None, slice_urn=None):
resources = list()
leases = list()
rspec = RSpec(rspec)
# Parse leases first, so that they can be completed when encountering
# their ids in resources
lease_map = dict() # id -> lease_dict
elements = rspec.xml.xpath('//ol:lease')
network = cls.get_network()
for el in elements:
try:
lease_tmp = cls.dict_from_elt(network, el.element)
start = calendar.timegm(
dateutil.parser.parse(
lease_tmp['valid_from']).utctimetuple())
end = calendar.timegm(
dateutil.parser.parse(
lease_tmp['valid_until']).utctimetuple())
lease = {
'lease_id': lease_tmp['id'],
'slice': slice_urn,
'start_time': start,
'end_time': end,
'duration': (end - start) / cls.get_grain(),
'granularity': cls.get_grain()
}
lease_map[lease_tmp['id']] = lease
except:
import traceback
Log.warning("this lease has not the right format")
traceback.print_exc()
# Parse nodes
for tag, resource_type in RESOURCE_TYPES.items():
if ':' in tag:
ns, _, tag = tag.partition(':')
XPATH_RESOURCE = "//%(ns)s:%(tag)s"
else:
XPATH_RESOURCE = "//default:%(tag)s | //%(tag)s"
elements = rspec.xml.xpath(XPATH_RESOURCE % locals())
for el in elements:
resource = cls.dict_from_elt(network, el.element,
LIST_ELEMENTS.get(resource_type))
if resource_type in cls.MAP:
resource = cls.dict_rename(resource, resource_type)
resource['network_hrn'] = network
#resource['facility_name'] = 'NITOS'
resource['facility_name'] = 'Wireless'
if 'hrn' in resource:
hrn = resource['hrn'].replace('\\', '')
elif 'component_id' in resource:
urn = resource['component_id']
hrn, type = urn_to_hrn(urn)
resource['hrn'] = hrn
resource['urn'] = urn
t_hrn = hrn.split('.')
if 'omf' in hrn:
if t_hrn[1] == 'nitos':
resource['testbed_name'] = 'NITOS-Volos'
resource['country'] = 'Greece'
resource['latitude'] = '39.3666667'
resource['longitude'] = '22.9458333'
elif t_hrn[1] == 'netmode':
resource['testbed_name'] = 'Netmode'
resource['country'] = 'Greece'
resource['latitude'] = '37.978377'
resource['longitude'] = '23.782707'
elif t_hrn[1] == 'gaia':
resource['testbed_name'] = 'UMU'
resource['country'] = 'Spain'
resource['latitude'] = '38.023681'
resource['longitude'] = '-1.174462'
elif t_hrn[1] == 'snu':
resource['testbed_name'] = 'SNU'
resource['country'] = 'South Korea'
resource['latitude'] = '37.460263'
resource['longitude'] = '126.953598'
elif t_hrn[1] == 'kaist':
resource['testbed_name'] = 'KAIST'
resource['country'] = 'South Korea'
resource['latitude'] = '36.369362'
resource['longitude'] = '127.364025'
elif t_hrn[1] == 'etri':
resource['testbed_name'] = 'ETRI'
resource['country'] = 'South Korea'
resource['latitude'] = '36.3808653'
resource['longitude'] = '127.3650244'
elif 'paris' in hrn:
resource[
'testbed_name'] = 'FIT NITOS ' + t_hrn[1].title()
# XXX Hardcoded for Paris @ UPMC
resource['country'] = 'France'
resource['latitude'] = '48.847104'
resource['longitude'] = '2.357499'
else:
resource['testbed_name'] = 'FIT ' + t_hrn[1].title()
resource['country'] = 'France'
else:
if 'component_manager_id' in resource:
t, tp = urn_to_hrn(resource['component_manager_id'])
resource['testbed_name'] = 'FIT ' + t.title()
else:
resource['testbed_name'] = 'FIT ' + t_hrn[1].title()
#resource['testbed_name'] = 'Nitos'
if 'services.login.username' in resource:
if resource['services.login.username']:
resource['login'] = {
'username': resource['services.login.username'],
'hostname': resource['services.login.hostname']
}
resource = set_status(resource)
resources.append(resource)
# Leases
if 'lease_ref.id_ref' in resource:
lease_id_refs = resource.pop('lease_ref.id_ref')
for lease_id_ref in lease_id_refs:
lease = copy.deepcopy(lease_map[lease_id_ref])
lease['resource'] = resource['urn']
leases.append(lease)
return {'resource': resources, 'lease': leases}
def build(self,
query,
metadata,
allowed_platforms,
allowed_capabilities,
user=None):
"""
Build the QueryPlan involving several Gateways according to a 3nf
graph and a user Query. If only one Gateway is involved, you should
use QueryPlan::build_simple.
Args:
query: The Query issued by the user.
metadata: The 3nf graph (DBGraph instance).
allowed_platforms: A list of platform names (list of String).
Which platforms the router is allowed to query.
Could be used to restrict the Query to a limited set of platforms
either because it is specified by the user Query or either due
to the Router configuration.
allowed_capabilities: A Capabilities instance or None.
Specify which capabilities the Router can perform if it is
involved as an intermediate Router between two other Routers.
TODO: metadata, allowed_platforms and allowed_platforms capabilities should be
deduced from the query, the router, and the user.
router.get_metadata()
router.get_allowed_platforms(query, user)
router.get_allowed_capabilities(query, user)
user: A User instance or None.
"""
root = metadata.find_node(query.get_from())
if not root:
raise RuntimeError(
"query_plan::build(): Cannot find %s in metadata, known tables are %s"
% (query.get_from(), sorted(set(metadata.get_table_names()))))
root_task = ExploreTask(root,
relation=None,
path=[],
parent=self,
depth=1)
root_task.addCallback(self.set_ast, query)
stack = Stack(root_task)
seen = {} # path -> set()
missing_fields = set()
missing_fields |= query.get_select()
missing_fields |= query.get_where().get_field_names()
missing_fields |= set(query.get_params().keys())
while missing_fields:
task = stack.pop()
if not task:
# Exploration ends here
Log.warning(
"Exploration terminated without finding fields: %r" %
missing_fields)
break
pathstr = '.'.join(task.path)
if not pathstr in seen:
seen[pathstr] = set()
# ROUTERV2
# foreign_key_fields are fields added because indirectly requested by the user.
# For example, he asked for slice.resource, which in fact will contain slice.resource.urn
foreign_key_fields = task.explore(stack,
missing_fields,
metadata,
allowed_platforms,
allowed_capabilities,
user,
seen[pathstr],
query_plan=self)
self.foreign_key_fields.update(foreign_key_fields)
while not stack.is_empty():
task = stack.pop()
task.cancel()
def main(self):
"""
\brief Runs a XMLRPC server
"""
Log.info("XMLRPC server daemon (%s) started." % sys.argv[0])
# NOTE it is important to import those files only after daemonization,
# since they open files we cannot easily preserve
from twisted.web import xmlrpc, server
# SSL support
from OpenSSL import SSL
from twisted.internet import ssl #, reactor
#from twisted.internet.protocol import Factory, Protocol
#from twisted.internet import reactor
# This also imports manifold.util.reactor_thread that uses reactor
from manifold.core.router import Router
assert not (Options().platform and Options().gateway), "Both gateway and platform cannot be specified at commandline"
# This imports twisted code so we need to import it locally
from manifold.core.xmlrpc_api import XMLRPCAPI
# This should be configurable
allowed_capabilities = Capabilities()
allowed_capabilities.selection = True
allowed_capabilities.projection = True
# XXX We should harmonize interfaces between Router and Forwarder
if Options().platform:
platforms = Storage.execute(Query().get('platform'), format='object')
# We pass a single platform to Forwarder
platform = [p for p in platforms if p.name == Options().platform][0]
self.interface = Forwarder(platform, allowed_capabilities)
elif Options().gateway:
# XXX user
# XXX Change Forwarded initializer
#DEPRECATED| platform = Platform(u'dummy', Options().gateway, self.get_gateway_config(Options().gateway), 'user')
platform = Platform(
platform = u'dummy',
gateway_type = Options().gateway,
config = self.get_gateway_config(Options().gateway),
auth_type = 'user'
)
self.interface = Forwarder(platform, allowed_capabilities)
else:
self.interface = Router()
try:
def verifyCallback(connection, x509, errnum, errdepth, ok):
if not ok:
print 'invalid cert from subject:', x509.get_subject()
print errnum, errdepth
return False
else:
print "Certs are fine", x509, x509.get_subject()
return True
ssl_path = Options().ssl_path
if not ssl_path or not os.path.exists(ssl_path):
print ""
print "You need to generate SSL keys and certificate in '%s' to be able to run manifold" % ssl_path
print ""
print "mkdir -p /etc/manifold/keys"
print "openssl genrsa 1024 > /etc/manifold/keys/server.key"
print "chmod 400 /etc/manifold/keys/server.key"
print "openssl req -new -x509 -nodes -sha1 -days 365 -key /etc/manifold/keys/server.key > /etc/manifold/keys/server.cert"
print ""
sys.exit(0)
server_key_file = "%s/server.key" % ssl_path
server_crt_file = "%s/server.cert" % ssl_path
Log.tmp("key, cert=", server_key_file, server_crt_file)
myContextFactory = ssl.DefaultOpenSSLContextFactory(server_key_file, server_crt_file)
ctx = myContextFactory.getContext()
ctx.set_verify(
SSL.VERIFY_PEER, # | SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
verifyCallback
)
# Since we have self-signed certs we have to explicitly
# tell the server to trust them.
#ctx.load_verify_locations("keys/ca.pem")
trusted_roots_path = Options().trusted_roots_path
if not trusted_roots_path or not os.path.exists(trusted_roots_path):
Log.warning("No trusted root found in %s. You won't be able to login using SSL client certificates" % trusted_roots_path)
ctx.load_verify_locations(None, ssl_path)
#ReactorThread().listenTCP(Options().xmlrpc_port, server.Site(XMLRPCAPI(self.interface, allowNone=True)))
ReactorThread().listenSSL(Options().xmlrpc_port, server.Site(XMLRPCAPI(self.interface, allowNone=True)), myContextFactory)
ReactorThread().start_reactor()
except Exception, e:
# TODO If database gets disconnected, we can sleep/attempt reconnection
Log.error("Error in XMLRPC API: %s" % str(e))
def get_relations(self, table, dbgraph):
"""
Compute which Relations connect the "self" Table (source node) to the
"table" Table (target node). We assume that the graph of table is
at least 2nf.
Args:
table: The target table
Returns:
A set of Relation instances connecting "self" and "table".
This set is empty iif the both Tables are unrelated.
"""
# We only test relations u --> v
u = self
v = table
relations = set()
u_key = u.keys.one()
v_key = v.keys.one()
if u.get_name() == v.get_name():
p = Predicate(u_key.get_name(), eq, v_key.get_name())
if u.get_platforms() > v.get_platforms():
relations.add(Relation(Relation.types.PARENT, p))
#else:
# relations.add(Relation(Relation.types.CHILD, p))
return relations
# Detect explicit Relation from u to v
for field in u.get_fields():
# 1. A field in u is explicitly typed againt v name
if field.get_type() == v.get_name():
if v_key.is_composite():
# We assume that u (for ex: traceroute) provides in the current field (ex: hops)
# a record containing at least the v's key (for ex: (agent, destination, first, ttl))
intersecting_fields = tuple(u.get_field_names() & v_key.get_field_names())
key_fields = list(v_key.get_field_names())
prefixed_key_fields = ["%s.%s" % (field.get_name(), kf) for kf in key_fields]
# Plain wrong
# p = Predicate(intersecting_fields, eq, intersecting_fields)
p = Predicate(prefixed_key_fields, eq, key_fields)
else:
p = Predicate(field.get_name(), eq, v_key.get_name())
if field.is_array():
relations.add(Relation(Relation.types.LINK_1N, p, name=field.get_name())) # LINK_1N_FORWARD
else:
if False: # field == key
relations.add(Relation(Relation.types.PARENT, p, name=field.get_name())) # in which direction ?????
else:
if field.is_local():
relations.add(Relation(Relation.types.LINK_11, p, name=field.get_name()))
else:
if v.is_child_of(u):
relations.add(Relation(Relation.types.CHILD, p))
elif u.is_child_of(v):
relations.add(Relation(Relation.types.PARENT, p))
else:
if field.get_name() in ['source', 'destination', 'agent', 'dns_target']:
Log.warning("Hardcoded source, agent, destination and dns_target as 1..1 relationships")
relations.add(Relation(Relation.types.LINK_11, p, name=field.get_name()))
else:
relations.add(Relation(Relation.types.LINK, p))
# BAD
#if v_key.is_composite():
# Log.warning("Link (2) unsupported between u=%s and v=%s: v has a composite key" % (u.get_name(), v.get_name()))
## 2. A field is typed like the key
#if field.get_type() == v_key.get_field().get_type():
# # In this case we might have inheritance
# # We should point to the toplevel class, ie. if key field type == platform name
# # We are back to the previous case.
# # a child class is an instance of the parent class, no it should be ok
# (3) A field point to part of v key (v is thus composite)
if field.get_type() not in BASE_TYPES and set([field.get_type()]) < v_key.get_field_types():
# What if several possible combinations
# How to consider inheritance ?
vfield = [f for f in v_key if f.get_type() == field.get_type()][0]
# XXX This case is not well supported, let's skip for now
if v.get_name() == "lease":
continue
p = Predicate(field.get_name(), eq, vfield.get_name())
relations.add(Relation(Relation.types.LINK_1N, p, name=field.get_name())) # LINK_1N_FORWARD ?
continue
# Following relations don't involve a single field
# (4) A bit more complex: u presents the set of fields that make up v key
# (5) A bit more complex: u presents part of the fields that make up v key
if relations:
return relations
# Detect implicit Relation from u to v
intersection = u.get_field_names() & v_key.get_field_names()
if intersection and intersection < v_key.get_field_names():
intersection = tuple(intersection)
if len(intersection) == 1:
intersection = intersection[0]
p = Predicate(intersection, eq, intersection)
relations.add(Relation(Relation.types.LINK_1N, p, name=v.get_name())) # LINK_1N_FORWARD # Name ?
# we don't continue otherwise we will find subsets of this set
# note: this code might replace following code operating on a single field
return relations
# --- REVERSE RELATIONS
for field in v.get_fields():
# (6) inv of (1) a field in v points to an existing type
# we could say we only look at key types at this stage
if field.get_type() == u.get_name():
if u_key.is_composite():
Log.warning("Link (6) unsupported between u=%s and v=%s: u has a composite key" % (u.get_name(), v.get_name()))
continue
p = Predicate(u_key.get_name(), eq, field.get_name())
if field.is_array():
relations.add(Relation(Relation.types.LINK_1N_BACKWARDS, p, name = v.get_name()))
### was: COLLECTION, p)) # a u is many v ? approve this type
#relations.add(Relation(Relation.types.COLLECTION, p)) # a u is many v ? approve this type
else:
# if u parent
if v.is_child_of(u):
relations.add(Relation(Relation.types.CHILD, p))
elif u.is_child_of(v):
relations.add(Relation(Relation.types.PARENT, p))
else:
relations.add(Relation(Relation.types.LINK_1N, p, name=v.get_name())) # LINK_1N_BACKWARDS
return relations
def get_user_config(self, user, platform):
# all are dict
platform_name = platform['platform']
platform_id = platform['platform_id']
auth_type = platform.get('auth_type', None)
if not auth_type:
Log.warning("'auth_type' is not set in platform = %s" % platform_name)
return None
# XXX platforms might have multiple auth types (like pam)
# XXX we should refer to storage
if auth_type in ["none", "default"]:
user_config = {}
# For default, take myslice account
elif auth_type == 'user':
if not user:
raise Exception, "Cannot use auth_type user when no user is configured"
user_id = user['user_id'] if user else None
# User account information
query_accounts = Query.get('local:account').filter_by('user_id', '==', user_id).filter_by('platform_id', '==', platform_id)
accounts = self.execute_local_query(query_accounts)
if accounts:
account = accounts[0]
user_config = account.get('config', None)
if user_config:
user_config = json.loads(user_config)
# XXX This should disappear with the merge with router-v2
if account['auth_type'] == 'reference':
ref_platform_name = user_config['reference_platform']
query_ref_platform = Query.get('local:platform').filter_by('platform', '==', ref_platform_name)
ref_platforms = self.execute_local_query(query_ref_platform)
if not ref_platforms:
raise Exception, 'Cannot find reference platform %s for platform %s' % (platform_name, ref_platform_name)
ref_platform = ref_platforms[0]
query_ref_account = Query.get('local:account').filter_by('user_id', '==', user_id).filter_by('platform_id', '==', ref_platform['platform_id'])
ref_accounts = self.execute_local_query(query_ref_account)
if not ref_accounts:
raise Exception, 'Cannot find account information for reference platform %s' % ref_platform_name
ref_account = ref_accounts[0]
user_config = ref_account.get('config', None)
if user_config:
user_config = json.loads(user_config)
else:
user_config = {}
else:
raise ValueError("This 'auth_type' not supported: %s" % auth_type)
return user_config
def _process_channel(cls, channel):
authority = 'urn:publicid:IDN+wall2.ilabt.iminds.be+authority+cm'
if (not 'component_manager_id' in channel) or (channel['component_manager_id'] != authority):
Log.warning("Authority is not wall2 - Ignore channel = ",channel)
#return None
return super(WiLabtParser, cls)._process_channel(channel)
def _process_node(cls, node):
authority = 'urn:publicid:IDN+wall2.ilabt.iminds.be+authority+cm'
if (not 'component_manager_id' in node) or (node['component_manager_id'] != authority):
Log.warning("Authority is not wall2 - Ignore node = ",node)
#return None
return super(WiLabtParser, cls)._process_node(node)
