def optimize_projection(self, fields): """ Propagate a SELECT clause through a FROM Node. Args: fields: A set of String instances (queried fields). """ if self.capabilities.projection or self.capabilities.fullquery: self.query.select().select(fields) if self.capabilities.projection: # Push fields into the From node return self else: # Provided fields is set to None if it corresponds to SELECT * provided_fields = self.get_query().get_select() # Test whether this From node can return every queried Fields. if provided_fields and fields - provided_fields: Log.warning( "From::optimize_projection: some requested fields (%s) are not provided by {%s} From node. Available fields are: {%s}" % (', '.join(list(fields - provided_fields)), self.get_query().get_from(), ', '.join( list(provided_fields)))) # If this From node returns more Fields than those explicitely queried # (because the projection capability is not enabled), create an additional # Projection Node above this From Node in order to guarantee that # we only return queried fields if not provided_fields or provided_fields - fields: return Projection(self, fields) #projection.query = self.query.copy().filter_by(filter) # XXX return self
def parse_manifest(cls, rspec, rspec_version = 'GENI 3', slice_urn = None, start_time = None): rspec = RSpec(rspec, version=rspec_version) _resources = cls._get_resources(rspec) _nodes = cls._get_nodes(rspec) # XXX Not supported yet #_channels = cls._get_channels(rspec) #_links = cls._get_links(rspec) _leases = cls._get_leases(rspec) # XXX Until WiLab supports Leases end_time = cls._get_expiration(rspec) if start_time is None: start_time = 1388530800 resources = list() resources.extend(cls._process_resources(_resources)) resources.extend(cls._process_nodes(_nodes)) #resources.extend(cls._process_channels(_channels)) #resources.extend(cls._process_links(_links)) Log.warning("XXX Until WiLab supports Leases") # XXX Generate Leases based on the Resources instead of Leases leases = cls._process_leases(resources, slice_urn, start_time, end_time) return {'resource': resources, 'lease': leases }
def right_callback(self, record): """ \brief Process records received from the right child \param record A dictionary representing the received record """ if record.is_last(): self._on_right_done() return # Skip records missing information necessary to join #DEPRECATED| if self.predicate.value not in record or not record[self.predicate.value]: #Log.tmp("%s <= %s" %(set(self.predicate.get_value()) , set(record.keys()))) if not set([self.predicate.get_value()]) <= set(record.keys()) \ or Record.is_empty_record(record, set([self.predicate.get_value()])): Log.warning("Missing LEFTJOIN predicate %s in right record %r: ignored" % \ (self.predicate, record)) return # We expect to receive information about keys we asked, and only these, # so we are confident the key exists in the map # XXX Dangers of duplicates ? key = Record.get_value(record, self.predicate.value) left_records = self.left_map.get(key, None) if left_records: for left_record in self.left_map[key]: left_record.update(record) self.send(left_record) del self.left_map[key]
def boot(self): """ Boot the Interface (prepare metadata, etc.). """ assert isinstance(self.platforms, list), "Invalid platforms" for platform in self.platforms: # Get platform configuration platform_config = platform['config'] if platform_config: platform_config = json.loads(platform_config) platform_name = platform['platform'] args = [None, platform_name, None, platform_config, {}, None] gateway = Gateway.get(platform['gateway_type'])(*args) try: announces = gateway.get_metadata() except Exception, e: # ROUTERV2 Log.warning("Cannot get metadata for platform %s: %s" % (platform_name, e)) # XXX Disable platform ? announces = list() self.metadata[platform_name] = list() for announce in announces: self.metadata[platform_name].append(announce)
def get_dialect_and_field_info(self, table): t = self.config[table] filename = t['filename'] with open(filename, 'rb') as f: sample = f.read(1024) dialect = csv.Sniffer().sniff(sample) self.has_headers[table] = csv.Sniffer().has_header(sample) HAS_FIELDS_OK, HAS_FIELDS_KO, HAS_FIELDS_ERR = range(1, 4) HAS_TYPES_OK, HAS_TYPES_KO, HAS_TYPES_ERR = range(1, 4) has_fields = HAS_FIELDS_KO has_types = HAS_TYPES_KO if isinstance(t, dict): if 'fields' in t: try: field_names, field_types = [], [] for name, type in t['fields']: field_names.append(name) has_fields = HAS_FIELDS_OK field_types.append(type) has_types = HAS_TYPES_OK except Exception, e: Log.warning( "Wrong format for fields in platform configuration") has_fields = HAS_FIELDS_ERR has_types = HAS_TYPES_ERR
def _process_lease(cls, lease): # Keep only necessary information in leases new_lease = dict() authority = 'urn:publicid:IDN+wilab2.ilabt.iminds.be+authority+cm' if (not 'component_manager_id' in lease) or (lease['component_manager_id'] != authority): Log.warning("Authority is not WiLab - Ignore lease = ",lease) #return None new_lease['resource'] = lease.pop('component_id') new_lease['lease_id'] = None new_lease['slice'] = lease.pop('slice_urn') new_lease['start_time'] = int(lease['start_time']) new_lease['duration'] = int(lease['duration']) if 'end_time' in lease: new_lease['end_time'] = int(lease['end_time']) if not 'end_time' in lease and set(['start_time', 'duration']) <= set(lease.keys()): new_lease['end_time'] = lease['start_time'] + lease['duration'] * cls.get_grain() elif not 'duration' in lease and set(lease.keys()) <= set(['start_time', 'end_time']): new_lease['duration'] = (lease['end_time'] - lease['start_time']) / cls.get_grain() # XXX GRANULARITY Hardcoded for the moment if 'granularity' not in lease: new_lease['granularity'] = cls.get_grain() else: new_lease['granularity'] = lease['granularity'] return new_lease
def xmlrpc_forward(self, request, query, annotations=None): """ """ Log.info("Incoming XMLRPC request, query = %r, annotations = %r" % (self.display_query(query), annotations)) if Options().disable_auth: Log.info("Authentication disabled by configuration") else: if not annotations or not 'authentication' in annotations: msg = "You need to specify an authentication token in annotations" return dict(ResultValue.get_error(ResultValue.FORBIDDEN, msg)) # We expect to find an authentication token in the annotations if annotations: auth = annotations.get('authentication', None) else: auth = {} auth['request'] = request # Check login password try: user = Auth(auth, self.interface).check() except Exception, e: Log.warning( "XMLRPCAPI::xmlrpc_forward: Authentication failed...: %s" % str(e)) msg = "Authentication failed: %s" % e return dict(ResultValue.get_error(ResultValue.FORBIDDEN, msg))
def xrn_hook(resource): urn = resource.get('component_id') if not urn: Log.warning('No urn !!!') return resource resource['urn'] = urn resource['hrn'] = urn_to_hrn(urn)[0] return resource
def make_metadata(self): """ Prepare metadata (e.g. Tables encapsulated in Announces instances) related to this Gateway. Metadata are in the generic case retrieved both by inspecting the pgsql schema and the .h file related to this Gateway. """ # Import metadata from pgsql schema. # By default, we only fetch tables and we ignore views. ######## announces_pgsql = self.make_metadata_from_names(self.get_table_names()) if not announces_pgsql: Log.warning("Cannot find metadata for platform %s: %s" % (self.platform, e)) else: Log.info("Tables imported from pgsql schema: %s" % [announce.get_table() for announce in announces_pgsql]) ### """ class table { string comment; field fields[]; key keys[]; }; class field { string comment; bool is_const; bool is_array; string type; }; class key { table table; /**< BACKWARD_1N */ field fields[]; }; """ # 1) router::boot: # for each gateway: # if gateway.type == postgresql: # router.instantiate_gateway(gateway) # PostgreSQLGateway::__init__(): # self.router.instantiate_gateway(platform = postgresql_metadata, config = config) # 2) d = self.router.forward(Query.get('object').select([name, field.name, field.type, field.comment, key]) # see core/interface.py:180 # 3) announces_pgsql = Announce.from_dict(d) # In this gateway inject field info in table info ######### # Fetch metadata from .h files (if any) announces_h = Announces.from_dot_h(self.get_platform(), self.get_gateway_type()) Log.info("Tables imported from .h schema: %s" % [announce.get_table() for announce in announces_h]) # Return the resulting announces return self.merge_announces( announces_pgsql, announces_h) if announces_h else announces_pgsql
def inject_at(self, query): """ Update From Nodes of the QueryPlan in order to take into account AT clause involved in a user Query. Args: query: The Query issued by the user. """ Log.warning("HARDCODED: AT injection in FROM Nodes: %r" % self.froms) for from_node in self.froms: from_node.query.timestamp = query.get_timestamp()
def get_slicename(self, filters): # XXX If slicename is not in WHERE of the Query it will cause an error filters = {'value': filters['slice_hrn']} fields = ['name'] plc_api = xmlrpclib.ServerProxy(API_URL, allow_none=True) result = plc_api.GetSliceTags(self._get_auth(), filters, fields) if not result: Log.warning("No Slice name for this hrn ", filters) return None else: return result[0]['name']
def get_location(cls, city): location = None try: #from geopy.geocoders import Nominatim #geolocator = Nominatim() #from geopy.geocoders import GeoNames #geolocator = GeoNames() from geopy.geocoders import GoogleV3 geolocator = GoogleV3() location = geolocator.geocode(city) except Exception, e: Log.warning("geopy.geocoders failed to get coordinates for city = ",city) Log.warning(e)
def get_cache(self, annotations=None): user = annotations.get('user') user_id = user.get('user_id') if user else None if not user_id: # Use global cache Log.warning("Use of global cache for query, annotations=%r" % (annotations, )) return self._cache # Use per-user cache if user_id not in self._cache_user: self._cache_user[user_id] = Cache() return self._cache_user[user_id]
def build_rspec(cls, slice_hrn, resources, leases, flowspace, vms, rspec_version=None): Log.warning("NitosBroker Parser build") rspec = [] cls.rspec_add_header(rspec) lease_map = cls.rspec_add_leases(rspec, leases) cls.rspec_add_resources(rspec, resources, lease_map) cls.rspec_add_footer(rspec) return "\n".join(rspec)
def left_callback(self, record): """ \brief Process records received by the left child \param record A dictionary representing the received record """ if record.is_last(): # left_done. Injection is not the right way to do this. # We need to insert a filter on the key in the right member predicate = Predicate(self.predicate.get_value(), included, self.left_map.keys()) if self.right.get_query().action == ACTION_CREATE: # XXX If multiple insert, we need to match the right ID with the # right inserted items if len(self.left_map.keys()) > 1: raise NotImplemented # Pass the id as a param keys = self.left_map.keys() if not keys: # No JOIN possible self.left_done = True self._on_right_done() return key = self.left_map.keys()[0] query = self.right.get_query() query.params[self.predicate.get_value()] = key else: # pass the id as a filter which is the normal behaviour self.right = self.right.optimize_selection( Filter().filter_by(predicate)) self.right.set_callback( self.right_callback) # already done in __init__ ? self.left_done = True self.right.start() return # Directly send records missing information necessary to join # XXXX !!! XXX XXX XXX if not Record.has_fields(record, self.predicate.get_field_names()): Log.warning("Missing LEFTJOIN predicate %s in left record %r : forwarding" % \ (self.predicate, record)) self.send(record) # Store the result in a hash for joining later hash_key = Record.get_value(record, self.predicate.key) if not hash_key in self.left_map: self.left_map[hash_key] = [] self.left_map[hash_key].append(record)
def _process_resource(cls, resource): """ Postprocess resources read from the RSpec. This applies to nodes, channels and links. In particular, it sets the urn, hrn, network_hrn, facility_name and testbed_name fields. """ urn = resource['component_id'] hrn, type = urn_to_hrn(resource['component_id']) resource['urn'] = urn resource['hrn'] = hrn resource['network_hrn'] = Xrn(resource['component_id']).authority[0] # network ? XXX # We also add 'facility' and 'testbed' fields resource['facility_name'] = cls.get_resource_facility_name(urn) resource['testbed_name'] = cls.get_resource_testbed_name(urn) if 'exclusive' not in resource: resource['exclusive'] = 'true' elif resource['exclusive'] is None: resource['exclusive'] = 'true' else: Log.warning("EXCLUSIVE = ",resource['exclusive']) #if 'location' in node: # if node['location']: # node['latitude'] = node['location']['latitude'] # node['longitude'] = node['location']['longitude'] # del node['location'] #else: # if the location is not provided, aproximate it from the city t_urn = resource['urn'].split('+') city = t_urn[3].split('.')[1] if city == 'iii': city = 'Institute for Information Industry, Taïwan 106' resource['country'] = 'Taiwan' else: resource['country'] = 'France' location = cls.get_location(city) if location is not None: resource['latitude'] = str(location.latitude) resource['longitude'] = str(location.longitude) return resource
def check(self): # Method.type_check() should have checked that all of the # mandatory fields were present. assert self.auth.has_key('Username') # Get record (must be enabled) try: query_users = Query.get('local:user').filter_by('email', '==', self.auth['Username'].lower()) user, = self.interface.execute_local_query(query_users) except Exception, e: import traceback traceback.print_exc() Log.warning("Authentication failed, delete expired sessions") query_sessions = Query.delete('local:session').filter_by('expires', '<', int(time.time())) try: self.interface.execute_local_query(query_sessions) except: pass raise AuthenticationFailure, "No such account (PW): %s" % e
def filter(self, query, record, annotations, is_query=True): # TMP CACHE DEBUG #import pdb #pdb.set_trace() for rule in self.rules: if not rule.match(query, annotations): continue target = Target.get(rule.target) if not target: Log.warning("Unknown target %s" % rule.target) continue # TODO: ROUTERV2 # Cache per user # Adding interface in order to access router.get_cache(annotations) decision, data = target(self._interface).process( query, record, annotations, is_query) if decision == TargetValue.ACCEPT: return (self.ACCEPT, None) elif decision == TargetValue.REWRITE: return (self.REWRITE, data) elif decision == TargetValue.CACHE_HIT: return (self.CACHE_HIT, data) elif decision == TargetValue.DENIED: return (self.DENIED, None) elif decision == TargetValue.ERROR: return (self.ERROR, data) elif decision == TargetValue.CONTINUE: continue ## Let's create a cache entry #if is_query: # # We are dealing with queries # cache = self._interface.get_cache(annotations) # # XXX TEMP HACK # try: # cache.new_entry(query) # except Exception,e: # Log.warning(e) # Default decision : ACCEPT return (self.ACCEPT, None)
def optimize_selection(self, filter): parent_filter, top_filter = Filter(), Filter() for predicate in filter: if predicate.get_field_names() <= self.parent.get_query( ).get_select(): parent_filter.add(predicate) else: Log.warning( "SubQuery::optimize_selection() is only partially implemented : %r" % predicate) top_filter.add(predicate) if parent_filter: self.parent = self.parent.optimize_selection(parent_filter) self.parent.set_callback(self.parent_callback) if top_filter: return Selection(self, top_filter) return self
def merge_announces(self, announces_pgsql, announces_h): # Merge metadata s = PostgreSQLGateway.get_colliding_announces(announces_pgsql, announces_h) if s: Log.warning( "merge_announces: colliding announces for table(s): {%s}" % ", ".join(s)) announces = list() announces.extend(announces_h) table_names = [ announce.get_table().get_name() for announce in announces_h ] for announce_pgsql in announces_pgsql: table_name = announce_pgsql.get_table().get_name() if table_name not in table_names: announces.append(announces_pgsql) table_names.append(table_name) return announces
def _get_expiration(cls, rspec): # get the expires tag in the header of the RSpec # convert it to timestamp # XXX Until WiLab supports Leases # this will be used as lease['end_time'] try: rspec_string = rspec.toxml() import xml.etree.ElementTree as ET rspec = ET.fromstring(rspec_string) expiration = rspec.get("expires") import time from datetime import datetime ret = int(time.mktime(datetime.strptime(expiration, "%Y-%m-%dT%H:%M:%SZ").timetuple())) return ret # XXX To be removed in Router-v2 except Exception, e: import traceback Log.warning("Exception in _get_expiration: %s" % e) traceback.print_exc() return None
def _process_leases(cls, leases, slice_urn, start_time, end_time): ret = list() try: for lease in leases: lease['slice_urn'] = slice_urn lease['start_time'] = start_time lease['end_time'] = end_time # duration in seconds from now till end_time duration = end_time - start_time # duration in minutes duration = duration / 60 lease['duration'] = int(duration) new_lease = cls._process_lease(lease) if not new_lease: continue ret.append(new_lease) # XXX To be removed in Router-v2 except Exception, e: import traceback Log.warning("Exception in _process_leases: %s" % e) traceback.print_exc()
def _process_link(cls, link): authority = 'urn:publicid:IDN+wall2.ilabt.iminds.be+authority+cm' if (not 'component_manager' in link) or (link['component_manager'] != authority): Log.warning("Authority is not wall2 - Ignore link = ",link) #return None return super(WiLabtParser, cls)._process_link(link)
def parse(cls, rspec, rspec_version=None, slice_urn=None): resources = list() leases = list() rspec = RSpec(rspec) # Parse leases first, so that they can be completed when encountering # their ids in resources lease_map = dict() # id -> lease_dict elements = rspec.xml.xpath('//ol:lease') network = cls.get_network() for el in elements: try: lease_tmp = cls.dict_from_elt(network, el.element) start = calendar.timegm( dateutil.parser.parse( lease_tmp['valid_from']).utctimetuple()) end = calendar.timegm( dateutil.parser.parse( lease_tmp['valid_until']).utctimetuple()) lease = { 'lease_id': lease_tmp['id'], 'slice': slice_urn, 'start_time': start, 'end_time': end, 'duration': (end - start) / cls.get_grain(), 'granularity': cls.get_grain() } lease_map[lease_tmp['id']] = lease except: import traceback Log.warning("this lease has not the right format") traceback.print_exc() # Parse nodes for tag, resource_type in RESOURCE_TYPES.items(): if ':' in tag: ns, _, tag = tag.partition(':') XPATH_RESOURCE = "//%(ns)s:%(tag)s" else: XPATH_RESOURCE = "//default:%(tag)s | //%(tag)s" elements = rspec.xml.xpath(XPATH_RESOURCE % locals()) for el in elements: resource = cls.dict_from_elt(network, el.element, LIST_ELEMENTS.get(resource_type)) if resource_type in cls.MAP: resource = cls.dict_rename(resource, resource_type) resource['network_hrn'] = network #resource['facility_name'] = 'NITOS' resource['facility_name'] = 'Wireless' if 'hrn' in resource: hrn = resource['hrn'].replace('\\', '') elif 'component_id' in resource: urn = resource['component_id'] hrn, type = urn_to_hrn(urn) resource['hrn'] = hrn resource['urn'] = urn t_hrn = hrn.split('.') if 'omf' in hrn: if t_hrn[1] == 'nitos': resource['testbed_name'] = 'NITOS-Volos' resource['country'] = 'Greece' resource['latitude'] = '39.3666667' resource['longitude'] = '22.9458333' elif t_hrn[1] == 'netmode': resource['testbed_name'] = 'Netmode' resource['country'] = 'Greece' resource['latitude'] = '37.978377' resource['longitude'] = '23.782707' elif t_hrn[1] == 'gaia': resource['testbed_name'] = 'UMU' resource['country'] = 'Spain' resource['latitude'] = '38.023681' resource['longitude'] = '-1.174462' elif t_hrn[1] == 'snu': resource['testbed_name'] = 'SNU' resource['country'] = 'South Korea' resource['latitude'] = '37.460263' resource['longitude'] = '126.953598' elif t_hrn[1] == 'kaist': resource['testbed_name'] = 'KAIST' resource['country'] = 'South Korea' resource['latitude'] = '36.369362' resource['longitude'] = '127.364025' elif t_hrn[1] == 'etri': resource['testbed_name'] = 'ETRI' resource['country'] = 'South Korea' resource['latitude'] = '36.3808653' resource['longitude'] = '127.3650244' elif 'paris' in hrn: resource[ 'testbed_name'] = 'FIT NITOS ' + t_hrn[1].title() # XXX Hardcoded for Paris @ UPMC resource['country'] = 'France' resource['latitude'] = '48.847104' resource['longitude'] = '2.357499' else: resource['testbed_name'] = 'FIT ' + t_hrn[1].title() resource['country'] = 'France' else: if 'component_manager_id' in resource: t, tp = urn_to_hrn(resource['component_manager_id']) resource['testbed_name'] = 'FIT ' + t.title() else: resource['testbed_name'] = 'FIT ' + t_hrn[1].title() #resource['testbed_name'] = 'Nitos' if 'services.login.username' in resource: if resource['services.login.username']: resource['login'] = { 'username': resource['services.login.username'], 'hostname': resource['services.login.hostname'] } resource = set_status(resource) resources.append(resource) # Leases if 'lease_ref.id_ref' in resource: lease_id_refs = resource.pop('lease_ref.id_ref') for lease_id_ref in lease_id_refs: lease = copy.deepcopy(lease_map[lease_id_ref]) lease['resource'] = resource['urn'] leases.append(lease) return {'resource': resources, 'lease': leases}
def build(self, query, metadata, allowed_platforms, allowed_capabilities, user=None): """ Build the QueryPlan involving several Gateways according to a 3nf graph and a user Query. If only one Gateway is involved, you should use QueryPlan::build_simple. Args: query: The Query issued by the user. metadata: The 3nf graph (DBGraph instance). allowed_platforms: A list of platform names (list of String). Which platforms the router is allowed to query. Could be used to restrict the Query to a limited set of platforms either because it is specified by the user Query or either due to the Router configuration. allowed_capabilities: A Capabilities instance or None. Specify which capabilities the Router can perform if it is involved as an intermediate Router between two other Routers. TODO: metadata, allowed_platforms and allowed_platforms capabilities should be deduced from the query, the router, and the user. router.get_metadata() router.get_allowed_platforms(query, user) router.get_allowed_capabilities(query, user) user: A User instance or None. """ root = metadata.find_node(query.get_from()) if not root: raise RuntimeError( "query_plan::build(): Cannot find %s in metadata, known tables are %s" % (query.get_from(), sorted(set(metadata.get_table_names())))) root_task = ExploreTask(root, relation=None, path=[], parent=self, depth=1) root_task.addCallback(self.set_ast, query) stack = Stack(root_task) seen = {} # path -> set() missing_fields = set() missing_fields |= query.get_select() missing_fields |= query.get_where().get_field_names() missing_fields |= set(query.get_params().keys()) while missing_fields: task = stack.pop() if not task: # Exploration ends here Log.warning( "Exploration terminated without finding fields: %r" % missing_fields) break pathstr = '.'.join(task.path) if not pathstr in seen: seen[pathstr] = set() # ROUTERV2 # foreign_key_fields are fields added because indirectly requested by the user. # For example, he asked for slice.resource, which in fact will contain slice.resource.urn foreign_key_fields = task.explore(stack, missing_fields, metadata, allowed_platforms, allowed_capabilities, user, seen[pathstr], query_plan=self) self.foreign_key_fields.update(foreign_key_fields) while not stack.is_empty(): task = stack.pop() task.cancel()
def main(self): """ \brief Runs a XMLRPC server """ Log.info("XMLRPC server daemon (%s) started." % sys.argv[0]) # NOTE it is important to import those files only after daemonization, # since they open files we cannot easily preserve from twisted.web import xmlrpc, server # SSL support from OpenSSL import SSL from twisted.internet import ssl #, reactor #from twisted.internet.protocol import Factory, Protocol #from twisted.internet import reactor # This also imports manifold.util.reactor_thread that uses reactor from manifold.core.router import Router assert not (Options().platform and Options().gateway), "Both gateway and platform cannot be specified at commandline" # This imports twisted code so we need to import it locally from manifold.core.xmlrpc_api import XMLRPCAPI # This should be configurable allowed_capabilities = Capabilities() allowed_capabilities.selection = True allowed_capabilities.projection = True # XXX We should harmonize interfaces between Router and Forwarder if Options().platform: platforms = Storage.execute(Query().get('platform'), format='object') # We pass a single platform to Forwarder platform = [p for p in platforms if p.name == Options().platform][0] self.interface = Forwarder(platform, allowed_capabilities) elif Options().gateway: # XXX user # XXX Change Forwarded initializer #DEPRECATED| platform = Platform(u'dummy', Options().gateway, self.get_gateway_config(Options().gateway), 'user') platform = Platform( platform = u'dummy', gateway_type = Options().gateway, config = self.get_gateway_config(Options().gateway), auth_type = 'user' ) self.interface = Forwarder(platform, allowed_capabilities) else: self.interface = Router() try: def verifyCallback(connection, x509, errnum, errdepth, ok): if not ok: print 'invalid cert from subject:', x509.get_subject() print errnum, errdepth return False else: print "Certs are fine", x509, x509.get_subject() return True ssl_path = Options().ssl_path if not ssl_path or not os.path.exists(ssl_path): print "" print "You need to generate SSL keys and certificate in '%s' to be able to run manifold" % ssl_path print "" print "mkdir -p /etc/manifold/keys" print "openssl genrsa 1024 > /etc/manifold/keys/server.key" print "chmod 400 /etc/manifold/keys/server.key" print "openssl req -new -x509 -nodes -sha1 -days 365 -key /etc/manifold/keys/server.key > /etc/manifold/keys/server.cert" print "" sys.exit(0) server_key_file = "%s/server.key" % ssl_path server_crt_file = "%s/server.cert" % ssl_path Log.tmp("key, cert=", server_key_file, server_crt_file) myContextFactory = ssl.DefaultOpenSSLContextFactory(server_key_file, server_crt_file) ctx = myContextFactory.getContext() ctx.set_verify( SSL.VERIFY_PEER, # | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, verifyCallback ) # Since we have self-signed certs we have to explicitly # tell the server to trust them. #ctx.load_verify_locations("keys/ca.pem") trusted_roots_path = Options().trusted_roots_path if not trusted_roots_path or not os.path.exists(trusted_roots_path): Log.warning("No trusted root found in %s. You won't be able to login using SSL client certificates" % trusted_roots_path) ctx.load_verify_locations(None, ssl_path) #ReactorThread().listenTCP(Options().xmlrpc_port, server.Site(XMLRPCAPI(self.interface, allowNone=True))) ReactorThread().listenSSL(Options().xmlrpc_port, server.Site(XMLRPCAPI(self.interface, allowNone=True)), myContextFactory) ReactorThread().start_reactor() except Exception, e: # TODO If database gets disconnected, we can sleep/attempt reconnection Log.error("Error in XMLRPC API: %s" % str(e))
def get_relations(self, table, dbgraph): """ Compute which Relations connect the "self" Table (source node) to the "table" Table (target node). We assume that the graph of table is at least 2nf. Args: table: The target table Returns: A set of Relation instances connecting "self" and "table". This set is empty iif the both Tables are unrelated. """ # We only test relations u --> v u = self v = table relations = set() u_key = u.keys.one() v_key = v.keys.one() if u.get_name() == v.get_name(): p = Predicate(u_key.get_name(), eq, v_key.get_name()) if u.get_platforms() > v.get_platforms(): relations.add(Relation(Relation.types.PARENT, p)) #else: # relations.add(Relation(Relation.types.CHILD, p)) return relations # Detect explicit Relation from u to v for field in u.get_fields(): # 1. A field in u is explicitly typed againt v name if field.get_type() == v.get_name(): if v_key.is_composite(): # We assume that u (for ex: traceroute) provides in the current field (ex: hops) # a record containing at least the v's key (for ex: (agent, destination, first, ttl)) intersecting_fields = tuple(u.get_field_names() & v_key.get_field_names()) key_fields = list(v_key.get_field_names()) prefixed_key_fields = ["%s.%s" % (field.get_name(), kf) for kf in key_fields] # Plain wrong # p = Predicate(intersecting_fields, eq, intersecting_fields) p = Predicate(prefixed_key_fields, eq, key_fields) else: p = Predicate(field.get_name(), eq, v_key.get_name()) if field.is_array(): relations.add(Relation(Relation.types.LINK_1N, p, name=field.get_name())) # LINK_1N_FORWARD else: if False: # field == key relations.add(Relation(Relation.types.PARENT, p, name=field.get_name())) # in which direction ????? else: if field.is_local(): relations.add(Relation(Relation.types.LINK_11, p, name=field.get_name())) else: if v.is_child_of(u): relations.add(Relation(Relation.types.CHILD, p)) elif u.is_child_of(v): relations.add(Relation(Relation.types.PARENT, p)) else: if field.get_name() in ['source', 'destination', 'agent', 'dns_target']: Log.warning("Hardcoded source, agent, destination and dns_target as 1..1 relationships") relations.add(Relation(Relation.types.LINK_11, p, name=field.get_name())) else: relations.add(Relation(Relation.types.LINK, p)) # BAD #if v_key.is_composite(): # Log.warning("Link (2) unsupported between u=%s and v=%s: v has a composite key" % (u.get_name(), v.get_name())) ## 2. A field is typed like the key #if field.get_type() == v_key.get_field().get_type(): # # In this case we might have inheritance # # We should point to the toplevel class, ie. if key field type == platform name # # We are back to the previous case. # # a child class is an instance of the parent class, no it should be ok # (3) A field point to part of v key (v is thus composite) if field.get_type() not in BASE_TYPES and set([field.get_type()]) < v_key.get_field_types(): # What if several possible combinations # How to consider inheritance ? vfield = [f for f in v_key if f.get_type() == field.get_type()][0] # XXX This case is not well supported, let's skip for now if v.get_name() == "lease": continue p = Predicate(field.get_name(), eq, vfield.get_name()) relations.add(Relation(Relation.types.LINK_1N, p, name=field.get_name())) # LINK_1N_FORWARD ? continue # Following relations don't involve a single field # (4) A bit more complex: u presents the set of fields that make up v key # (5) A bit more complex: u presents part of the fields that make up v key if relations: return relations # Detect implicit Relation from u to v intersection = u.get_field_names() & v_key.get_field_names() if intersection and intersection < v_key.get_field_names(): intersection = tuple(intersection) if len(intersection) == 1: intersection = intersection[0] p = Predicate(intersection, eq, intersection) relations.add(Relation(Relation.types.LINK_1N, p, name=v.get_name())) # LINK_1N_FORWARD # Name ? # we don't continue otherwise we will find subsets of this set # note: this code might replace following code operating on a single field return relations # --- REVERSE RELATIONS for field in v.get_fields(): # (6) inv of (1) a field in v points to an existing type # we could say we only look at key types at this stage if field.get_type() == u.get_name(): if u_key.is_composite(): Log.warning("Link (6) unsupported between u=%s and v=%s: u has a composite key" % (u.get_name(), v.get_name())) continue p = Predicate(u_key.get_name(), eq, field.get_name()) if field.is_array(): relations.add(Relation(Relation.types.LINK_1N_BACKWARDS, p, name = v.get_name())) ### was: COLLECTION, p)) # a u is many v ? approve this type #relations.add(Relation(Relation.types.COLLECTION, p)) # a u is many v ? approve this type else: # if u parent if v.is_child_of(u): relations.add(Relation(Relation.types.CHILD, p)) elif u.is_child_of(v): relations.add(Relation(Relation.types.PARENT, p)) else: relations.add(Relation(Relation.types.LINK_1N, p, name=v.get_name())) # LINK_1N_BACKWARDS return relations
def get_user_config(self, user, platform): # all are dict platform_name = platform['platform'] platform_id = platform['platform_id'] auth_type = platform.get('auth_type', None) if not auth_type: Log.warning("'auth_type' is not set in platform = %s" % platform_name) return None # XXX platforms might have multiple auth types (like pam) # XXX we should refer to storage if auth_type in ["none", "default"]: user_config = {} # For default, take myslice account elif auth_type == 'user': if not user: raise Exception, "Cannot use auth_type user when no user is configured" user_id = user['user_id'] if user else None # User account information query_accounts = Query.get('local:account').filter_by('user_id', '==', user_id).filter_by('platform_id', '==', platform_id) accounts = self.execute_local_query(query_accounts) if accounts: account = accounts[0] user_config = account.get('config', None) if user_config: user_config = json.loads(user_config) # XXX This should disappear with the merge with router-v2 if account['auth_type'] == 'reference': ref_platform_name = user_config['reference_platform'] query_ref_platform = Query.get('local:platform').filter_by('platform', '==', ref_platform_name) ref_platforms = self.execute_local_query(query_ref_platform) if not ref_platforms: raise Exception, 'Cannot find reference platform %s for platform %s' % (platform_name, ref_platform_name) ref_platform = ref_platforms[0] query_ref_account = Query.get('local:account').filter_by('user_id', '==', user_id).filter_by('platform_id', '==', ref_platform['platform_id']) ref_accounts = self.execute_local_query(query_ref_account) if not ref_accounts: raise Exception, 'Cannot find account information for reference platform %s' % ref_platform_name ref_account = ref_accounts[0] user_config = ref_account.get('config', None) if user_config: user_config = json.loads(user_config) else: user_config = {} else: raise ValueError("This 'auth_type' not supported: %s" % auth_type) return user_config
def _process_channel(cls, channel): authority = 'urn:publicid:IDN+wall2.ilabt.iminds.be+authority+cm' if (not 'component_manager_id' in channel) or (channel['component_manager_id'] != authority): Log.warning("Authority is not wall2 - Ignore channel = ",channel) #return None return super(WiLabtParser, cls)._process_channel(channel)
def _process_node(cls, node): authority = 'urn:publicid:IDN+wall2.ilabt.iminds.be+authority+cm' if (not 'component_manager_id' in node) or (node['component_manager_id'] != authority): Log.warning("Authority is not wall2 - Ignore node = ",node) #return None return super(WiLabtParser, cls)._process_node(node)