def _publishConnection(self, destUrl, username, password, contact): connectionInfo = { "doc_type": "connection_description", "gateway_connection": False, "connection_id": uuid.uuid1().hex, "destination_node_url": destUrl, "doc_version": "0.10.0", "source_node_url": request.host_url, "active": True, "doc_scope": "node", "X_contact": contact, } try: urllib2.urlopen(destUrl) conn = NodeConnectivityModel(connectionInfo) result = conn.save() destinationURL = urlparse.urljoin(destUrl.strip(), "incoming") if bool(username) and bool(password): sourceLRNode.addDistributeCredentialFor(destinationURL, username, password) if result["OK"]: return json.dumps({"status": "okay", "msg": "Your end point was successfully registered"}) else: return json.dumps({"status": "error", "msg": result["error"]}) except httplib.HTTPException: return abort(400, "Invalid destination URL") except ValueError: abort(400, "Invalid URL format") except: import sys exc_type, exc_val = sys.exc_info()[:2] abort(400, "{0}: {1}".format(exc_type, exc_val))
def _publishConnection(self,destUrl,username,password,contact): connectionInfo = { "doc_type": "connection_description", "gateway_connection": False, "connection_id": uuid.uuid1().hex, "destination_node_url": destUrl, "doc_version": "0.10.0", "source_node_url": request.host_url, "active": True, "doc_scope": "node", "X_contact": contact } try: urllib2.urlopen(destUrl) conn = NodeConnectivityModel(connectionInfo) result = conn.save() destinationURL = urlparse.urljoin(destUrl.strip(),"incoming") if bool(username) and bool(password): sourceLRNode.addDistributeCredentialFor(destinationURL,username,password) if result["OK"]: return json.dumps({"status":"okay", "msg":"Your end point was successfully registered"}) else: return json.dumps({"status":"error", "msg":result['error']}) except httplib.HTTPException: return abort(400, "Invalid destination URL") except ValueError: abort(400, "Invalid URL format") except: import sys exc_type, exc_val = sys.exc_info()[:2] abort(400, "{0}: {1}".format(exc_type, exc_val))
def doDistribution(destinationNode, server, sourceUrl, destinationUrl, lock): # We want to always use the replication filter function to replicate # only distributable doc and filter out any other type of documents. # However we don't have any query arguments until we test if there is any filter. replicationOptions={'filter':ResourceDataModel.REPLICATION_FILTER, 'query_params': None} # If the destination node is using an filter and is not custom use it # as the query params for the filter function if ((destinationNode.filterDescription is not None) and (destinationNode.filterDescription.custom_filter == False)): replicationOptions['query_params'] = destinationNode.filterDescription.specData #if distinationNode['distribute service'] .service_auth["service_authz"] is not None: #log.info("Destination node '{}' require authentication".format(destinationUrl)) #Try to get the user name and password the url. credential = sourceLRNode.getDistributeCredentialFor(destinationUrl) if credential is not None: parsedUrl = urlparse.urlparse(destinationUrl) destinationUrl = destinationUrl.replace(parsedUrl.netloc, "{0}:{1}@{2}".format( credential['username'], credential['password'], parsedUrl.netloc)) log.info("\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}".format( sourceUrl, destinationUrl, str(replicationOptions))) if replicationOptions['query_params'] is None: del replicationOptions['query_params'] results = server.replicate(sourceUrl, destinationUrl, **replicationOptions) log.debug("Replication results: "+str(results)) with lock: server = couchdb.Server(appConfig['couchdb.url']) db = server[appConfig['couchdb.db.node']] doc = db[appConfig['lr.nodestatus.docid']] doc['last_out_sync'] = h.nowToISO8601Zformat() doc['out_sync_node'] = destinationNode.nodeDescription.node_name db[appConfig['lr.nodestatus.docid']] = doc
def index(self, format='html'): """GET /policy: All items in the collection""" if sourceLRNode.isServiceAvailable( NodeServiceModel.ADMINISTRATIVE) == False: return "Administrative service is not available" return json.dumps(sourceLRNode.networkPolicyDescription.specData)
def index(self, format='html'): """GET /description: All items in the collection""" if sourceLRNode.isServiceAvailable(NodeServiceModel.ADMINISTRATIVE) == False: return "Administrative service is not available" data = {} data['timestamp'] = str(datetime.utcnow()) data.update(sourceLRNode.config) return json.dumps(data)
def index(self, format='html'): """GET /distribute: All items in the collection""" # url('distribute') distributeInfo = {'OK': True} if sourceLRNode.isServiceAvailable(NodeServiceModel.DISTRIBUTE) == False: distributeInfo['OK'] = False else: distributeInfo['node_config'] = sourceLRNode.config distributeInfo['distribute_sink_url'] = urlparse.urljoin(request.url,self.resource_data) log.info("received distribute request...returning: \n"+json.dumps(distributeInfo)) return json.dumps(distributeInfo)
def doDistribution(connectionInfo, server, sourceUrl): # We want to always use the replication filter function to replicate # only distributable doc and filter out any other type of documents. # However we don't have any query arguments until we test if there is any filter. replicationOptions = { "filter": ResourceDataModel.REPLICATION_FILTER, "source": sourceUrl, "connection_id": connectionInfo["connection_id"], "query_params": None, } # If the destination node is using an filter and is not custom use it # as the query params for the filter function if (connectionInfo["destinationNodeInfo"].filter_description is not None) and ( connectionInfo["destinationNodeInfo"].filter_description.get("custom_filter") == False ): replicationOptions["query_params"] = connectionInfo["destinationNodeInfo"].filter_description # if distinationNode['distribute service'] .service_auth["service_authz"] is not None: # log.info("Destination node '{}' require authentication".format(destinationUrl)) # Try to get the user name and password the url # destinationUrl = connectionInfo['destinationNodeInfo'].resource_data_url destinationUrl = connectionInfo["destinationNodeInfo"].incoming_url credential = sourceLRNode.getDistributeCredentialFor(destinationUrl) if credential is not None: parsedUrl = urlparse.urlparse(destinationUrl) destinationUrl = destinationUrl.replace( parsedUrl.netloc, "{0}:{1}@{2}".format(credential["username"], credential["password"], parsedUrl.netloc), ) if replicationOptions["query_params"] is None: del replicationOptions["query_params"] replicationOptions["target"] = destinationUrl request = urllib2.Request( urlparse.urljoin(appConfig["couchdb.url"], "_replicator"), headers={"Content-Type": "application/json"}, data=json.dumps(replicationOptions), ) log.info( "\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}".format( sourceUrl, destinationUrl, pprint.pformat(replicationOptions) ) ) results = json.load(urllib2.urlopen(request)) connectionInfo["replication_results"] = results distributeResults.put(connectionInfo) log.debug("Replication results: " + pprint.pformat(results))
def index(self, format='html'): """GET /services: All items in the collection""" if sourceLRNode.isServiceAvailable('Network Node Services') == False: return "Administrative service is not available" data = {} data['timestamp'] = str(datetime.datetime.utcnow()) data['node_id'] = sourceLRNode.nodeDescription.node_id data['active'] = sourceLRNode.nodeDescription.active data['node_name'] = sourceLRNode.nodeDescription.node_name data['services'] = [] for s in sourceLRNode.nodeServices: data['services'].append(s.specData) return json.dumps(data)
def _getDistinationInfo(self, connection): # Make sure we only have one slash in the url path. More than one #confuses pylons routing libary. destinationURL = urlparse.urljoin(connection.destination_node_url.strip(), "destination") request = urllib2.Request(destinationURL) credential = sourceLRNode.getDistributeCredentialFor(destinationURL) if credential is not None: base64string = base64.encodestring('%s:%s' % (credential['username'],credential['password'])).replace("\n", "") request.add_header("Authorization", "Basic %s" % base64string) log.info("\n\nAccess destination node at: "+pprint.pformat(request.__dict__)) return json.load(urllib2.urlopen(request))
def create(self): """POST / distribute start distribution""" def doDistribution(destinationNode, server, sourceUrl, destinationUrl): filterFunction = (ResourceDataModel._defaultDB.name+"/" + ResourceDataModel.DEFAULT_FILTER) # We want to always use the default filter function to filter # out design document on replication. But we don't have any # query arguments until we test if there is any filter. replicationOptions={'filter':filterFunction, 'query_params': None} # If the destination node is using an filter and is not custom use it # as the query params for the filter function if ((destinationNode.filterDescription is not None) and (destinationNode.filterDescription.custom_filter == False)): replicationOptions['query_params'] = destinationNode.filterDescription.specData log.info("\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}".format( sourceUrl, destinationUrl, str(replicationOptions))) if replicationOptions['query_params'] is None: del replicationOptions['query_params'] results = server.replicate(sourceUrl, destinationUrl, **replicationOptions) log.debug("Replication results: "+str(results)) log.info("Distribute.......\n") #Check if the distribte service is available on the node. if(sourceLRNode.isServiceAvailable(NodeServiceModel.DISTRIBUTE) == False): log.info("Distribute not available on node ") return if((sourceLRNode.connections is None) or (len(sourceLRNode.connections) ==0)): log.info("No connection present for distribution") return log.info("Connections: "+str(sourceLRNode.connections)+"\n") for connectionInfo in self._getDistributeDestinations(): replicationArgs = (connectionInfo['destinationNode'], defaultCouchServer, self.resource_data, connectionInfo['distributeInfo']['distribute_sink_url']) # Use a thread to do the actual replication. replicationThread = threading.Thread(target=doDistribution, args=replicationArgs) replicationThread.start()
def doDistribution(destinationNode, server, sourceUrl, destinationUrl, lock): # We want to always use the replication filter function to replicate # only distributable doc and filter out any other type of documents. # However we don't have any query arguments until we test if there is any filter. replicationOptions = { 'filter': ResourceDataModel.REPLICATION_FILTER, 'query_params': None } # If the destination node is using an filter and is not custom use it # as the query params for the filter function if ((destinationNode.filterDescription is not None) and (destinationNode.filterDescription.custom_filter == False)): replicationOptions[ 'query_params'] = destinationNode.filterDescription.specData #if distinationNode['distribute service'] .service_auth["service_authz"] is not None: #log.info("Destination node '{}' require authentication".format(destinationUrl)) #Try to get the user name and password the url. credential = sourceLRNode.getDistributeCredentialFor( destinationUrl) if credential is not None: parsedUrl = urlparse.urlparse(destinationUrl) destinationUrl = destinationUrl.replace( parsedUrl.netloc, "{0}:{1}@{2}".format(credential['username'], credential['password'], parsedUrl.netloc)) log.info( "\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}" .format(sourceUrl, destinationUrl, str(replicationOptions))) if replicationOptions['query_params'] is None: del replicationOptions['query_params'] results = server.replicate(sourceUrl, destinationUrl, **replicationOptions) log.debug("Replication results: " + str(results)) with lock: server = couchdb.Server(appConfig['couchdb.url']) db = server[appConfig['couchdb.db.node']] doc = db[appConfig['lr.nodestatus.docid']] doc['last_out_sync'] = h.nowToISO8601Zformat() doc['out_sync_node'] = destinationNode.nodeDescription.node_name db[appConfig['lr.nodestatus.docid']] = doc
def doDistribution(connectionInfo, server, sourceUrl): # We want to always use the replication filter function to replicate # only distributable doc and filter out any other type of documents. # However we don't have any query arguments until we test if there is any filter. replicationOptions={'filter':ResourceDataModel.REPLICATION_FILTER, 'source':sourceUrl, 'connection_id': connectionInfo['connection_id'], 'query_params': None} # If the destination node is using an filter and is not custom use it # as the query params for the filter function if ((connectionInfo['destinationNodeInfo'].filter_description is not None ) and (connectionInfo['destinationNodeInfo'].filter_description.get('custom_filter') == False)): replicationOptions['query_params'] =connectionInfo['destinationNodeInfo'].filter_description #if distinationNode['distribute service'] .service_auth["service_authz"] is not None: #log.info("Destination node '{}' require authentication".format(destinationUrl)) #Try to get the user name and password the url #destinationUrl = connectionInfo['destinationNodeInfo'].resource_data_url destinationUrl = connectionInfo['destinationNodeInfo'].incoming_url credential = sourceLRNode.getDistributeCredentialFor(destinationUrl) if credential is not None: parsedUrl = urlparse.urlparse(destinationUrl) destinationUrl = destinationUrl.replace(parsedUrl.netloc, "{0}:{1}@{2}".format( credential['username'], credential['password'], parsedUrl.netloc)) if replicationOptions['query_params'] is None: del replicationOptions['query_params'] replicationOptions['target'] = destinationUrl authz_header = h.getBasicAuthHeaderFromURL(appConfig['couchdb.url.dbadmin']); authz_header.update( { 'Content-Type': 'application/json'}) request = urllib2.Request(urlparse.urljoin(appConfig['couchdb.url'], '_replicator'), headers=authz_header, data = json.dumps(replicationOptions)) log.info("\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}".format( sourceUrl, destinationUrl, pprint.pformat(replicationOptions))) results = json.load(urllib2.urlopen(request)) connectionInfo['replication_results'] = results distributeResults.put(connectionInfo) log.debug("Replication results: " + pprint.pformat(results))
def index(self, format='html'): """GET /policy: All items in the collection""" if sourceLRNode.isServiceAvailable(NodeServiceModel.ADMINISTRATIVE) == False: return "Administrative service is not available" return json.dumps(sourceLRNode.networkPolicyDescription.specData)
def _getDistributeDestinations(self): """"Method to test the connections and returns a list of destionation node if the connections are valid""" nodeDestinationList = [] gatewayConnectionList = [] for connection in sourceLRNode.connections: # Make sure that the connection is active if connection.active == False: continue destinationLRNode = None if connection.gateway_connection == True: gatewayConnectionList.append(connection) try: # Make sure we only have one slash in the url path. More than one #confuses pylons routing libary. destinationURL = urlparse.urljoin( connection.destination_node_url.strip(), "distribute") request = urllib2.Request(destinationURL) credential = sourceLRNode.getDistributeCredentialFor( destinationURL) if credential is not None: base64string = base64.encodestring( '%s:%s' % (credential['username'], credential['password'])).replace("\n", "") request.add_header("Authorization", "Basic %s" % base64string) log.info("\n\nAccess destination node at: " + pprint.pformat(request.__dict__)) distributeInfo = json.load(urllib2.urlopen(request)) destinationLRNode = LRNodeModel(distributeInfo['node_config']) except Exception as ex: log.exception(ex) continue # Use of local variable to store if the connection is gateway connection. It is # done this way to deal with mismatch between node de and connection # description. isGatewayConnection = ( (sourceLRNode.nodeDescription.gateway_node == True) and (destinationLRNode.nodeDescription.gateway_node == True)) # Skip the connection if there is any mismatch between the connection and # the node data. if isGatewayConnection != connection.gateway_connection: log.info( "Skip connection. 'gateway_connection' mismatch between node and connection data" ) continue # Only one gateway connection is allowed, faulty network description if len(gatewayConnectionList) > 1: log.info( "***Abort distribution. More than one gateway node connection" ) #Clear the node destination list no distribution is network description # is faulty nodeDestinationList = [] break #Calcuate if the connection is gateway one, if so #cannot distribute across non social communities if ((sourceLRNode.communityDescription.community_id != destinationLRNode.communityDescription.community_id) and ((sourceLRNode.communityDescription.social_community == False) or (destinationLRNode.communityDescription.social_community == False))): log.info("Cannot distribute across non social communities") continue # Cannot distribute across networks (or communities) unless gateway if ((isGatewayConnection == False) and ((sourceLRNode.communityDescription.community_id != destinationLRNode.communityDescription.community_id) or (sourceLRNode.networkDescription.network_id != destinationLRNode.networkDescription.network_id))): log.info( "Different Network. Cannot distribute across networks (or communities) unless gateway" ) continue # Gateway must only distribute across different networks. if ((isGatewayConnection == True) and (sourceLRNode.networkDescription.network_id == destinationLRNode.networkDescription.network_id)): log.info( "Gateway must only distribute across different networks") continue # Only gateways can distribute on gateway connection. This is really for # catching mismatch in the data where a connection says it is between # gateways when the nodes are not both gateways. if ((connection.gateway_connection == True) and ((sourceLRNode.nodeDescription.gateway_node == False) or (destinationLRNode.nodeDescription.gateway_node == False))): log.info("Only gateways can distribute on gateway connection") continue nodeInfo = { "distributeInfo": distributeInfo, "distribute_sink_url": distributeInfo["distribute_sink_url"], "destinationNode": destinationLRNode } nodeDestinationList.append(nodeInfo) return nodeDestinationList
def _getDistributeDestinations(self): """"Method to test the connections and returns a list of destionation node if the connections are valid""" nodeDestinationList =[] gatewayConnectionList = [] for connection in sourceLRNode.connections: # Make sure that the connection is active if connection.active == False: continue destinationLRNode = None if connection.gateway_connection == True: gatewayConnectionList.append(connection) try: # Make sure we only have one slash in the url path. More than one #confuses pylons routing libary. destinationURL = urlparse.urljoin(connection.destination_node_url.strip(), "distribute") request = urllib2.Request(destinationURL) credential = sourceLRNode.getDistributeCredentialFor(destinationURL) if credential is not None: base64string = base64.encodestring('%s:%s' % (credential['username'],credential['password'])).replace("\n", "") request.add_header("Authorization", "Basic %s" % base64string) log.info("\n\nAccess destination node at: "+pprint.pformat(request.__dict__)) distributeInfo = json.load(urllib2.urlopen(request)) destinationLRNode = LRNodeModel(distributeInfo['node_config']) except Exception as ex: log.exception(ex) continue # Use of local variable to store if the connection is gateway connection. It is # done this way to deal with mismatch between node de and connection # description. isGatewayConnection = ( (sourceLRNode.nodeDescription.gateway_node == True) and (destinationLRNode.nodeDescription.gateway_node ==True)) # Skip the connection if there is any mismatch between the connection and # the node data. if isGatewayConnection != connection.gateway_connection: log.info("Skip connection. 'gateway_connection' mismatch between node and connection data") continue # Only one gateway connection is allowed, faulty network description if len(gatewayConnectionList) > 1: log.info("***Abort distribution. More than one gateway node connection") #Clear the node destination list no distribution is network description # is faulty nodeDestinationList = [] break #Calcuate if the connection is gateway one, if so #cannot distribute across non social communities if ((sourceLRNode.communityDescription.community_id != destinationLRNode.communityDescription.community_id) and ((sourceLRNode.communityDescription.social_community == False) or (destinationLRNode.communityDescription.social_community == False))): log.info("Cannot distribute across non social communities") continue # Cannot distribute across networks (or communities) unless gateway if((isGatewayConnection == False) and ((sourceLRNode.communityDescription.community_id != destinationLRNode.communityDescription.community_id) or (sourceLRNode.networkDescription.network_id != destinationLRNode.networkDescription.network_id))): log.info("Different Network. Cannot distribute across networks (or communities) unless gateway") continue # Gateway must only distribute across different networks. if((isGatewayConnection ==True) and (sourceLRNode.networkDescription.network_id == destinationLRNode.networkDescription.network_id)): log.info("Gateway must only distribute across different networks") continue # Only gateways can distribute on gateway connection. This is really for # catching mismatch in the data where a connection says it is between # gateways when the nodes are not both gateways. if((connection.gateway_connection == True) and ((sourceLRNode.nodeDescription.gateway_node == False) or (destinationLRNode.nodeDescription.gateway_node == False))): log.info("Only gateways can distribute on gateway connection") continue nodeInfo = { "distributeInfo": distributeInfo, "destinationBaseUrl":connection.destination_node_url, "destinationNode":destinationLRNode} nodeDestinationList.append(nodeInfo) return nodeDestinationList
def index(self, format="html"): """GET /status: All items in the collection""" if sourceLRNode.isServiceAvailable(NodeServiceModel.ADMINISTRATIVE) == False: return "Administrative service not is available" return json.dumps(sourceLRNode.status)