def _publishConnection(self, destUrl, username, password, contact):
connectionInfo = {
"doc_type": "connection_description",
"gateway_connection": False,
"connection_id": uuid.uuid1().hex,
"destination_node_url": destUrl,
"doc_version": "0.10.0",
"source_node_url": request.host_url,
"active": True,
"doc_scope": "node",
"X_contact": contact,
}
try:
urllib2.urlopen(destUrl)
conn = NodeConnectivityModel(connectionInfo)
result = conn.save()
destinationURL = urlparse.urljoin(destUrl.strip(), "incoming")
if bool(username) and bool(password):
sourceLRNode.addDistributeCredentialFor(destinationURL, username, password)
if result["OK"]:
return json.dumps({"status": "okay", "msg": "Your end point was successfully registered"})
else:
return json.dumps({"status": "error", "msg": result["error"]})
except httplib.HTTPException:
return abort(400, "Invalid destination URL")
except ValueError:
abort(400, "Invalid URL format")
except:
import sys
exc_type, exc_val = sys.exc_info()[:2]
abort(400, "{0}: {1}".format(exc_type, exc_val))
def _publishConnection(self,destUrl,username,password,contact):
connectionInfo = {
"doc_type": "connection_description",
"gateway_connection": False,
"connection_id": uuid.uuid1().hex,
"destination_node_url": destUrl,
"doc_version": "0.10.0",
"source_node_url": request.host_url,
"active": True,
"doc_scope": "node",
"X_contact": contact
}
try:
urllib2.urlopen(destUrl)
conn = NodeConnectivityModel(connectionInfo)
result = conn.save()
destinationURL = urlparse.urljoin(destUrl.strip(),"incoming")
if bool(username) and bool(password):
sourceLRNode.addDistributeCredentialFor(destinationURL,username,password)
if result["OK"]:
return json.dumps({"status":"okay", "msg":"Your end point was successfully registered"})
else:
return json.dumps({"status":"error", "msg":result['error']})
except httplib.HTTPException:
return abort(400, "Invalid destination URL")
except ValueError:
abort(400, "Invalid URL format")
except:
import sys
exc_type, exc_val = sys.exc_info()[:2]
abort(400, "{0}: {1}".format(exc_type, exc_val))
def doDistribution(destinationNode, server, sourceUrl, destinationUrl, lock):
# We want to always use the replication filter function to replicate
# only distributable doc and filter out any other type of documents.
# However we don't have any query arguments until we test if there is any filter.
replicationOptions={'filter':ResourceDataModel.REPLICATION_FILTER,
'query_params': None}
# If the destination node is using an filter and is not custom use it
# as the query params for the filter function
if ((destinationNode.filterDescription is not None) and
(destinationNode.filterDescription.custom_filter == False)):
replicationOptions['query_params'] = destinationNode.filterDescription.specData
#if distinationNode['distribute service'] .service_auth["service_authz"] is not None:
#log.info("Destination node '{}' require authentication".format(destinationUrl))
#Try to get the user name and password the url.
credential = sourceLRNode.getDistributeCredentialFor(destinationUrl)
if credential is not None:
parsedUrl = urlparse.urlparse(destinationUrl)
destinationUrl = destinationUrl.replace(parsedUrl.netloc, "{0}:{1}@{2}".format(
credential['username'], credential['password'], parsedUrl.netloc))
log.info("\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}".format(
sourceUrl, destinationUrl, str(replicationOptions)))
if replicationOptions['query_params'] is None:
del replicationOptions['query_params']
results = server.replicate(sourceUrl, destinationUrl, **replicationOptions)
log.debug("Replication results: "+str(results))
with lock:
server = couchdb.Server(appConfig['couchdb.url'])
db = server[appConfig['couchdb.db.node']]
doc = db[appConfig['lr.nodestatus.docid']]
doc['last_out_sync'] = h.nowToISO8601Zformat()
doc['out_sync_node'] = destinationNode.nodeDescription.node_name
db[appConfig['lr.nodestatus.docid']] = doc
def index(self, format='html'):
"""GET /policy: All items in the collection"""
if sourceLRNode.isServiceAvailable(
NodeServiceModel.ADMINISTRATIVE) == False:
return "Administrative service is not available"
return json.dumps(sourceLRNode.networkPolicyDescription.specData)
def index(self, format='html'):
"""GET /description: All items in the collection"""
if sourceLRNode.isServiceAvailable(NodeServiceModel.ADMINISTRATIVE) == False:
return "Administrative service is not available"
data = {}
data['timestamp'] = str(datetime.utcnow())
data.update(sourceLRNode.config)
return json.dumps(data)
def index(self, format='html'):
"""GET /distribute: All items in the collection"""
# url('distribute')
distributeInfo = {'OK': True}
if sourceLRNode.isServiceAvailable(NodeServiceModel.DISTRIBUTE) == False:
distributeInfo['OK'] = False
else:
distributeInfo['node_config'] = sourceLRNode.config
distributeInfo['distribute_sink_url'] = urlparse.urljoin(request.url,self.resource_data)
log.info("received distribute request...returning: \n"+json.dumps(distributeInfo))
return json.dumps(distributeInfo)
def doDistribution(connectionInfo, server, sourceUrl):
# We want to always use the replication filter function to replicate
# only distributable doc and filter out any other type of documents.
# However we don't have any query arguments until we test if there is any filter.
replicationOptions = {
"filter": ResourceDataModel.REPLICATION_FILTER,
"source": sourceUrl,
"connection_id": connectionInfo["connection_id"],
"query_params": None,
}
# If the destination node is using an filter and is not custom use it
# as the query params for the filter function
if (connectionInfo["destinationNodeInfo"].filter_description is not None) and (
connectionInfo["destinationNodeInfo"].filter_description.get("custom_filter") == False
):
replicationOptions["query_params"] = connectionInfo["destinationNodeInfo"].filter_description
# if distinationNode['distribute service'] .service_auth["service_authz"] is not None:
# log.info("Destination node '{}' require authentication".format(destinationUrl))
# Try to get the user name and password the url
# destinationUrl = connectionInfo['destinationNodeInfo'].resource_data_url
destinationUrl = connectionInfo["destinationNodeInfo"].incoming_url
credential = sourceLRNode.getDistributeCredentialFor(destinationUrl)
if credential is not None:
parsedUrl = urlparse.urlparse(destinationUrl)
destinationUrl = destinationUrl.replace(
parsedUrl.netloc,
"{0}:{1}@{2}".format(credential["username"], credential["password"], parsedUrl.netloc),
)
if replicationOptions["query_params"] is None:
del replicationOptions["query_params"]
replicationOptions["target"] = destinationUrl
request = urllib2.Request(
urlparse.urljoin(appConfig["couchdb.url"], "_replicator"),
headers={"Content-Type": "application/json"},
data=json.dumps(replicationOptions),
)
log.info(
"\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}".format(
sourceUrl, destinationUrl, pprint.pformat(replicationOptions)
)
)
results = json.load(urllib2.urlopen(request))
connectionInfo["replication_results"] = results
distributeResults.put(connectionInfo)
log.debug("Replication results: " + pprint.pformat(results))
def index(self, format='html'):
"""GET /services: All items in the collection"""
if sourceLRNode.isServiceAvailable('Network Node Services') == False:
return "Administrative service is not available"
data = {}
data['timestamp'] = str(datetime.datetime.utcnow())
data['node_id'] = sourceLRNode.nodeDescription.node_id
data['active'] = sourceLRNode.nodeDescription.active
data['node_name'] = sourceLRNode.nodeDescription.node_name
data['services'] = []
for s in sourceLRNode.nodeServices:
data['services'].append(s.specData)
return json.dumps(data)
def _getDistinationInfo(self, connection):
# Make sure we only have one slash in the url path. More than one
#confuses pylons routing libary.
destinationURL = urlparse.urljoin(connection.destination_node_url.strip(),
"destination")
request = urllib2.Request(destinationURL)
credential = sourceLRNode.getDistributeCredentialFor(destinationURL)
if credential is not None:
base64string = base64.encodestring('%s:%s' % (credential['username'],credential['password'])).replace("\n", "")
request.add_header("Authorization", "Basic %s" % base64string)
log.info("\n\nAccess destination node at: "+pprint.pformat(request.__dict__))
return json.load(urllib2.urlopen(request))
def index(self, format='html'):
"""GET /services: All items in the collection"""
if sourceLRNode.isServiceAvailable('Network Node Services') == False:
return "Administrative service is not available"
data = {}
data['timestamp'] = str(datetime.datetime.utcnow())
data['node_id'] = sourceLRNode.nodeDescription.node_id
data['active'] = sourceLRNode.nodeDescription.active
data['node_name'] = sourceLRNode.nodeDescription.node_name
data['services'] = []
for s in sourceLRNode.nodeServices:
data['services'].append(s.specData)
return json.dumps(data)
def create(self):
"""POST / distribute start distribution"""
def doDistribution(destinationNode, server, sourceUrl, destinationUrl):
filterFunction = (ResourceDataModel._defaultDB.name+"/" +
ResourceDataModel.DEFAULT_FILTER)
# We want to always use the default filter function to filter
# out design document on replication. But we don't have any
# query arguments until we test if there is any filter.
replicationOptions={'filter':filterFunction,
'query_params': None}
# If the destination node is using an filter and is not custom use it
# as the query params for the filter function
if ((destinationNode.filterDescription is not None) and
(destinationNode.filterDescription.custom_filter == False)):
replicationOptions['query_params'] = destinationNode.filterDescription.specData
log.info("\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}".format(
sourceUrl, destinationUrl, str(replicationOptions)))
if replicationOptions['query_params'] is None:
del replicationOptions['query_params']
results = server.replicate(sourceUrl, destinationUrl, **replicationOptions)
log.debug("Replication results: "+str(results))
log.info("Distribute.......\n")
#Check if the distribte service is available on the node.
if(sourceLRNode.isServiceAvailable(NodeServiceModel.DISTRIBUTE) == False):
log.info("Distribute not available on node ")
return
if((sourceLRNode.connections is None) or
(len(sourceLRNode.connections) ==0)):
log.info("No connection present for distribution")
return
log.info("Connections: "+str(sourceLRNode.connections)+"\n")
for connectionInfo in self._getDistributeDestinations():
replicationArgs = (connectionInfo['destinationNode'],
defaultCouchServer,
self.resource_data,
connectionInfo['distributeInfo']['distribute_sink_url'])
# Use a thread to do the actual replication.
replicationThread = threading.Thread(target=doDistribution,
args=replicationArgs)
replicationThread.start()
def doDistribution(destinationNode, server, sourceUrl, destinationUrl,
lock):
# We want to always use the replication filter function to replicate
# only distributable doc and filter out any other type of documents.
# However we don't have any query arguments until we test if there is any filter.
replicationOptions = {
'filter': ResourceDataModel.REPLICATION_FILTER,
'query_params': None
}
# If the destination node is using an filter and is not custom use it
# as the query params for the filter function
if ((destinationNode.filterDescription is not None) and
(destinationNode.filterDescription.custom_filter == False)):
replicationOptions[
'query_params'] = destinationNode.filterDescription.specData
#if distinationNode['distribute service'] .service_auth["service_authz"] is not None:
#log.info("Destination node '{}' require authentication".format(destinationUrl))
#Try to get the user name and password the url.
credential = sourceLRNode.getDistributeCredentialFor(
destinationUrl)
if credential is not None:
parsedUrl = urlparse.urlparse(destinationUrl)
destinationUrl = destinationUrl.replace(
parsedUrl.netloc,
"{0}:{1}@{2}".format(credential['username'],
credential['password'],
parsedUrl.netloc))
log.info(
"\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}"
.format(sourceUrl, destinationUrl, str(replicationOptions)))
if replicationOptions['query_params'] is None:
del replicationOptions['query_params']
results = server.replicate(sourceUrl, destinationUrl,
**replicationOptions)
log.debug("Replication results: " + str(results))
with lock:
server = couchdb.Server(appConfig['couchdb.url'])
db = server[appConfig['couchdb.db.node']]
doc = db[appConfig['lr.nodestatus.docid']]
doc['last_out_sync'] = h.nowToISO8601Zformat()
doc['out_sync_node'] = destinationNode.nodeDescription.node_name
db[appConfig['lr.nodestatus.docid']] = doc
def doDistribution(connectionInfo, server, sourceUrl):
# We want to always use the replication filter function to replicate
# only distributable doc and filter out any other type of documents.
# However we don't have any query arguments until we test if there is any filter.
replicationOptions={'filter':ResourceDataModel.REPLICATION_FILTER,
'source':sourceUrl,
'connection_id': connectionInfo['connection_id'],
'query_params': None}
# If the destination node is using an filter and is not custom use it
# as the query params for the filter function
if ((connectionInfo['destinationNodeInfo'].filter_description is not None ) and
(connectionInfo['destinationNodeInfo'].filter_description.get('custom_filter') == False)):
replicationOptions['query_params'] =connectionInfo['destinationNodeInfo'].filter_description
#if distinationNode['distribute service'] .service_auth["service_authz"] is not None:
#log.info("Destination node '{}' require authentication".format(destinationUrl))
#Try to get the user name and password the url
#destinationUrl = connectionInfo['destinationNodeInfo'].resource_data_url
destinationUrl = connectionInfo['destinationNodeInfo'].incoming_url
credential = sourceLRNode.getDistributeCredentialFor(destinationUrl)
if credential is not None:
parsedUrl = urlparse.urlparse(destinationUrl)
destinationUrl = destinationUrl.replace(parsedUrl.netloc, "{0}:{1}@{2}".format(
credential['username'], credential['password'], parsedUrl.netloc))
if replicationOptions['query_params'] is None:
del replicationOptions['query_params']
replicationOptions['target'] = destinationUrl
authz_header = h.getBasicAuthHeaderFromURL(appConfig['couchdb.url.dbadmin']);
authz_header.update( { 'Content-Type': 'application/json'})
request = urllib2.Request(urlparse.urljoin(appConfig['couchdb.url'], '_replicator'),
headers=authz_header,
data = json.dumps(replicationOptions))
log.info("\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}".format(
sourceUrl, destinationUrl, pprint.pformat(replicationOptions)))
results = json.load(urllib2.urlopen(request))
connectionInfo['replication_results'] = results
distributeResults.put(connectionInfo)
log.debug("Replication results: " + pprint.pformat(results))
def index(self, format='html'):
"""GET /policy: All items in the collection"""
if sourceLRNode.isServiceAvailable(NodeServiceModel.ADMINISTRATIVE) == False:
return "Administrative service is not available"
return json.dumps(sourceLRNode.networkPolicyDescription.specData)
def _getDistributeDestinations(self):
""""Method to test the connections and returns a list of destionation node
if the connections are valid"""
nodeDestinationList = []
gatewayConnectionList = []
for connection in sourceLRNode.connections:
# Make sure that the connection is active
if connection.active == False:
continue
destinationLRNode = None
if connection.gateway_connection == True:
gatewayConnectionList.append(connection)
try:
# Make sure we only have one slash in the url path. More than one
#confuses pylons routing libary.
destinationURL = urlparse.urljoin(
connection.destination_node_url.strip(), "distribute")
request = urllib2.Request(destinationURL)
credential = sourceLRNode.getDistributeCredentialFor(
destinationURL)
if credential is not None:
base64string = base64.encodestring(
'%s:%s' % (credential['username'],
credential['password'])).replace("\n", "")
request.add_header("Authorization",
"Basic %s" % base64string)
log.info("\n\nAccess destination node at: " +
pprint.pformat(request.__dict__))
distributeInfo = json.load(urllib2.urlopen(request))
destinationLRNode = LRNodeModel(distributeInfo['node_config'])
except Exception as ex:
log.exception(ex)
continue
# Use of local variable to store if the connection is gateway connection. It is
# done this way to deal with mismatch between node de and connection
# description.
isGatewayConnection = (
(sourceLRNode.nodeDescription.gateway_node == True)
and (destinationLRNode.nodeDescription.gateway_node == True))
# Skip the connection if there is any mismatch between the connection and
# the node data.
if isGatewayConnection != connection.gateway_connection:
log.info(
"Skip connection. 'gateway_connection' mismatch between node and connection data"
)
continue
# Only one gateway connection is allowed, faulty network description
if len(gatewayConnectionList) > 1:
log.info(
"***Abort distribution. More than one gateway node connection"
)
#Clear the node destination list no distribution is network description
# is faulty
nodeDestinationList = []
break
#Calcuate if the connection is gateway one, if so
#cannot distribute across non social communities
if ((sourceLRNode.communityDescription.community_id !=
destinationLRNode.communityDescription.community_id) and
((sourceLRNode.communityDescription.social_community == False)
or (destinationLRNode.communityDescription.social_community
== False))):
log.info("Cannot distribute across non social communities")
continue
# Cannot distribute across networks (or communities) unless gateway
if ((isGatewayConnection == False) and
((sourceLRNode.communityDescription.community_id !=
destinationLRNode.communityDescription.community_id) or
(sourceLRNode.networkDescription.network_id !=
destinationLRNode.networkDescription.network_id))):
log.info(
"Different Network. Cannot distribute across networks (or communities) unless gateway"
)
continue
# Gateway must only distribute across different networks.
if ((isGatewayConnection == True)
and (sourceLRNode.networkDescription.network_id
== destinationLRNode.networkDescription.network_id)):
log.info(
"Gateway must only distribute across different networks")
continue
# Only gateways can distribute on gateway connection. This is really for
# catching mismatch in the data where a connection says it is between
# gateways when the nodes are not both gateways.
if ((connection.gateway_connection == True) and
((sourceLRNode.nodeDescription.gateway_node == False) or
(destinationLRNode.nodeDescription.gateway_node == False))):
log.info("Only gateways can distribute on gateway connection")
continue
nodeInfo = {
"distributeInfo": distributeInfo,
"distribute_sink_url": distributeInfo["distribute_sink_url"],
"destinationNode": destinationLRNode
}
nodeDestinationList.append(nodeInfo)
return nodeDestinationList
def _getDistributeDestinations(self):
""""Method to test the connections and returns a list of destionation node
if the connections are valid"""
nodeDestinationList =[]
gatewayConnectionList = []
for connection in sourceLRNode.connections:
# Make sure that the connection is active
if connection.active == False:
continue
destinationLRNode = None
if connection.gateway_connection == True:
gatewayConnectionList.append(connection)
try:
# Make sure we only have one slash in the url path. More than one
#confuses pylons routing libary.
destinationURL = urlparse.urljoin(connection.destination_node_url.strip(),
"distribute")
request = urllib2.Request(destinationURL)
credential = sourceLRNode.getDistributeCredentialFor(destinationURL)
if credential is not None:
base64string = base64.encodestring('%s:%s' % (credential['username'],credential['password'])).replace("\n", "")
request.add_header("Authorization", "Basic %s" % base64string)
log.info("\n\nAccess destination node at: "+pprint.pformat(request.__dict__))
distributeInfo = json.load(urllib2.urlopen(request))
destinationLRNode = LRNodeModel(distributeInfo['node_config'])
except Exception as ex:
log.exception(ex)
continue
# Use of local variable to store if the connection is gateway connection. It is
# done this way to deal with mismatch between node de and connection
# description.
isGatewayConnection = (
(sourceLRNode.nodeDescription.gateway_node == True) and
(destinationLRNode.nodeDescription.gateway_node ==True))
# Skip the connection if there is any mismatch between the connection and
# the node data.
if isGatewayConnection != connection.gateway_connection:
log.info("Skip connection. 'gateway_connection' mismatch between node and connection data")
continue
# Only one gateway connection is allowed, faulty network description
if len(gatewayConnectionList) > 1:
log.info("***Abort distribution. More than one gateway node connection")
#Clear the node destination list no distribution is network description
# is faulty
nodeDestinationList = []
break
#Calcuate if the connection is gateway one, if so
#cannot distribute across non social communities
if ((sourceLRNode.communityDescription.community_id !=
destinationLRNode.communityDescription.community_id) and
((sourceLRNode.communityDescription.social_community == False) or
(destinationLRNode.communityDescription.social_community == False))):
log.info("Cannot distribute across non social communities")
continue
# Cannot distribute across networks (or communities) unless gateway
if((isGatewayConnection == False) and
((sourceLRNode.communityDescription.community_id !=
destinationLRNode.communityDescription.community_id) or
(sourceLRNode.networkDescription.network_id !=
destinationLRNode.networkDescription.network_id))):
log.info("Different Network. Cannot distribute across networks (or communities) unless gateway")
continue
# Gateway must only distribute across different networks.
if((isGatewayConnection ==True) and
(sourceLRNode.networkDescription.network_id ==
destinationLRNode.networkDescription.network_id)):
log.info("Gateway must only distribute across different networks")
continue
# Only gateways can distribute on gateway connection. This is really for
# catching mismatch in the data where a connection says it is between
# gateways when the nodes are not both gateways.
if((connection.gateway_connection == True) and
((sourceLRNode.nodeDescription.gateway_node == False) or
(destinationLRNode.nodeDescription.gateway_node == False))):
log.info("Only gateways can distribute on gateway connection")
continue
nodeInfo = { "distributeInfo": distributeInfo,
"destinationBaseUrl":connection.destination_node_url,
"destinationNode":destinationLRNode}
nodeDestinationList.append(nodeInfo)
return nodeDestinationList
def index(self, format="html"):
"""GET /status: All items in the collection"""
if sourceLRNode.isServiceAvailable(NodeServiceModel.ADMINISTRATIVE) == False:
return "Administrative service not is available"
return json.dumps(sourceLRNode.status)
