def __init__(self,hostname=get_hostname()): ''' Constructor ''' self.server=hostname self.kernel=Essential() self.db_name =self.kernel.get_db_name() self.db_port = self.kernel.get_db_port() self.connection = MongoClient(hostname, self.db_port) self.db=self.connection[self.db_name]
return (_ple_deployment['path']) try: from yanoama.core.essential import Essential, \ check_hostname, log, build_hosts_file from yanoama.backend.mongo import Mongo except ImportError: sys.path.append(get_install_path()) #import yanoama modules alternatively from yanoama.core.essential import Essential, \ check_hostname, log, build_hosts_file from yanoama.backend.mongo import Mongo if __name__ == '__main__': db=Mongo() kernel=Essential() for coordinator in kernel.get_coordinators(): if db.am_i_a_member_of_this(coordinator): build_hosts_file(coordinator) sys.exit(0) build_hosts_file() hostname='coordinator' log(hostname+' ('+check_hostname(hostname)+'), done.') sys.exit(0)
class Mongo: ''' classdocs ''' def __init__(self,hostname=get_hostname()): ''' Constructor ''' self.server=hostname self.kernel=Essential() self.db_name =self.kernel.get_db_name() self.db_port = self.kernel.get_db_port() self.connection = MongoClient(hostname, self.db_port) self.db=self.connection[self.db_name] def convert_dict_keys_from_dots_to_colons(self,my_dict): #this assumes that #keys are stringsjust the first key #has colons converted_dict={} for key in my_dict.keys(): converted_dict[key.replace('.',':')]=my_dict[key] return converted_dict def recover_dict_keys_from_colons_to_dots(self,my_dict): #this assumes that #keys are stringsjust the first key #has colons converted_dict={} for key in my_dict.keys(): converted_dict[key.replace(':','.')]=my_dict[key] return converted_dict def cleanup_nodes(self,hostname_list,nodes): #clean up for hostname in hostname_list: del nodes[hostname] def remove_indexes_nodes(self,indexes_list,nodes): #clean up hostname_list=[] hostnames=nodes.keys() for index in indexes_list: hostname_list.append(hostnames[index]) self.cleanup_nodes(hostname_list, nodes) def save_raw_nodes(self,nodes): #this randomly selects nodes #first convert hostnames #comma-separated to dot- #separated, mongo db does #not accept keys with dots latency=self.kernel.get_latency() group_size=self.kernel.get_group_size(self.server) converted_nodes=self.convert_dict_keys_from_dots_to_colons(nodes) too_far_nodes=[] #first clean up nodes that are too far for hostname in converted_nodes: if converted_nodes[hostname]>latency: too_far_nodes.append(hostname) #clean up self.cleanup_nodes(too_far_nodes, converted_nodes) #pick nodes randomly start=0 stop=len(converted_nodes) random_indexes=sample(xrange(start,stop),\ max(0,stop-group_size)) self.remove_indexes_nodes(random_indexes, converted_nodes) nodes_latency_measurements=self.db.nodes_latency_measurements nodes_latency_measurements.drop() nodes_latency_measurements.insert(converted_nodes) def get_peers(self,hostname=get_hostname()): """Connects to a coordinator MongoBD database and retrieves the list of member nodes of this storage domain. It gets information from db['nodes_latency_measurements'] entry. Port number and database name come from the main configuration file (yanoama.conf) Keyword arguments: coordinator -- coordinator to connect and retrieve information (default localhost) Returns: dictionary -- members information where keys are the hostnames """ if hostname == self.server: return self.recover_dict_keys_from_colons_to_dots(self.db['nodes_latency_measurements'].find_one({}, {'_id': False}).copy()) else: connection = MongoClient(hostname, self.db_port) db=connection[self.db_name] return self.recover_dict_keys_from_colons_to_dots(db['nodes_latency_measurements'].find_one({}, {'_id': False}).copy()) def save_peers(self,nodes): """Save the current list of member nodes to the local instance of MongoDB database. Port and database name come from the main configuration file (/etc/yanoama.conf). Keyword arguments: nodes -- list of current members of this storage domain """ peers=self.db.peers peers.drop() peers.insert(self.convert_dict_keys_from_dots_to_colons(nodes)) def am_i_a_member_of_this(self,coordinator): """Checks if this node is member of the 'coordinator' storage domain. Port and database name come from the main configuration file (/etc/yanoama.conf). Keyword arguments: nodes -- coordinator hostname Returns: boolean -- member or not """ connection = MongoClient(coordinator, self.db_port) db = connection[self.db_name] recovered_peers = self.recover_dict_keys_from_colons_to_dots(db['peers'].find_one({}, {'_id': False})) return get_hostname() in recovered_peers.keys()
import os import sys try: from yanoama.core.essential import Essential, \ install_cron_job, get_hostname except ImportError: sys.path.append(get_install_path()) #import yanoama modules alternatively from yanoama.core.essential import Essential, \ install_cron_job, get_hostname #deployment is based on the node role #that's either coordinator or peer if __name__ == '__main__': """Deploy yanoama based on the node role, either peer or coordinator. """ kernel=Essential() info="" if (len(sys.argv)==2): if sys.argv[1] == "db_port": sys.stdout.write("%d" % kernel.get_db_port()) else: sys.stdout.write('') sys.exit(0)
from yanoama.core.essential import Essential, \ get_hostname, log from yanoama.backend.mongo import Mongo except ImportError: sys.path.append(get_install_path()) #import yanoama modules alternatively from yanoama.core.essential import Essential, \ get_hostname, log from yanoama.backend.mongo import Mongo if __name__ == '__main__': kernel=Essential() db = Mongo() local_peers=db.get_peers() for coordinator in kernel.get_coordinators(get_hostname()): peers=db.get_peers(coordinator) to_be_removed=[] for hostname in peers.keys(): if hostname in local_peers: if local_peers[hostname]>peers[hostname]: to_be_removed.append(hostname) for hostname in to_be_removed: del local_peers[hostname] db.save_peers(local_peers) #log/print out to the standard output log('current number of members:'+str(len(local_peers.keys()))+', done.')
except ImportError: sys.path.append(get_install_path()) #import yanoama modules alternatively from yanoama.core.essential import Essential, \ install_cron_job, get_hostname #deployment is based on the node role #that's either coordinator or peer if __name__ == '__main__': """Deploy yanoama based on the node role, either peer or coordinator. """ kernel=Essential() #update services file with temp file, #rest must be done by command line temp_services_file='/tmp/services' f = open(temp_services_file, 'w') f.write('#local services'+'\n') f.write('pilot\t\t'+str(kernel.get_pilot_port())+'/tcp\n') f.write('mongo\t\t'+str(kernel.get_db_port())+'/tcp\n') f.write('mongo_replication\t\t'+str(kernel.get_db_replication_port())+'/tcp\n') f.close() if get_hostname() in kernel.get_coordinators().keys(): """this is a coordinator #here the two-step installation procedure #1) install cron jobs: get_rtt.py and # peering.py