Exemplo n.º 1
0
def rsync_filestore_setup(global_objects, operation="check"):
    '''
    TBD
    '''
    _hostname = global_objects["filestore"]["hostname"]
    _protocol = global_objects["filestore"]["protocol"]
    _username = global_objects["filestore"]["username"]
    _port = global_objects["filestore"]["port"]
    _usage = global_objects["filestore"]["usage"].lower()
    _base_dir = global_objects["space"]["base_dir"]
    _stores_wk_dir = global_objects["space"]["stores_working_dir"]
    _log_dir = global_objects["space"]["log_dir"]

    try:
        _name, _ip = hostname2ip(_hostname)

        if operation == "check":

            if _usage == "shared":

                _hostport = int(global_objects["filestore"]["port"])

                if not pre_check_port(_hostname, _hostport, _protocol):
                    _proc_man = ProcessManagement(username="******")
                    _rsync_pid = _proc_man.get_pid_from_cmdline(
                        "rsync --daemon")

                    _cmd = "rsync --daemon"

                    if not _rsync_pid:
                        _msg = "Unable to detect a shared rsync server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

            else:
                _usage = "private"

                _proc_man = ProcessManagement(username=_username)

                _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf"
                _cmd = "rsync --daemon --config " + _config_file_fn

                if not access(_config_file_fn, F_OK):
                    # File was deleted, but the rsync process is still dangling
                    _proc_man.run_os_command("sudo pkill -9 -f " +
                                             _config_file_fn)

                _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)

                if not _rsyslog_pid:

                    _proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir +
                                             '/' + _username + "_rsyncd.pid")

                    global_objects["filestore"][
                        "port"] = _proc_man.get_free_port(
                            global_objects["filestore"]["port"],
                            protocol="tcp")

                    _hostport = int(global_objects["filestore"]["port"])

                    _config_file_contents = global_objects["filestore"][
                        "config_string"].replace('_', ' ')
                    _config_file_contents = _config_file_contents.replace(
                        "DOLLAR", '$')
                    _config_file_contents = _config_file_contents.replace(
                        "REPLEQUAL", '=')
                    _config_file_contents = _config_file_contents.replace(
                        "REPLPORT", str(_hostport))
                    _config_file_contents = _config_file_contents.replace(
                        "REPLLOGDIR", _log_dir)
                    _config_file_contents = _config_file_contents.replace(
                        "REPLUSERU", _username + '_')
                    _config_file_contents = _config_file_contents.replace(
                        "REPLUSER", _username)
                    _config_file_contents = _config_file_contents.replace(
                        "REPLBASEDIR", _base_dir)
                    _config_file_contents = _config_file_contents.replace(
                        "REPLSTORESWORKINGDIR",
                        global_objects["space"]["stores_working_dir"])
                    _config_file_contents = _config_file_contents.replace(
                        ';', '\n')
                    _config_file_contents = _config_file_contents.replace(
                        "--", ';')

                    _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf"
                    _config_file_fd = open(_config_file_fn, 'w')
                    _config_file_fd.write(_config_file_contents)
                    _config_file_fd.close()

                    _rsyslog_pid = _proc_man.start_daemon("sudo " + _cmd)

                    if not _rsyslog_pid:
                        _msg = "Unable to detect a private rsyslog server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

                else:
                    _config_file_fd = open(_config_file_fn, 'r')
                    _config_file_contents = _config_file_fd.readlines()
                    _config_file_fd.close()

                    for _line in _config_file_contents:
                        if _line.count("port="):
                            global_objects["filestore"]["port"] = _line.split(
                                '=')[1]
                            _hostport = int(
                                global_objects["filestore"]["port"])
                            break

        _nh_conn = Nethashget(_hostname)

        _nh_conn.nmap(_hostport, _protocol)
        _msg = "A File Store of the kind \"rsync\" (" + _usage + ") "
        _msg += "on node " + _hostname + ", " + _protocol
        _msg += " port " + str(_hostport) + " seems to be running."
        cbdebug(_msg)
        _status = 0
        return _status, _msg

    except socket.herror:
        _status = 1200
        _msg = "The IP address \"" + _hostname + "\" - used by the rsync "
        _msg += " daemon - is not mapped to a Hostname. "
        _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
        raise StoreSetupException(_msg, 9)

    except socket.gaierror:
        _status = 1200
        _msg = "The Hostname \"" + _hostname + "\" - used by the rsync"
        _msg += " daemon - is not mapped to an IP. "
        _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
        raise StoreSetupException(_msg, 9)

    except ProcessManagement.ProcessManagementException, obj:
        _status = str(obj.status)
        _msg = str(obj.msg)
        raise StoreSetupException(_msg, 9)
Exemplo n.º 2
0
def redis_objectstore_setup(global_objects, operation, cloud_name=None):
    '''
    TBD
    '''
    _protocol = global_objects["objectstore"]["protocol"]
    _hostname = global_objects["objectstore"]["host"]
    _databaseid = int(global_objects["objectstore"]["dbid"])
    _timeout = float(global_objects["objectstore"]["timout"])
    _username = global_objects["objectstore"]["username"]
    _usage = global_objects["objectstore"]["usage"].lower()

    try:
        _instance_dir = global_objects["space"]["instance_dir"]

        if operation == "check":

            _stores_path = global_objects["space"]["stores_working_dir"]
            if not path.exists(_stores_path):
                cbdebug("Making stores working directory: " + _stores_path)
                mkdir(_stores_path)

            if _usage == "shared":
                _hostport = int(global_objects["objectstore"]["port"])
                _proc_man = ProcessManagement(username="******")

                if not pre_check_port(_hostname, _hostport, _protocol):
                    _redis_pid = _proc_man.get_pid_from_cmdline("redis-server")

                    _cmd = "/usr/local/bin/redis-server /etc/redis.conf"
                    if not _redis_pid:
                        _msg = "Unable to detect a shared Redis server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

            else:
                _usage = "private"

                _config_file_fn = _stores_path + '/' + _username + "_redis.conf"
                _cmd = "redis-server " + _config_file_fn

                _proc_man = ProcessManagement(username=_username)

                _redis_pid = _proc_man.get_pid_from_cmdline("redis-server")

                if not _redis_pid:
                    global_objects["objectstore"][
                        "port"] = _proc_man.get_free_port(
                            global_objects["objectstore"]["port"],
                            protocol="tcp")
                    _hostport = int(global_objects["objectstore"]["port"])

                    _config_file_contents = global_objects["objectstore"][
                        "config_string"].replace('_', ' ')
                    _config_file_contents = _config_file_contents.replace(
                        "REPLPORT", str(_hostport))
                    _config_file_contents = _config_file_contents.replace(
                        "REPLSTORESWORKINGDIR",
                        global_objects["space"]["stores_working_dir"])
                    _config_file_contents = _config_file_contents.replace(
                        ';', '\n')

                    _config_file_fd = open(_config_file_fn, 'w')
                    _config_file_fd.write(_config_file_contents)
                    _config_file_fd.close()

                    _redis_pid = _proc_man.start_daemon(_cmd)

                    if not _redis_pid:
                        _msg = "Unable to detect a private Redis server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)
                else:
                    global_objects["objectstore"][
                        "port"] = _proc_man.get_port_from_pid(_redis_pid[0])
                    _hostport = int(global_objects["objectstore"]["port"])

            _nh_conn = Nethashget(_hostname)

            _nh_conn.nmap(_hostport, _protocol)
            _msg = "An Object Store of the kind \"Redis\" (" + _usage + ") "
            _msg += "on node " + _hostname + ", " + _protocol
            _msg += " port " + str(_hostport) + ", database id \""
            _msg += str(_databaseid) + "\" seems to be running."
            _status = 0

        else:
            if not cloud_name:
                raise StoreSetupException(
                    "Name of cloud is required for the 'initialize' mode", 22)

            operation = "initialize"

            _hostport = int(global_objects["objectstore"]["port"])

            _collection_names = [ "reported_management_vm_metric_names", \
                                 "reported_runtime_os_host_metric_names", \
                                 "reported_runtime_os_vm_metric_names", \
                                 "host_management_metrics_header", \
                                 "vm_management_metrics_header", \
                                 "host_runtime_os_metrics_header", \
                                 "vm_runtime_os_metrics_header", \
                                 "vm_runtime_app_metrics_header", \
                                 "trace_header" ]

            for _collection_name in _collection_names:
                for _component in global_objects["mon_defaults"][
                        _collection_name].split(','):
                    if _component.lower() in global_objects["mon_defaults"]:
                        global_objects["mon_defaults"][_collection_name] = \
                        global_objects["mon_defaults"][_collection_name].replace(_component, \
                                                                                 global_objects["mon_defaults"][_component.lower()] + ',')
                global_objects["mon_defaults"][_collection_name] = \
                global_objects["mon_defaults"][_collection_name][:-1].replace(",,",',')

            _rmc = RedisMgdConn(global_objects["objectstore"])

            # First we remove the leftovers from previous experiments.
            if _rmc.initialize_object_store(cloud_name, global_objects, True):
                if not path.exists(_instance_dir):
                    mkdir(_instance_dir)

                for _file_name in listdir(_instance_dir):
                    _file_name = path.join(_instance_dir, _file_name)
                    if path.isdir(_file_name):
                        rmtree(_file_name)

                _msg = "Folders (but not data) underneath experiment "
                _msg += "directory " + _instance_dir + " were removed."
                cbdebug(_msg)

                _msg = "The Redis datastore was successfully initialized on server " + _hostname
                _msg += ", port " + str(_hostport) + ", database id \"" + str(
                    _databaseid)
                _msg += "\"."
                cbdebug(_msg)
                _status = 0

            else:
                _msg = "The Object Store of the kind \"Redis\" was successfully initialized "
                _msg += "on node " + _hostname + ". To change its "
                _msg += "attributes/state, use the *alter commands"
                _msg += "(e.g., cldalter, vmcalter, vmalter) or explicity detach "
                _msg += "and attach this cloud back to this experiment."
                cbdebug(_msg)
                _status = 0

        return _status, _msg

    except NetworkException, obj:
        _msg = "An Object Store of the kind \"Redis\" on node "
        _msg += _hostname + ", " + _protocol + " port " + str(_hostport)
        _msg += ", database id \"" + str(_databaseid)
        _msg += "\" seems to be down: " + str(obj.msg) + '.'
        cberr(_msg)
        raise StoreSetupException(_msg, 8)
Exemplo n.º 3
0
def syslog_logstore_setup(global_objects, operation="check"):
    '''
    TBD
    '''
    _hostname = global_objects["logstore"]["hostname"]
    _protocol = global_objects["logstore"]["protocol"]
    _username = global_objects["logstore"]["username"]
    _usage = global_objects["logstore"]["usage"].lower()
    _stores_wk_dir = global_objects["space"]["stores_working_dir"]
    _log_dir = global_objects["space"]["log_dir"]

    try:
        _name, _ip = hostname2ip(_hostname)

        if operation == "check":

            if _usage == "shared":

                _hostport = int(global_objects["logstore"]["port"])

                if not pre_check_port(_hostname, _hostport, _protocol):
                    _proc_man = ProcessManagement(username="******")
                    _rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd")

                    _cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid "

                    if not _rsyslog_pid:
                        _msg = "Unable to detect a shared rsyslog server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

            else:
                _usage = "private"

                _proc_man = ProcessManagement(username=_username)

                _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf"
                _cmd = "rsyslogd -f " + _config_file_fn + " " + "-i " + _stores_wk_dir + "/rsyslog.pid"

                if not access(_config_file_fn, F_OK):
                    # File was deleted, but the rsyslog process is still dangling
                    _proc_man.run_os_command("sudo pkill -9 -f " +
                                             _config_file_fn)

                if not access(_log_dir, W_OK):
                    # The directory does not even exist, kill any rsyslog processes still dangling
                    _proc_man.run_os_command("sudo pkill -9 -f " +
                                             _config_file_fn)
                    _proc_man.run_os_command("sudo mkdir -p " + _log_dir +
                                             " && sudo chmod 777 " + _log_dir)

                _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)

                if not _rsyslog_pid:

                    global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"],\
                                                                                 protocol = "udp")
                    _hostport = int(global_objects["logstore"]["port"])

                    _config_file_contents = global_objects["logstore"][
                        "config_string"].replace('_', ' ')
                    _config_file_contents = _config_file_contents.replace(
                        "DOLLAR", '$')
                    _config_file_contents = _config_file_contents.replace(
                        "RSYSLOG", "RSYSLOG_")
                    _config_file_contents = _config_file_contents.replace(
                        "REPLPORT", str(_hostport))
                    _config_file_contents = _config_file_contents.replace(
                        "REPLLOGDIR", _log_dir)
                    _config_file_contents = _config_file_contents.replace(
                        "REPLUSER", _username + '_')
                    _config_file_contents = _config_file_contents.replace(
                        ';', '\n')
                    _config_file_contents = _config_file_contents.replace(
                        "--", ';')

                    _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf"
                    _config_file_fd = open(_config_file_fn, 'w')
                    _config_file_fd.write(_config_file_contents)
                    _config_file_fd.close()

                    _rsyslog_pid = _proc_man.start_daemon(_cmd)

                    if not _rsyslog_pid:
                        _msg = "Unable to detect a private rsyslog server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

                else:
                    _config_file_fd = open(_config_file_fn, 'r')
                    _config_file_contents = _config_file_fd.readlines()
                    _config_file_fd.close()

                    for _line in _config_file_contents:
                        if _line.count("UDPServerRun"):
                            global_objects["logstore"]["port"] = _line.split(
                            )[1]
                            _hostport = int(global_objects["logstore"]["port"])
                            break

        _nh_conn = Nethashget(_hostname)

        _nh_conn.nmap(_hostport, _protocol)
        _msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") "
        _msg += "on node " + _hostname + ", " + _protocol
        _msg += " port " + str(_hostport) + " seems to be running."
        cbdebug(_msg)
        _status = 0
        return _status, _msg

    except socket.herror:
        _status = 1200
        _msg = "The IP address \"" + _hostname + "\" - used by the rsyslog "
        _msg += " daemon - is not mapped to a Hostname. "
        _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
        raise StoreSetupException(_msg, 9)

    except socket.gaierror:
        _status = 1200
        _msg = "The Hostname \"" + _hostname + "\" - used by the rsyslog"
        _msg += " daemon - is not mapped to an IP. "
        _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
        raise StoreSetupException(_msg, 9)

    except ProcessManagement.ProcessManagementException, obj:
        _status = str(obj.status)
        _msg = str(obj.msg)
        raise StoreSetupException(_msg, 9)
Exemplo n.º 4
0
def mongodb_metricstore_setup(global_objects, operation="check"):
    '''
    TBD
    '''
    _protocol = global_objects["metricstore"]["protocol"]
    _hostname = global_objects["metricstore"]["host"]
    _databaseid = global_objects["metricstore"]["database"]
    _timeout = float(global_objects["metricstore"]["timeout"])
    _username = global_objects["mon_defaults"]["username"]
    _usage = global_objects["metricstore"]["usage"].lower()

    try:
        if operation == "check":

            if _usage == "shared":

                _hostport = int(global_objects["metricstore"]["port"])

                if not pre_check_port(_hostname, _hostport, _protocol):
                    _proc_man = ProcessManagement(username="******")
                    _mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")

                    _cmd = "/usr/local/bin/mongod -f /etc/mongod.conf --pidfilepath /var/run/mongod.pid"
                    if not _mongodb_pid:
                        _msg = "Unable to detect a shared MongoDB server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

            else:
                _usage = "private"

                _config_file_fn = global_objects["space"][
                    "stores_working_dir"] + '/' + _username + "_mongod.conf"
                _cmd = "mkdir -p " + global_objects["space"][
                    "stores_working_dir"] + "/logs; mongod -f " + _config_file_fn + " --pidfilepath " + global_objects[
                        "space"]["stores_working_dir"] + "/mongod.pid"

                _proc_man = ProcessManagement(username=_username)
                _mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")

                if not _mongodb_pid:
                    global_objects["metricstore"][
                        "port"] = _proc_man.get_free_port(
                            global_objects["metricstore"]["port"],
                            protocol="tcp")
                    _hostport = int(global_objects["metricstore"]["port"])

                    _config_file_contents = global_objects["metricstore"][
                        "config_string"].replace('_', ' ')
                    _config_file_contents = _config_file_contents.replace(
                        "REPLPORT", str(_hostport))
                    _config_file_contents = _config_file_contents.replace(
                        "REPLSTORESWORKINGDIR",
                        global_objects["space"]["stores_working_dir"])
                    _config_file_contents = _config_file_contents.replace(
                        "--", '=')
                    _config_file_contents = _config_file_contents.replace(
                        ';', '\n')

                    _config_file_fn = global_objects["space"][
                        "stores_working_dir"] + '/' + _username + "_mongod.conf"
                    _config_file_fd = open(_config_file_fn, 'w')
                    _config_file_fd.write(_config_file_contents)
                    _config_file_fd.close()

                    _mongodb_pid = _proc_man.start_daemon(_cmd)

                    sleep(5)

                    if not _mongodb_pid:
                        _msg = "Unable to detect a private MongoDB server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

                else:
                    global_objects["metricstore"][
                        "port"] = _proc_man.get_port_from_pid(_mongodb_pid[0])
                    _hostport = int(global_objects["metricstore"]["port"])

            _nh_conn = Nethashget(_hostname)

            _nh_conn.nmap(_hostport, _protocol)
            _msg = "A Metric Store of the kind \"MongoDB\" (" + _usage + ") "
            _msg += "on node " + _hostname + ", " + _protocol
            _msg += " port " + str(_hostport) + ", database id \"" + str(
                _databaseid)
            _msg += "\" seems to be running."
            cbdebug(_msg)
            _status = 0

        else:
            operation = "initialize"
            _mmc = MongodbMgdConn(global_objects["metricstore"])
            _mmc.initialize_metric_store(_username)

            _msg = "The Metric Store of the kind \"MongoDB\" was successfully initialized "
            _msg += "on node: " + str(global_objects["metricstore"])
            cbdebug(_msg)
            _status = 0

            _status = 0

        return _status, _msg

    except ProcessManagement.ProcessManagementException, obj:
        _status = str(obj.status)
        _msg = str(obj.msg)
        raise StoreSetupException(_msg, 9)
Exemplo n.º 5
0
def redis_objectstore_setup(global_objects, operation, cloud_name = None) :
    '''
    TBD
    '''
    _protocol = global_objects["objectstore"]["protocol"]
    _hostname = global_objects["objectstore"]["host"]
    _databaseid = int(global_objects["objectstore"]["dbid"])
    _timeout = float(global_objects["objectstore"]["timout"])
    _username = global_objects["objectstore"]["username"]
    _usage = global_objects["objectstore"]["usage"].lower()

    try :
        _instance_dir = global_objects["space"]["instance_dir"]

        if operation == "check" :

            _stores_path = global_objects["space"]["stores_working_dir"]
            if not path.exists(_stores_path) :
                cbdebug("Making stores working directory: " + _stores_path)
                mkdir(_stores_path)
                
            if _usage == "shared" :
                _hostport = int(global_objects["objectstore"]["port"])
                _proc_man =  ProcessManagement(username = "******")

                if not pre_check_port(_hostname, _hostport, _protocol) :
                    _redis_pid = _proc_man.get_pid_from_cmdline("redis-server")

                    _cmd = "/usr/local/bin/redis-server /etc/redis.conf"
                    if not _redis_pid :
                        _msg = "Unable to detect a shared Redis server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"                    
                        print _msg
                        exit(8)

            else :
                _usage = "private"

                _config_file_fn = _stores_path + '/' + _username + "_redis.conf"
                _cmd = "redis-server " + _config_file_fn

                _proc_man =  ProcessManagement(username = _username)
                
                _redis_pid = _proc_man.get_pid_from_cmdline("redis-server")      

                if not _redis_pid :
                    global_objects["objectstore"]["port"] = _proc_man.get_free_port(global_objects["objectstore"]["port"], protocol = "tcp")
                    _hostport = int(global_objects["objectstore"]["port"])

                    _config_file_contents = global_objects["objectstore"]["config_string"].replace('_', ' ')
                    _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
                    _config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"])
                    _config_file_contents = _config_file_contents.replace(';','\n')

                    _config_file_fd = open(_config_file_fn, 'w')
                    _config_file_fd.write(_config_file_contents)
                    _config_file_fd.close()

                    _redis_pid = _proc_man.start_daemon(_cmd)

                    if not _redis_pid :
                        _msg = "Unable to detect a private Redis server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)
                else :
                    global_objects["objectstore"]["port"] = _proc_man.get_port_from_pid(_redis_pid[0]) 
                    _hostport = int(global_objects["objectstore"]["port"])

            _nh_conn = Nethashget(_hostname)

            _nh_conn.nmap(_hostport, _protocol)
            _msg = "An Object Store of the kind \"Redis\" (" + _usage + ") "
            _msg += "on node " + _hostname + ", " + _protocol 
            _msg += " port " + str(_hostport) + ", database id \""
            _msg += str(_databaseid) + "\" seems to be running."
            _status = 0

        else :
            if not cloud_name :
                raise StoreSetupException("Name of cloud is required for the 'initialize' mode", 22)
            
            operation = "initialize"

            _hostport = int(global_objects["objectstore"]["port"])

            _collection_names = [ "reported_management_vm_metric_names", \
                                 "reported_runtime_os_host_metric_names", \
                                 "reported_runtime_os_vm_metric_names", \
                                 "host_management_metrics_header", \
                                 "vm_management_metrics_header", \
                                 "host_runtime_os_metrics_header", \
                                 "vm_runtime_os_metrics_header", \
                                 "vm_runtime_app_metrics_header", \
                                 "trace_header" ]

            for _collection_name in _collection_names : 
                for _component in global_objects["mon_defaults"][_collection_name].split(',') :
                    if _component.lower() in global_objects["mon_defaults"] :
                        global_objects["mon_defaults"][_collection_name] = \
                        global_objects["mon_defaults"][_collection_name].replace(_component, \
                                                                                 global_objects["mon_defaults"][_component.lower()] + ',')
                global_objects["mon_defaults"][_collection_name] = \
                global_objects["mon_defaults"][_collection_name][:-1].replace(",,",',')

            _rmc = RedisMgdConn(global_objects["objectstore"])

            # First we remove the leftovers from previous experiments.
            if _rmc.initialize_object_store(cloud_name, global_objects, True) :
                if not path.exists(_instance_dir) :
                    mkdir(_instance_dir)

                for _file_name in listdir(_instance_dir) :
                    _file_name = path.join(_instance_dir, _file_name)
                    if path.isdir(_file_name) :
                        rmtree(_file_name)

                _msg = "Folders (but not data) underneath experiment "
                _msg += "directory " + _instance_dir + " were removed."
                cbdebug(_msg)

                _msg = "The Redis datastore was successfully initialized on server " + _hostname
                _msg += ", port " + str(_hostport) + ", database id \"" + str(_databaseid)
                _msg += "\"."
                cbdebug(_msg)
                _status = 0

            else :
                _msg = "The Object Store of the kind \"Redis\" was successfully initialized "
                _msg += "on node " + _hostname + ". To change its "
                _msg += "attributes/state, use the *alter commands"
                _msg += "(e.g., cldalter, vmcalter, vmalter) or explicity detach "
                _msg += "and attach this cloud back to this experiment."
                cbdebug(_msg)
                _status = 0

        return _status, _msg
    
    except NetworkException, obj :
        _msg = "An Object Store of the kind \"Redis\" on node "
        _msg += _hostname + ", " + _protocol + " port " + str(_hostport)
        _msg += ", database id \"" + str(_databaseid)
        _msg += "\" seems to be down: " + str(obj.msg) + '.'
        cberr(_msg)
        raise StoreSetupException(_msg, 8)
Exemplo n.º 6
0
def rsync_filestore_setup(global_objects, operation = "check") :
    '''
    TBD
    '''
    _hostname = global_objects["filestore"]["hostname"]
    _protocol = global_objects["filestore"]["protocol"]
    _username = global_objects["filestore"]["username"]
    _port = global_objects["filestore"]["port"]
    _usage = global_objects["filestore"]["usage"].lower()
    _base_dir = global_objects["space"]["base_dir"]
    _stores_wk_dir = global_objects["space"]["stores_working_dir"]
    _log_dir = global_objects["space"]["log_dir"]

    try :
        _name, _ip = hostname2ip(_hostname)        
        
        if operation == "check" :

            if _usage == "shared" :

                _hostport = int(global_objects["filestore"]["port"])
                
                if not pre_check_port(_hostname, _hostport, _protocol) :
                    _proc_man =  ProcessManagement(username = "******")
                    _rsync_pid = _proc_man.get_pid_from_cmdline("rsync --daemon")
    
                    _cmd = "rsync --daemon"

                    if not _rsync_pid :
                        _msg = "Unable to detect a shared rsync server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"                    
                        print _msg
                        exit(8)

            else :
                _usage = "private"

                _proc_man =  ProcessManagement(username = _username)
                
                _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf"
                _cmd = "rsync --daemon --config " + _config_file_fn

                if not access(_config_file_fn, F_OK) :
                    # File was deleted, but the rsync process is still dangling
                    _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn)

                _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)

                if not _rsyslog_pid :

                    _proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _username + "_rsyncd.pid")
                    
                    global_objects["filestore"]["port"] = _proc_man.get_free_port(global_objects["filestore"]["port"], protocol = "tcp")

                    _hostport = int(global_objects["filestore"]["port"])
                    
                    _config_file_contents = global_objects["filestore"]["config_string"].replace('_', ' ')
                    _config_file_contents = _config_file_contents.replace("DOLLAR", '$')
                    _config_file_contents = _config_file_contents.replace("REPLEQUAL", '=')                    
                    _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
                    _config_file_contents = _config_file_contents.replace("REPLLOGDIR", _log_dir)
                    _config_file_contents = _config_file_contents.replace("REPLUSERU", _username + '_')
                    _config_file_contents = _config_file_contents.replace("REPLUSER", _username)                    
                    _config_file_contents = _config_file_contents.replace("REPLBASEDIR", _base_dir)
                    _config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"])                                         
                    _config_file_contents = _config_file_contents.replace(';','\n')
                    _config_file_contents = _config_file_contents.replace("--", ';')

                    _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf"
                    _config_file_fd = open(_config_file_fn, 'w')
                    _config_file_fd.write(_config_file_contents)
                    _config_file_fd.close()
                    
                    _rsyslog_pid = _proc_man.start_daemon("sudo " + _cmd)

                    if not _rsyslog_pid :
                        _msg = "Unable to detect a private rsyslog server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

                else :
                    _config_file_fd = open(_config_file_fn, 'r')
                    _config_file_contents = _config_file_fd.readlines()
                    _config_file_fd.close()

                    for _line in _config_file_contents :
                        if _line.count("port=") :
                            global_objects["filestore"]["port"] = _line.split('=')[1]
                            _hostport = int(global_objects["filestore"]["port"])
                            break

        _nh_conn = Nethashget(_hostname)
        
        _nh_conn.nmap(_hostport, _protocol)
        _msg = "A File Store of the kind \"rsync\" (" + _usage + ") "
        _msg += "on node " + _hostname + ", " + _protocol
        _msg += " port " + str(_hostport) + " seems to be running."
        cbdebug(_msg)
        _status = 0
        return _status, _msg

    except socket.herror:
        _status = 1200
        _msg = "The IP address \"" + _hostname + "\" - used by the rsync "
        _msg += " daemon - is not mapped to a Hostname. "
        _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
        raise StoreSetupException(_msg, 9)

    except socket.gaierror:
        _status = 1200
        _msg = "The Hostname \"" + _hostname + "\" - used by the rsync"
        _msg += " daemon - is not mapped to an IP. "
        _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
        raise StoreSetupException(_msg, 9)
    
    except ProcessManagement.ProcessManagementException, obj :
        _status = str(obj.status)
        _msg = str(obj.msg)
        raise StoreSetupException(_msg, 9)
Exemplo n.º 7
0
def mongodb_metricstore_setup(global_objects, operation = "check") :
    '''
    TBD
    '''
    _protocol = global_objects["metricstore"]["protocol"]
    _hostname = global_objects["metricstore"]["host"]
    _databaseid = global_objects["metricstore"]["database"]
    _timeout = float(global_objects["metricstore"]["timeout"])
    _username = global_objects["mon_defaults"]["username"]
    _usage = global_objects["metricstore"]["usage"].lower()

    try :
        if operation == "check" :

            if _usage == "shared" :          

                _hostport = int(global_objects["metricstore"]["port"])
                
                if not pre_check_port(_hostname, _hostport, _protocol) :
                    _proc_man =  ProcessManagement(username = "******")
                    _mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")
    
                    _cmd = "/usr/local/bin/mongod -f /etc/mongod.conf --pidfilepath /var/run/mongod.pid"
                    if not _mongodb_pid :
                        _msg = "Unable to detect a shared MongoDB server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"                    
                        print _msg
                        exit(8)

            else :
                _usage = "private"

                _config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_mongod.conf"
                _cmd = "mkdir -p " + global_objects["space"]["stores_working_dir"]  + "/logs; mongod -f " + _config_file_fn + " --pidfilepath " + global_objects["space"]["stores_working_dir"] + "/mongod.pid"
    
                _proc_man =  ProcessManagement(username = _username)
                _mongodb_pid = _proc_man.get_pid_from_cmdline("mongod -f")

                if not _mongodb_pid :
                    global_objects["metricstore"]["port"] = _proc_man.get_free_port(global_objects["metricstore"]["port"], protocol = "tcp")
                    _hostport = int(global_objects["metricstore"]["port"])

                    _config_file_contents = global_objects["metricstore"]["config_string"].replace('_', ' ')
                    _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
                    _config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"])
                    _config_file_contents = _config_file_contents.replace("--", '=')
                    _config_file_contents = _config_file_contents.replace(';','\n')

                    _config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_mongod.conf"
                    _config_file_fd = open(_config_file_fn, 'w')
                    _config_file_fd.write(_config_file_contents)
                    _config_file_fd.close()
    
                    _mongodb_pid = _proc_man.start_daemon(_cmd)
                    
                    sleep(5)

                    if not _mongodb_pid :
                        _msg = "Unable to detect a private MongoDB server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

                else :
                    global_objects["metricstore"]["port"] = _proc_man.get_port_from_pid(_mongodb_pid[0])
                    _hostport = int(global_objects["metricstore"]["port"])

            _nh_conn = Nethashget(_hostname)

            _nh_conn.nmap(_hostport, _protocol)
            _msg = "A Metric Store of the kind \"MongoDB\" (" + _usage + ") "
            _msg += "on node " + _hostname + ", " + _protocol
            _msg += " port " + str(_hostport) + ", database id \"" + str(_databaseid)
            _msg += "\" seems to be running."
            cbdebug(_msg)
            _status = 0

        else:
            operation = "initialize"
            _mmc = MongodbMgdConn(global_objects["metricstore"])
            _mmc.initialize_metric_store(_username)
            
            _msg = "The Metric Store of the kind \"MongoDB\" was successfully initialized "
            _msg += "on node: " + str(global_objects["metricstore"])
            cbdebug(_msg)
            _status = 0
            
            _status = 0
            
        return _status, _msg

    except ProcessManagement.ProcessManagementException, obj :
        _status = str(obj.status)
        _msg = str(obj.msg)
        raise StoreSetupException(_msg, 9)
Exemplo n.º 8
0
def syslog_logstore_setup(global_objects, operation = "check") :
    '''
    TBD
    '''
    _hostname = global_objects["logstore"]["hostname"]
    _protocol = global_objects["logstore"]["protocol"]
    _username = global_objects["logstore"]["username"]
    _usage = global_objects["logstore"]["usage"].lower()
    _stores_wk_dir = global_objects["space"]["stores_working_dir"]
    _log_dir = global_objects["space"]["log_dir"]

    try :
        _name, _ip = hostname2ip(_hostname)        
        
        if operation == "check" :

            if _usage == "shared" :

                _hostport = int(global_objects["logstore"]["port"])
                
                if not pre_check_port(_hostname, _hostport, _protocol) :
                    _proc_man =  ProcessManagement(username = "******")
                    _rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd")
    
                    _cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid "

                    if not _rsyslog_pid :
                        _msg = "Unable to detect a shared rsyslog server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"                    
                        print _msg
                        exit(8)

            else :
                _usage = "private"

                _proc_man =  ProcessManagement(username = _username)

                _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf"
                _cmd = "rsyslogd -f " + _config_file_fn + " " + "-i " + _stores_wk_dir + "/rsyslog.pid"

                if not access(_config_file_fn, F_OK) :
                    # File was deleted, but the rsyslog process is still dangling
                    _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn)

                if not access(_log_dir, W_OK) :
                    # The directory does not even exist, kill any rsyslog processes still dangling
                    _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn)                    
                    _proc_man.run_os_command("sudo mkdir -p " + _log_dir + " && sudo chmod 777 " + _log_dir)

                _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)     

                if not _rsyslog_pid :

                    global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"],\
                                                                                 protocol = "udp")
                    _hostport = int(global_objects["logstore"]["port"])

                    _config_file_contents = global_objects["logstore"]["config_string"].replace('_', ' ')
                    _config_file_contents = _config_file_contents.replace("DOLLAR", '$')
                    _config_file_contents = _config_file_contents.replace("RSYSLOG", "RSYSLOG_")
                    _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
                    _config_file_contents = _config_file_contents.replace("REPLLOGDIR", _log_dir)
                    _config_file_contents = _config_file_contents.replace("REPLUSER", _username + '_')                    
                    _config_file_contents = _config_file_contents.replace(';','\n')
                    _config_file_contents = _config_file_contents.replace("--", ';')

                    _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf"
                    _config_file_fd = open(_config_file_fn, 'w')
                    _config_file_fd.write(_config_file_contents)
                    _config_file_fd.close()

                    _rsyslog_pid = _proc_man.start_daemon(_cmd)
             
                    if not _rsyslog_pid :
                        _msg = "Unable to detect a private rsyslog server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

                else :
                    _config_file_fd = open(_config_file_fn, 'r')
                    _config_file_contents = _config_file_fd.readlines()
                    _config_file_fd.close()

                    for _line in _config_file_contents :
                        if _line.count("UDPServerRun") :
                            global_objects["logstore"]["port"] = _line.split()[1]
                            _hostport = int(global_objects["logstore"]["port"])
                            break

        _nh_conn = Nethashget(_hostname)

        _nh_conn.nmap(_hostport, _protocol)
        _msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") "
        _msg += "on node " + _hostname + ", " + _protocol
        _msg += " port " + str(_hostport) + " seems to be running."
        cbdebug(_msg)
        _status = 0
        return _status, _msg

    except socket.herror:
        _status = 1200
        _msg = "The IP address \"" + _hostname + "\" - used by the rsyslog "
        _msg += " daemon - is not mapped to a Hostname. "
        _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
        raise StoreSetupException(_msg, 9)


    except socket.gaierror:
        _status = 1200
        _msg = "The Hostname \"" + _hostname + "\" - used by the rsyslog"
        _msg += " daemon - is not mapped to an IP. "
        _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS."
        raise StoreSetupException(_msg, 9)
    
    except ProcessManagement.ProcessManagementException, obj :
        _status = str(obj.status)
        _msg = str(obj.msg)
        raise StoreSetupException(_msg, 9)
Exemplo n.º 9
0
def syslog_logstore_setup(global_objects, operation = "check") :
    '''
    TBD
    '''
    _hostname = global_objects["logstore"]["hostname"]
    _protocol = global_objects["logstore"]["protocol"]
    _username = global_objects["logstore"]["username"]
    _usage = global_objects["logstore"]["usage"].lower()

    try :
        if operation == "check" :

            if _usage == "shared" :

                _hostport = int(global_objects["logstore"]["port"])
                
                if not pre_check_port(_hostname, _hostport, _protocol) :
                    _proc_man =  ProcessManagement(username = "******")
                    _rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd")
    
                    _cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid -c 5"
                    if not _rsyslog_pid :
                        _msg = "Unable to detect a shared rsyslog server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"                    
                        print _msg
                        exit(8)

            else :
                _usage = "private"

                _config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_rsyslog.conf"
                _cmd = "rsyslogd -f " + _config_file_fn + " -c 4 " + "-i " + global_objects["space"]["stores_working_dir"] + "/rsyslog.pid"

                _proc_man =  ProcessManagement(username = _username)
                _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd)     

                if not _rsyslog_pid :
                    global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"], protocol = "udp")
                    _hostport = int(global_objects["logstore"]["port"])
                    
                    _config_file_contents = global_objects["logstore"]["config_string"].replace('_', ' ')
                    _config_file_contents = _config_file_contents.replace("DOLLAR", '$')
                    _config_file_contents = _config_file_contents.replace("RSYSLOG", "RSYSLOG_")
                    _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport))
                    _config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"])
                    _config_file_contents = _config_file_contents.replace(';','\n')
                    _config_file_contents = _config_file_contents.replace("--", ';')

                    _config_file_fn = global_objects["space"]["stores_working_dir"] + '/' + _username + "_rsyslog.conf"
                    _config_file_fd = open(_config_file_fn, 'w')
                    _config_file_fd.write(_config_file_contents)
                    _config_file_fd.close()

                    _rsyslog_pid = _proc_man.start_daemon(_cmd)
             
                    if not _rsyslog_pid :
                        _msg = "Unable to detect a private rsyslog server daemon running. "
                        _msg += "Please try to start one (e.g., " + _cmd + ")"
                        print _msg
                        exit(8)

                else :
                    _config_file_fd = open(_config_file_fn, 'r')
                    _config_file_contents = _config_file_fd.readlines()
                    _config_file_fd.close()

                    for _line in _config_file_contents :
                        if _line.count("UDPServerRun") :
                            global_objects["logstore"]["port"] = _line.split()[1]
                            _hostport = int(global_objects["logstore"]["port"])
                            break

        _nh_conn = Nethashget(_hostname)

        _nh_conn.nmap(_hostport, _protocol)
        _msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") "
        _msg += "on node " + _hostname + ", " + _protocol
        _msg += " port " + str(_hostport) + " seems to be running."
        cbdebug(_msg)
        _status = 0
        return _status, _msg

    except ProcessManagement.ProcessManagementException, obj :
        _status = str(obj.status)
        _msg = str(obj.msg)
        raise StoreSetupException(_msg, 9)