Exemplo n.º 1
0
class FGInstances:

    instance = {}
    userinfo = []
    projectinfo = {}

    default_projectinfo = {"ProjectId": None,
                           "Title": None,
                           "Institution": None,
                           "ProjectLead": None,
                           "Discipline": None}

                           # "Completed":None,
                           #"Department":None",
                           #"Keywords":None,
                           #"Results":None}
    pp = None
    db = None
    in_the_future = None
    in_the_past = None
    first_date = None
    last_date = None
    cloudplatform = None

    def __init__(self):

        self.db = FGDatabase()
        self.in_the_future = datetime.strptime(
            "3000-01-01 00:00:00", '%Y-%m-%d %H:%M:%S')
        self.in_the_past = datetime.strptime(
            "1970-01-01 00:00:00", '%Y-%m-%d %H:%M:%S')
        self.pp = pprint.PrettyPrinter(indent=0)
        self.first_date = datetime.strptime(
            "3000-01-01 00:00:00", '%Y-%m-%d %H:%M:%S')
        self.last_date = datetime.strptime(
            "1981-01-01 00:00:00", '%Y-%m-%d %H:%M:%S')

    def clear(self, opts=None):
        if not opts or opts == "instance":
            self.instance = {}
        if not opts or opts == "userinfo":
            self.userinfo = []
        if not opts or opts == "projectinfo":
            self.projectinfo = {}

    def get_instance(self, index=None):
        return self.get_data(index)

    def get_data(self, index=None, withInfo=False):
        return self.get_instance_with_info(index, withInfo)

    def get_instance_with_info(self, index, withInfo):

        # This should be returned with same type but this isn't. if
        # index comes with it, returns 'dict' and without it, returns
        # list. what a unconsistent.
        if isinstance(index, (int, long)):
            instances = [self.instance[index]]
        else:
            instances = self.instance

        if not withInfo:
            return instances

        res = []
        for ins in instances:
            try:
                userinfo = self.get_userinfo({"ownerid": ins["ownerId"],
                                              "username": ins["ownerId"]})
                if userinfo:
                    ins = dict(userinfo.items() + ins.items())

                    # Extension for project information
                    prj_id = userinfo["project"]
                    if prj_id:
                        if prj_id.startswith("fg"):
                            prj_id = int(prj_id[2:])
                        projectinfo = self.get_projectinfo(
                            prj_id)  # Remove 'fg' prefix
                        if projectinfo:
                            ins = dict(projectinfo.items() + ins.items())
                else:
                    ins = dict(ins.items() + self.default_projectinfo.items())

                res.append(ins)
            except:
                pass

        # MISSING error exception for empty res return. it may occur
        # index error of the call in FGMetrics. i.e. ... [0]
        return res

    """

    REPLACEMENT FOR get_userinfo get_projectinfo

    def _get_info(self, type, index=None):
        if isinstance(index, (int, long)):
            return type[index]
        elif isinstance(index, (dict)):
            # Search
            for key, val in index.iteritems():
                for element in type:
                    if key in element and type[key] == val:
                        return element
        elif index in None:
            return type
        else:
            return None

    def get_userinfo(self, index=None):
        return _get_info(self.userinfo, index):

    def get_projectinfo(self, index=None):
        return _get_info(self.projectinfo, index):

    """

    def get_userinfo(self, index=None):
        """should be replaced with above code"""
        if isinstance(index, (int, long)):
            return self.userinfo[index]
        elif isinstance(index, (dict)):
            # Search
            for key, val in index.iteritems():
                for userinfo in self.userinfo:
                    if key in userinfo and userinfo[key] == val:
                        return userinfo
        elif index is None:
            if not self.userinfo:
                self.read_userinfo_from_db()
            return self.userinfo
        else:
            return None

    def get_projectinfo(self, index=None):
        """should be replaced with above code"""
        if isinstance(index, (int, long)):
            return self.projectinfo[index]
        elif isinstance(index, (dict)):
            for key, val in index.iteritems():
                for projectinfo in self.projectinfo:
                    if key in projectinfo and projectinfo[key] == val:
                        return projectinfo
        elif index is None:
            if not self.projectinfo:
                self.read_projectinfo_from_db()
            return self.projectinfo
        else:
            return None

    def count(self):
        return len(self.instance)

    def read_from_db(self):
        self.read_instances()

    def read_instances(self, querydict={}, optional=""):
        self.clear("instance")
        self.instance = self.db.read(querydict, optional)

    def read_projectinfo_from_db(self):

        res = {}
        self.clear("projectinfo")
        prjinfo_list = self.db.read_projectinfo()
        res = {element["ProjectId"]: element for element in prjinfo_list}
        self.projectinfo = res

    def read_userinfo_from_db(self):
        self.read_userinfo()

    def read_userinfo(self, querydict={}, optional=""):

        self.clear("userinfo")
        self.userinfo = self.db.read_userinfo(querydict, optional)

    def read_userinfo_detail(self):
        self.clear("userinfo")
        self.userinfo = self.db.get_userinfo_detail()

    def read_cloudplatform(self, refresh=False):
        if not self.cloudplatform or refresh:
            self.cloudplatform = self.db.read_cloudplatform()
        return self.cloudplatform

    def get_cloudplatform_id(self, querydict={}):
        class ContinueOutOfALoop(Exception):
            pass
        for row in self.read_cloudplatform():
            try:
                for key in querydict:
                    if row[key] != querydict[key]:
                        raise ContinueOutOfALoop
                return row["cloudPlatformId"]
            except ContinueOutOfALoop:
                continue
        return None

    def write_to_db(self):
        for key_current in self.instance:
            self.db.write(self.instance[key_current])

    def write_userinfo_to_db(self):
        for key_current in self.userinfo:
            self.db.write_userinfo(key_current)

    def add(self, datarecord):
        """prints the information for each instance"""
        if datarecord["linetype"] == "print_ccInstance":
            instanceId = datarecord["instanceId"]
            ownerId = datarecord["ownerId"]
            timestamp = datarecord["ts"]
            status = datarecord["state"].lower()
            t = datarecord["date"]

            id = instanceId + " " + ownerId + " " + str(timestamp)

            instance = self.instance

            try:
                current = instance[id]
                # if were wereto do a data base this line needs to be
                # replaced
            except:
                current = datarecord

            # if not ("t_end" in current):
            try:
                current["t_end"]
            except:
            # time in the future
                f = self.in_the_future

                current["trace"] = {
                    "pending": {"start": f, "stop": t},
                    "teardown": {"start": f, "stop": t},
                    "extant": {"start": f, "stop": t}
                }
                current["t_end"] = current["date"]
                current["t_start"] = current["ts"]  # for naming consitency
                current["duration"] = 0.0

            current["t_end"] = max(current["t_end"], t)
            current["trace"][status]["start"] = min(
                current["trace"][status]["start"], t)
            current["trace"][status]["stop"] = max(
                current["trace"][status]["stop"], t)

            instance[id] = current

    def update_traceinfo(self, row):
        ''' New version of add function  '''
        res = self.set_datetime(row)
        updated_res = self.update_to_list(res)
        return updated_res

    def set_datetime(self, row):

        future = self.in_the_future
        past = self.in_the_past
        # Set datetime
        row["t_start"] = row["ts"]
        row["t_end"] = self.get_t_end(row)
        row["duration"] = self.get_t_delta(row)
        row["trace"] = {
            "pending": {"start": future, "stop": past, "queue": deque("", 10)},
            "extant": {"start": future, "stop": past, "queue": deque("", 10)},
            "teardown": {"start": future, "stop": past, "queue": deque("", 10)}
        }

        return row

    def update_to_list(self, row):
        ''' Reflect to the current list '''

        instanceId = row["instanceId"]
        ownerId = row["ownerId"]
        ts = row["ts"]
        key = instanceId + " " + ownerId + " " + str(ts)

        res = self.update_trace_datetime(key, row)
        # Update - this needs to be changed.
        if row["date"] > res["date"]:
            res["state"] = row["state"]
            res["date"] = row[
                "date"]  # max(res["date"], row["date"]) # we don't need date column
            res["duration"] = row[
                "duration"]  # max(res["duration"], row["duration"])
            res["t_end"] = row["t_end"]  # min(res["t_end"], row["t_end"])
        self.instance[key] = res

        return res

    def update_trace_datetime(self, key, new):

        if key in self.instance:
            old = self.instance[key]
        else:
            old = new

        state = new["state"].lower()
        old["trace"][state]["queue"].append(new["date"])
        queue = old["trace"][state]["queue"]
        old["trace"][state]["start"] = min(
            queue)  # min(old["trace"][state]["start"], new["date"])
        old["trace"][state]["stop"] = max(
            queue)  # max(old["trace"][state]["stop"], new["date"])

        return old

    def get_previous_state(self, state):
        state = state.lower()
        if state == "pending":
            return state
        elif state == "extant":
            return "pending"
        elif state == "teardown":
            return "extant"

    def get_t_end(self, row):
        return row["date"]
        '''
        if row["state"] == "Teardown":
                return row["date"]
        else:
                return self.in_the_future
        '''

    def get_t_delta(self, row):

        start = row["t_start"]
        last = row["date"]

        if row["state"] == "Teardown":
            if row["t_end"]:
                last = max(row["date"], row["t_end"])

        t_delta = (last - start).total_seconds()
        if t_delta < 0:
            t_delta = timedelta(0).total_seconds()
        return t_delta

    def refresh(self):
        """calculates how long each instance runs in seconds"""
        for i in self.instance:
            values = self.instance[i]
            if values["state"] == "Teardown":
                t_delta = values["t_end"] - values["ts"]
            else:
                t_delta = values["date"] - values["ts"]

            if t_delta.total_seconds() < 0:
                t_delta = values["t_end"] - values["t_end"]
            values["duration"] = str(t_delta.total_seconds())

    def set_userinfo(self):
        for i in self.instance:
            ownerid = self.instance[i]["ownerId"]
            try:
                new_userinfo = retrieve_userinfo_ldap(ownerid)
                self.add_userinfo(new_userinfo)
            except:
                continue

    def add_userinfo(self, new_userinfo):
        if self.userinfo.count(new_userinfo) == 0:
            self.userinfo.append(new_userinfo)

    '''
Exemplo n.º 2
0
class FGConverter:

    # future = instances.in_the_future
    # past = instances.in_the_past

    s_date = None
    e_date = None

    platform = None
    platform_version = None
    hostname = None
    confname = None
    dbname_nova = None
    dbname_keystone = None

    query = None
    rows = None  # from database
    records = None  # for fg database

    userinfo = None
    cloudplatform = None

    argparser = None

    def __init__(self):
        self.db = FGDatabase()
        self.db_dest = FGDatabase()

    def __del__(self):
        self.db_close()

    def convert_to_fg(self):
        self.check_platform()
        self.db_connect()
        self.convert_instance()
        self.convert_userinfo()
        self.db_close()

    def check_platform(self):
        _check = getattr(self, "_check_platform_" + self.platform)
        _check()

    def _check_platform_nimbus(self):

        self.platform_version = self.platform_version or FGConst.DEFAULT_NIMBUS_VERSION
        self.db.db_type = self.db.db_type or FGConst.DEFAULT_NIMBUS_DB

        # this query is for sqlite3 because [timestamp] is only used on
        # sqlite3?
        self.query = 'SELECT t1.time as "t_start [timestamp]",\
                    t3.time as "t_end [timestamp]",\
                    t1.uuid as instanceId,\
                    t2.dn,\
                    t1.cpu_count as ccvm_cores,\
                    t1.memory as ccvm_mem,\
                    t1.vmm as serviceTag \
                    from create_events t1, user t2, remove_events t3 \
                    on t1.user_id=t2.id and t1.uuid=t3.uuid \
                    where t1.time >= \'' + str(self.s_date) + '\' and t3.time <= \'' + str(self.e_date) + '\''

    def _check_platform_openstack(self):

        if not self.dbname_nova or not self.dbname_keystone or not self.db.dbhost or not self.db.dbuser or not self.db.dbpasswd:
            msg = "db info is missing"
            print msg
            raise ValueError(msg)

        self.platform_version = self.platform_version or FGConst.DEFAULT_OPENSTACK_VERSION
        self.db.db_type = self.db.db_type or FGConst.DEFAULT_OPENSTACK_DB
        self.db.dbname = self.dbname_nova

        self.query = 'SELECT created_at as trace_pending_start, \
                    launched_at as trace_extant_start,\
                    terminated_at as trace_teardown_start, \
                    deleted_at as trace_teardown_stop, \
                    id,\
                    user_id as ownerId,\
                    project_id as accountId,\
                    image_ref as emiId,\
                    kernel_id as kernelId,\
                    ramdisk_id as ramdiskId,\
                    key_data as keyName,\
                    vm_state as state,\
                    memory_mb as ccvm_mem, \
                    vcpus as ccvm_cores, \
                    host as serviceTag, \
                    reservation_id as reservationId,\
                    created_at as t_start,  \
                    COALESCE(deleted_at, terminated_at, updated_at) as t_end, \
                    uuid as instanceId, \
                    access_ip_v4 as ccnet_publicIp,\
                    ephemeral_gb as ccvm_disk \
                    from instances \
                    where updated_at >= \'' + str(self.s_date) + '\' and updated_at <= \'' + str(self.e_date) + '\''
        # COALESCE(launched_at, created_at, scheduled_at) as t_start, \

    def db_connect(self):
        self.db.connect()
        self.db_dest.conf()
        self.db_dest.connect()

    def db_close(self):
        self.db.close()
        self.db_dest.close()

    def read_from_source(self):
        self.rows = self.db.query(self.query)

    def write_to_dest(self):
        self.db_dest.write_instance(self.records)

    def convert_instance(self):
        self.read_from_source()
        self.map_to_fg()
        self.write_to_dest()

    def convert_userinfo(self):
        _convert = getattr(self, "_convert_userinfo_of_" + self.platform)
        _convert()

    def _convert_userinfo_of_nimbus(self):
        return

    def _convert_userinfo_of_openstack(self):
        res = self._read_userinfo_of_nova_with_project_info()
        self.db_dest.write_userinfo(res)

    def _read_userinfo_of_nova_with_project_info(self):
        keystone = self.db
        keystone.dbname = self.dbname_keystone
        keystone.connect()
        # This is for Grizzly
        userinfo = keystone.query("select user_id, tenant_id, user.name as \
                                  user_name, tenant.name as tenant_name \
                                  from user_tenant_membership, tenant, user \
                                  where user.id=user_tenant_membership.user_id \
                                  and tenant.id=user_tenant_membership.tenant_id")  # select id, name from user")
        if not userinfo:
            #This is for Folsom
            userinfo = keystone.query("select user_id, project_id as tenant_id,\
                                      user.name as user_name, project.name as \
                                      tenant_name from user_project_metadata, \
                                      project, user where \
                                      user.id=user_project_metadata.user_id and \
                                      project.id=user_project_metadata.project_id")
 
        keystone.close()
        records = []
        for row in userinfo:
            try:
                res = FGUtility.retrieve_userinfo_ldap(row["user_name"])
                if not res:
                    res = {}
                res["ownerid"] = row["user_id"]
                res["username"] = row["user_name"]
                res["project"] = row["tenant_name"]
                res["hostname"] = self.hostname
                print res
                records.append(res)
            except:
                print sys.exc_info()
                raise
        return records

    def read_cloudplatform(self):
        if self.cloudplatform:
            return
        self.cloudplatform = self.db_dest.read_cloudplatform()

    def get_cloudplatform_id(self, querydict={}):
        class ContinueOutOfALoop(Exception):
            pass
        self.read_cloudplatform()
        for row in self.cloudplatform:
            try:
                for key in querydict:
                    if row[key] != querydict[key]:
                        raise ContinueOutOfALoop
                return row["cloudPlatformId"]
            except ContinueOutOfALoop:
                continue
        return None

    def map_to_fg(self):
        rows = self.rows
        records = []

        whereclause = {"platform": self.platform, "hostname":
                       self.hostname, "version": self.platform_version}
        cloudplatformid = self.get_cloudplatform_id(whereclause)

            # 1. draw mapping table between openstack 'instances' and fg 'instance' table.
            # 2. define each column to know what exactly it means
            # 3. leave comments for missing and skipped columns
            # 4. check search options to see it is validate

        for row in rows:
            record = row

            try:
                record["instanceId"] = record["instanceId"][:15]
                record["ts"] = record["t_start"]
                # record["calltype"] = ""
                # record["userData"] = ""
                # record["kernelId"] = ""
                # record["emiURL"] = ""
                # record["t_start"] = row["t_start"]
                # record["t_end"] = row["t_end"]
                if record["t_end"] and record["t_start"]:
                    record["duration"] = (record[
                                          "t_end"] - record["t_start"]).total_seconds()
                # record["trace"] = {
                #    "pending" : { "start" : self.future, "stop" : self.past, "queue" : deque("",10)},
                #    "extant" : { "start" : self.future, "stop" : self.past, "queue" : deque("",10)},
                #    "teardown" : { "start" : self.future, "stop" : self.past, "queue" : deque("",10)}
                #    }
                # record["serviceTag"] = row["serviceTag"] or ""
                # record["groupNames"] = ""
                # record["keyName"] = ""
                # record["msgtype"] = ""
                # record["volumesSize"] = 0.0
                # record["linetype"] = ""
                if "dn" in record and not "ownerId" in record:
                    if len(record["dn"].split("CN=")) > 1:
                        record["ownerId"] = record["dn"].split("CN=")[1]
                    else:
                        record["ownerId"] = record["dn"]
                try:
                    del record["dn"]
                except:
                    pass
                record["date"] = record["t_start"]
                # record["id"] = 0
                # record["ncHostIdx"] = 0
                # record["ccvm"] = { "mem" : record["ccvm_mem"], "cores" : record["ccvm_cores"], "disk" : record[0 }
                # if "ccvm_disk" in row:
                #    record["ccvm"]["disk"] = row["ccvm_disk"]
                # if "emiId" in row:
                #    record["emiId"] = row["emiId"]
                # else:
                #    record["emiId"] = ""
                # record["ccnet"] = { "publicIp" : "", "privateMac" : "",
                # "networkIndex" : "", "vlan" : "", "privateIp" : "" }

                # record["ramdiskURL"] = ""
                # record["accountId"] = ""
                # record["kernelURL"] = ""
                # record["ramdiskId"] = ""
                # record["volumes"] = ""
                # record["launchIndex"] = 0
                # record["bundleTaskStateName"] = ""
                # record["reservationId"] = ""
                record["platform"] = self.platform
                # record["euca_hostname"] = self.hostname
                # record["euca_version"] = self.platform_version
                # record["state"] = "Teardown" # need to be changed
                if not "state" in record:
                    record["state"] = "Teardown"
                record["state"] = self.convert_state(record["state"])
                record["cloudPlatformIdRef"] = cloudplatformid
                records.append(record)
            except:
                print sys.exc_info()
                print record
                pass

        self.records = records

    def convert_state(self, state):
        if self.platform == "openstack":
            if state == "active":
                return "Extant"
            elif state == "building":
                return "Pending"
            elif state == "deleted" or "shutoff":
                return "Teardown"
            elif state == "error":
                return state
        return state

    def set_instance_conf(self, confname=""):
        if confname and len(confname) > 0:
            self.db_dest.set_conf(confname)
            self.db_dest.update_conf()

    def set_parser(self):
        def_s_date = "19700101"
        def_e_date = "29991231"
        def_conf = "futuregrid.cfg"
        def_nova = "nova"
        def_keystone = "keystone"
        def_db = "mysql"

        parser = argparse.ArgumentParser()
        parser.add_argument("-s", "--from", dest="s_date", default=def_s_date,
                            help="Start date to begin parsing (type: YYYYMMDD)")
        parser.add_argument("-e", "--to", dest="e_date", default=def_e_date,
                            help="End date to finish parsing (type: YYYYMMDD)")

        parser.add_argument("-p", "--platform", required=True,
                            help="Cloud platform name, required. (e.g. nimbus, openstack, eucalyptus, etc)")
        parser.add_argument("-pv", "--platform_version",
                            help="Cloud platform version. (e.g. 2.9 for nimbus, essex for openstack, and  2.0 or 3.1 for eucalyptus)")
        parser.add_argument("-n", "--hostname", required=True,
                            help="Hostname of the cloud platform, required. (e.g., hotel, sierra, india, alamo, foxtrot)")
        parser.add_argument("--conf", dest="conf",
                            help="futuregrid.cfg filepath (e.g. $HOME/.futuregrid/futuregrid.cfg)")

        parser.add_argument("-db", "--database", default=def_db,
                            help="database type to load (e.g. mysql or sqlite3)")

        # sqlite3 for nimbus
        parser.add_argument("-i", "--file", dest="input_file",
                            help="the sqlite3 filename with path (e.g. /home/metric/nimbus/alamo/alamo)")

        # mysql for openstack
        parser.add_argument("-dbn", "--dbname_nova", default=def_nova,
                            help="Database of nova to use")
        parser.add_argument("-dbk", "--dbname_keystone", default=def_keystone,
                            help="Database of keystone to use")
        parser.add_argument("-dh", "--dbhost",
                            help="Connect to database host")
        parser.add_argument("-du", "--dbuser",
                            help="User for login of database")
        parser.add_argument("-dp", "--dbpass",
                            help="Password to use when connecting to database server")
        parser.add_argument("-dP", "--dbport", default=3306,
                            help="Port number to use for connection or 3306 for default")

        args = parser.parse_args()
        print args

        try:

            self.s_date = datetime.strptime(args.s_date, "%Y%m%d")
            self.e_date = datetime.strptime(args.e_date, "%Y%m%d")
            self.platform = args.platform
            self.platform_version = args.platform_version
            self.hostname = args.hostname
            self.confname = self.set_instance_conf(args.conf)

            self.db.db_type = args.database
            self.dbname_nova = args.dbname_nova
            self.dbname_keystone = args.dbname_keystone
            self.db.dbhost = args.dbhost
            self.db.dbuser = args.dbuser
            self.db.dbpasswd = args.dbpass
            self.db.dbport = args.dbport

            self.db.set_sqlite3_file(args.input_file)

        except:
            pass  # print sys.exc_info()[0]

        self.argparser = parser