class ListVMs(View): def __init__(self): self.db = FGDatabase() self.db.conf() self.db.connect() self.cloudservice = None self.data = None @mimerender def dispatch_request(self): self.read_cloud_service() self.read_vms() res = self.data return {"message": res} def read_cloud_service(self): res = self.db.read_cloudplatform() new_res = {} for cloud in res: new_res[cloud['cloudPlatformId']] = cloud self.cloudservice = new_res def read_vms(self): cursor = self.db.cursor table = self.db.instance_table table2 = self.db.cloudplatform_table query = "select %(table2)s.platform as CLOUDNAME, DATE_FORMAT(date, '%%Y %%b') as 'MONTH YEAR', count(*) as VALUE from %(table)s, %(table2)s group by %(table2)s.platform, YEAR(date), MONTH(date)" % vars() try: cursor.execute(query) self.data = cursor.fetchall() except: print sys.exc_info() def map_cloudname(self): for record in self.data: try: cloudname = self.cloudservice[record['cloudplatformidref']] except: print record
class FGConverter: # future = instances.in_the_future # past = instances.in_the_past s_date = None e_date = None platform = None platform_version = None hostname = None confname = None dbname_nova = None dbname_keystone = None query = None rows = None # from database records = None # for fg database userinfo = None cloudplatform = None argparser = None def __init__(self): self.db = FGDatabase() self.db_dest = FGDatabase() def __del__(self): self.db_close() def convert_to_fg(self): self.check_platform() self.db_connect() self.convert_instance() self.convert_userinfo() self.db_close() def check_platform(self): _check = getattr(self, "_check_platform_" + self.platform) _check() def _check_platform_nimbus(self): self.platform_version = self.platform_version or FGConst.DEFAULT_NIMBUS_VERSION self.db.db_type = self.db.db_type or FGConst.DEFAULT_NIMBUS_DB # this query is for sqlite3 because [timestamp] is only used on # sqlite3? self.query = 'SELECT t1.time as "t_start [timestamp]",\ t3.time as "t_end [timestamp]",\ t1.uuid as instanceId,\ t2.dn,\ t1.cpu_count as ccvm_cores,\ t1.memory as ccvm_mem,\ t1.vmm as serviceTag \ from create_events t1, user t2, remove_events t3 \ on t1.user_id=t2.id and t1.uuid=t3.uuid \ where t1.time >= \'' + str(self.s_date) + '\' and t3.time <= \'' + str(self.e_date) + '\'' def _check_platform_openstack(self): if not self.dbname_nova or not self.dbname_keystone or not self.db.dbhost or not self.db.dbuser or not self.db.dbpasswd: msg = "db info is missing" print msg raise ValueError(msg) self.platform_version = self.platform_version or FGConst.DEFAULT_OPENSTACK_VERSION self.db.db_type = self.db.db_type or FGConst.DEFAULT_OPENSTACK_DB self.db.dbname = self.dbname_nova self.query = 'SELECT created_at as trace_pending_start, \ launched_at as trace_extant_start,\ terminated_at as trace_teardown_start, \ deleted_at as trace_teardown_stop, \ id,\ user_id as ownerId,\ project_id as accountId,\ image_ref as emiId,\ kernel_id as kernelId,\ ramdisk_id as ramdiskId,\ key_data as keyName,\ vm_state as state,\ memory_mb as ccvm_mem, \ vcpus as ccvm_cores, \ host as serviceTag, \ reservation_id as reservationId,\ created_at as t_start, \ COALESCE(deleted_at, terminated_at, updated_at) as t_end, \ uuid as instanceId, \ access_ip_v4 as ccnet_publicIp,\ ephemeral_gb as ccvm_disk \ from instances \ where updated_at >= \'' + str(self.s_date) + '\' and updated_at <= \'' + str(self.e_date) + '\'' # COALESCE(launched_at, created_at, scheduled_at) as t_start, \ def db_connect(self): self.db.connect() self.db_dest.conf() self.db_dest.connect() def db_close(self): self.db.close() self.db_dest.close() def read_from_source(self): self.rows = self.db.query(self.query) def write_to_dest(self): self.db_dest.write_instance(self.records) def convert_instance(self): self.read_from_source() self.map_to_fg() self.write_to_dest() def convert_userinfo(self): _convert = getattr(self, "_convert_userinfo_of_" + self.platform) _convert() def _convert_userinfo_of_nimbus(self): return def _convert_userinfo_of_openstack(self): res = self._read_userinfo_of_nova_with_project_info() self.db_dest.write_userinfo(res) def _read_userinfo_of_nova_with_project_info(self): keystone = self.db keystone.dbname = self.dbname_keystone keystone.connect() # This is for Grizzly userinfo = keystone.query("select user_id, tenant_id, user.name as \ user_name, tenant.name as tenant_name \ from user_tenant_membership, tenant, user \ where user.id=user_tenant_membership.user_id \ and tenant.id=user_tenant_membership.tenant_id") # select id, name from user") if not userinfo: #This is for Folsom userinfo = keystone.query("select user_id, project_id as tenant_id,\ user.name as user_name, project.name as \ tenant_name from user_project_metadata, \ project, user where \ user.id=user_project_metadata.user_id and \ project.id=user_project_metadata.project_id") keystone.close() records = [] for row in userinfo: try: res = FGUtility.retrieve_userinfo_ldap(row["user_name"]) if not res: res = {} res["ownerid"] = row["user_id"] res["username"] = row["user_name"] res["project"] = row["tenant_name"] res["hostname"] = self.hostname print res records.append(res) except: print sys.exc_info() raise return records def read_cloudplatform(self): if self.cloudplatform: return self.cloudplatform = self.db_dest.read_cloudplatform() def get_cloudplatform_id(self, querydict={}): class ContinueOutOfALoop(Exception): pass self.read_cloudplatform() for row in self.cloudplatform: try: for key in querydict: if row[key] != querydict[key]: raise ContinueOutOfALoop return row["cloudPlatformId"] except ContinueOutOfALoop: continue return None def map_to_fg(self): rows = self.rows records = [] whereclause = {"platform": self.platform, "hostname": self.hostname, "version": self.platform_version} cloudplatformid = self.get_cloudplatform_id(whereclause) # 1. draw mapping table between openstack 'instances' and fg 'instance' table. # 2. define each column to know what exactly it means # 3. leave comments for missing and skipped columns # 4. check search options to see it is validate for row in rows: record = row try: record["instanceId"] = record["instanceId"][:15] record["ts"] = record["t_start"] # record["calltype"] = "" # record["userData"] = "" # record["kernelId"] = "" # record["emiURL"] = "" # record["t_start"] = row["t_start"] # record["t_end"] = row["t_end"] if record["t_end"] and record["t_start"]: record["duration"] = (record[ "t_end"] - record["t_start"]).total_seconds() # record["trace"] = { # "pending" : { "start" : self.future, "stop" : self.past, "queue" : deque("",10)}, # "extant" : { "start" : self.future, "stop" : self.past, "queue" : deque("",10)}, # "teardown" : { "start" : self.future, "stop" : self.past, "queue" : deque("",10)} # } # record["serviceTag"] = row["serviceTag"] or "" # record["groupNames"] = "" # record["keyName"] = "" # record["msgtype"] = "" # record["volumesSize"] = 0.0 # record["linetype"] = "" if "dn" in record and not "ownerId" in record: if len(record["dn"].split("CN=")) > 1: record["ownerId"] = record["dn"].split("CN=")[1] else: record["ownerId"] = record["dn"] try: del record["dn"] except: pass record["date"] = record["t_start"] # record["id"] = 0 # record["ncHostIdx"] = 0 # record["ccvm"] = { "mem" : record["ccvm_mem"], "cores" : record["ccvm_cores"], "disk" : record[0 } # if "ccvm_disk" in row: # record["ccvm"]["disk"] = row["ccvm_disk"] # if "emiId" in row: # record["emiId"] = row["emiId"] # else: # record["emiId"] = "" # record["ccnet"] = { "publicIp" : "", "privateMac" : "", # "networkIndex" : "", "vlan" : "", "privateIp" : "" } # record["ramdiskURL"] = "" # record["accountId"] = "" # record["kernelURL"] = "" # record["ramdiskId"] = "" # record["volumes"] = "" # record["launchIndex"] = 0 # record["bundleTaskStateName"] = "" # record["reservationId"] = "" record["platform"] = self.platform # record["euca_hostname"] = self.hostname # record["euca_version"] = self.platform_version # record["state"] = "Teardown" # need to be changed if not "state" in record: record["state"] = "Teardown" record["state"] = self.convert_state(record["state"]) record["cloudPlatformIdRef"] = cloudplatformid records.append(record) except: print sys.exc_info() print record pass self.records = records def convert_state(self, state): if self.platform == "openstack": if state == "active": return "Extant" elif state == "building": return "Pending" elif state == "deleted" or "shutoff": return "Teardown" elif state == "error": return state return state def set_instance_conf(self, confname=""): if confname and len(confname) > 0: self.db_dest.set_conf(confname) self.db_dest.update_conf() def set_parser(self): def_s_date = "19700101" def_e_date = "29991231" def_conf = "futuregrid.cfg" def_nova = "nova" def_keystone = "keystone" def_db = "mysql" parser = argparse.ArgumentParser() parser.add_argument("-s", "--from", dest="s_date", default=def_s_date, help="Start date to begin parsing (type: YYYYMMDD)") parser.add_argument("-e", "--to", dest="e_date", default=def_e_date, help="End date to finish parsing (type: YYYYMMDD)") parser.add_argument("-p", "--platform", required=True, help="Cloud platform name, required. (e.g. nimbus, openstack, eucalyptus, etc)") parser.add_argument("-pv", "--platform_version", help="Cloud platform version. (e.g. 2.9 for nimbus, essex for openstack, and 2.0 or 3.1 for eucalyptus)") parser.add_argument("-n", "--hostname", required=True, help="Hostname of the cloud platform, required. (e.g., hotel, sierra, india, alamo, foxtrot)") parser.add_argument("--conf", dest="conf", help="futuregrid.cfg filepath (e.g. $HOME/.futuregrid/futuregrid.cfg)") parser.add_argument("-db", "--database", default=def_db, help="database type to load (e.g. mysql or sqlite3)") # sqlite3 for nimbus parser.add_argument("-i", "--file", dest="input_file", help="the sqlite3 filename with path (e.g. /home/metric/nimbus/alamo/alamo)") # mysql for openstack parser.add_argument("-dbn", "--dbname_nova", default=def_nova, help="Database of nova to use") parser.add_argument("-dbk", "--dbname_keystone", default=def_keystone, help="Database of keystone to use") parser.add_argument("-dh", "--dbhost", help="Connect to database host") parser.add_argument("-du", "--dbuser", help="User for login of database") parser.add_argument("-dp", "--dbpass", help="Password to use when connecting to database server") parser.add_argument("-dP", "--dbport", default=3306, help="Port number to use for connection or 3306 for default") args = parser.parse_args() print args try: self.s_date = datetime.strptime(args.s_date, "%Y%m%d") self.e_date = datetime.strptime(args.e_date, "%Y%m%d") self.platform = args.platform self.platform_version = args.platform_version self.hostname = args.hostname self.confname = self.set_instance_conf(args.conf) self.db.db_type = args.database self.dbname_nova = args.dbname_nova self.dbname_keystone = args.dbname_keystone self.db.dbhost = args.dbhost self.db.dbuser = args.dbuser self.db.dbpasswd = args.dbpass self.db.dbport = args.dbport self.db.set_sqlite3_file(args.input_file) except: pass # print sys.exc_info()[0] self.argparser = parser