def instances_list(p_isJSON,
p_isVERBOSE,
p_region,
p_instance,
p_instance_type=None,
p_cloud=None,
p_resource=None):
try:
filter_params = {}
filter_params['region'] = p_region
filter_params['instance'] = p_instance
filter_params['resource'] = p_resource
from cloud import CloudConnection
cloud = CloudConnection(cloud=p_cloud)
client = cloud.get_client(instance_type=p_instance_type)
client.isJson = p_isJSON
client.verbose = p_isVERBOSE
instances_list = client.get_list(filter_params)
if p_isJSON:
json_dict = {}
json_dict['data'] = instances_list
json_dict['state'] = 'completed'
print(json.dumps([json_dict]))
elif not p_isVERBOSE:
keys = client.header_keys
headers = client.header_titles
print("")
print(api.format_data_to_table(instances_list, keys, headers))
except Exception as e:
util.exit_message(str(e), 1, p_isJSON)
return (0)
def list_apt_packages(p_repo, p_isJSON):
cli_ver = get_cli_ver(p_repo)
raw_list = util.read_file_string(get_apt_repo_file_name())
repoList = []
repoDict = {}
for line in raw_list.splitlines():
data = line.split()
if len(data) != 2:
continue
if data[0] == "Package:":
repoDict['component'] = data[1]
if data[0] == "Version:":
version = data[1]
if data[0] == "Filename:":
repoDict['filename'] = data[1]
p1 = version.find(".pgdg" + cli_ver)
if p1 > 0:
repoDict['version'] = version[0:p1]
repoList.append(repoDict)
repoDict = {}
keys = ['component', 'version']
headers = ['Component', 'Version']
if p_isJSON:
print(json.dumps(repoList, sort_keys=True, indent=2))
else:
print(api.format_data_to_table(repoList, keys, headers))
return (0)
def list(p_isJSON):
repo_dict = get_json_file(PGDG_REPO_LIST, p_isJSON)
os = util.get_os()
kount = 0
lList = []
for rl in repo_dict:
lDict = {}
if os == rl['os']:
kount = kount + 1
lDict['repo'] = rl['repo']
if is_installed(rl['repo']):
lDict['status'] = "Installed"
else:
lDict['status'] = ""
lList.append(lDict)
if kount == 0:
msg = "No repo's available for os = " + os
util.exit_message(msg, 1, p_isJSON)
keys = ['repo', 'status']
headers = ['Repo', 'Status']
if p_isJSON:
print(json.dumps(lList, sort_keys=True, indent=2))
else:
print(api.format_data_to_table(lList, keys, headers))
return 0
def list_defaults(p_isJSON):
try:
c = con.cursor()
sql = "SELECT user_group, user_name, section, d_key, d_value \n" + \
" FROM defaults \n" + \
"ORDER BY 1, 2, 3, 4"
c.execute(sql)
dataset = c.fetchall()
dict = []
for data in dataset:
d = {}
d['group'] = str(data[0])
d['user'] = str(data[1])
d['section'] = str(data[2])
d['key'] = str(data[3])
d['value'] = str(data[4])
dict.append(d)
except Exception as e:
fatal_sql_error(e, sql, "meta.list_defaults()")
if p_isJSON:
print(json.dumps(dict, sort_keys=True, indent=2))
else:
print("")
print(api.format_data_to_table(dict,
["group", "user", "section", "key", "value"],
["Group", "User", "Section", "Key", "Value"]))
return(0)
def cloud_metalist(p_isJSON,
p_isVERBOSE,
p_type,
p_region,
p_version=None,
cloud="aws",
instance_type="db",
group=""):
from cloud import CloudConnection
cloud = CloudConnection(cloud=cloud)
client = cloud.get_client(instance_type=instance_type, region=p_region)
if p_type == "instance-class":
col_names = [
"DBInstanceClass", "v_cpu", "memory", "iops", "network_performance"
]
col_titles = [
"DBInstanceClass", "vCPU", "Memory", "IOPS Optimized",
"Network Performance"
]
db_instance_meta_list = meta_list("", "", "aws-rds", "", True)
result = client.instance_class(
p_version, db_instance_meta_list=db_instance_meta_list)
elif p_type == "rds-versions":
col_names = ["DBEngineVersionDescription", "EngineVersion"]
col_titles = ["DBEngineVersionDescription", "EngineVersion"]
result = client.rds_versions()
elif p_type == "res-group":
col_names = ["name", "region"]
col_titles = ["ResourceGroup", "Region"]
result = client.get_resource_group()
elif p_type == "instance-type":
col_names = [
"name", "number_of_cores", "resource_disk_size_in_mb",
"memory_in_mb", "max_data_disk_count", "os_disk_size_in_mb"
]
col_titles = [
"Name", "number_of_cores", "resource_disk_size_in_mb",
"memory_in_mb", "max_data_disk_count", "os_disk_size_in_mb"
]
result = client.get_instance_types(p_region)
elif p_type == "storage-accounts":
col_names = ["name", "location"]
col_titles = ["Name", "location"]
result = client.storage_accounts(group)
else:
col_names = ["vpc", "subnet_group"]
col_titles = ["VPC", "Subnet Group"]
result = client.subnet_groups(res_group=group)
if p_isJSON:
print(json.dumps(result, indent=2))
else:
print(api.format_data_to_table(result, col_names, col_titles))
return (0)
def list(p_isJSON):
l_list = meta.get_lab_list()
keys = ['lab', 'disp_name', 'enabled', 'credit_to']
headers = ['Lab', 'Name', 'Enabled?', 'Credits']
if p_isJSON:
print(json.dumps(l_list, sort_keys=True, indent=2))
else:
print(api.format_data_to_table(l_list, keys, headers))
return 0
def meta_list(p_isJSON, p_isVERBOSE, p_meta, p_instance, return_dict=False):
repo = util.get_value("GLOBAL", "REPO")
url = repo + "/" + p_meta + ".txt"
try:
response = urllib2.urlopen(url, timeout=15)
meta_string = response.read()
except Exception as e:
util.exit_message("Cannot retrieve METALIST '" + url + "'", 1,
p_isJSON)
dict = []
kount = 0
for line in meta_string.splitlines():
kount = kount + 1
word = line.split("\t")
for i in range(len(word)):
word[i] = word[i].strip()
## process header rows
if kount < 3:
if kount == 1:
## row 1 contains column names
num_cols = len(word)
col_names = word
else:
## row 2 contains column titles
col_titles = word
continue
## process valid detail rows
if len(word) == num_cols:
if p_instance:
if word[0] != p_instance:
continue
d = {}
for i in range(len(word)):
d[col_names[i]] = word[i]
dict.append(d)
if return_dict:
return dict
if p_isJSON:
print(json.dumps(dict, sort_keys=True, indent=2))
else:
print("")
print(api.format_data_to_table(dict, col_names, col_titles))
return (0)
def pglist(p_isJSON, p_isVERBOSE, p_region="", p_email=""):
try:
c = con.cursor()
sql = "SELECT u.email, g.name as server_group, s.name as server_name, \n" + \
" s.host, s.port, s.maintenance_db as db, s.username as db_user, \n" + \
" s.id as sid, g.id as gid, s.password as pwd \n" + \
" FROM server s, user u, servergroup g \n" + \
" WHERE s.user_id = u.id AND s.servergroup_id = g.id \n" + \
" AND u.email LIKE ? \n" + \
"ORDER BY 1, 2, 3"
c.execute(sql, [p_email])
svr_list = c.fetchall()
svrs = []
for row in svr_list:
svr_dict = {}
svr_dict['email'] = str(row[0])
svr_dict['server_group'] = str(row[1])
svr_dict['server_name'] = str(row[2])
svr_dict['host'] = str(row[3])
svr_dict['port'] = str(row[4])
svr_dict['db'] = str(row[5])
svr_dict['db_user'] = str(row[6])
svr_dict['sid'] = str(row[7])
svr_dict['gid'] = str(row[8])
has_pwd = False
if row[9]:
has_pwd = True
svr_dict['has_pwd'] = has_pwd
svrs.append(svr_dict)
except Exception as error:
msg = "pgDevOps must be installed & initialized."
util.exit_message(msg, 1, p_isJSON)
keys = [
'email', 'server_group', 'server_name', 'host', 'port', 'db', 'db_user'
]
headers = [
'Email Address', 'Server Group', 'Server Name', 'Host', 'Port', 'DB',
'DB User'
]
if p_isJSON:
print(json.dumps(svrs, sort_keys=True, indent=2))
else:
print("")
print(api.format_data_to_table(svrs, keys, headers))
return (0)
def list_packages(p_repo, p_SHOWDUPS, p_isJSON, p_isEXTRA):
if not is_repo(p_repo, p_isJSON):
util.exit_message(p_repo + " is not a valid REPO.", 1, p_isJSON)
[repo_type, name, url, package, key,
pkg_filter] = get_repo(p_repo, p_isJSON)
if not is_installed(p_repo):
util.exit_message(p_repo + " is not registered.", 1, p_isJSON)
options = ""
if p_SHOWDUPS:
options = "--showduplicates"
if util.get_os() in APT_LIST:
return list_apt_packages(p_repo, p_isJSON)
os = util.get_os()
if os == "el6":
cmd = "yum list all | grep " + p_repo
else:
cmd = "yum repo-pkgs " + p_repo + " list " + options
cmd = cmd + " | awk '"
## filter package list unless asked to show --extra or --test
kount = 0
if not p_isEXTRA:
for p in pkg_filter:
kount = kount + 1
ps = "/" + p.replace('.', '\.') + "/"
if kount > 1:
cmd = cmd + " || " + ps
else:
cmd = cmd + ps
cmd = "sudo " + cmd + " { print }' | awk '!/debug/ && !/docs/ { print }'"
outp = util.getoutput(cmd)
my_logger.info("\n$ " + cmd + "\n\n" + str(outp))
repoList = []
for line in outp.splitlines():
data = line.split()
if len(data) != 3:
continue
repoDict = {}
p1 = data[0].find('.')
pkg_nm = data[0][0:p1]
p2 = data[1].find('.rhel')
if p2 > 0:
pkg_ver = data[1][0:p2]
else:
pkg_ver = data[1]
status = ""
if data[2].startswith("@"):
status = "Installed"
repoDict['component'] = pkg_nm
repoDict['version'] = pkg_ver
repoDict['status'] = status
if pkg_nm > "":
repoList.append(repoDict)
keys = ['component', 'version', 'status']
headers = ['Component', 'Version', 'Status']
if p_isJSON:
print(json.dumps(repoList, sort_keys=True, indent=2))
else:
print(api.format_data_to_table(repoList, keys, headers))
return (0)
def get_list(p_isOLD,
p_isExtensions,
p_isJSON,
p_isTEST,
p_showLATEST,
p_comp=None,
p_relnotes=None,
p_return=False):
# r_sup_plat = util.like_pf("r.sup_plat")
r_sup_plat = "1 = 1"
if p_isOLD:
exclude_comp = ""
else:
exclude_comp = " AND v.component NOT IN (SELECT component FROM components)"
parent_comp_condition = ""
installed_category_conditions = " AND p.category > 0 "
available_category_conditions = " AND p.is_extension = 0"
ext_component = ""
if p_isExtensions:
installed_category_conditions = " AND p.is_extension = 1"
available_category_conditions = " AND p.is_extension = 1"
if p_comp != "all":
ext_component = " AND parent = '" + p_comp + "' "
installed = \
"SELECT p.category, g.description as category_desc, g.short_desc as short_cat_desc, \n" + \
" c.component, c.version, c.port, c.status, r.stage, \n" + \
" coalesce((select is_current from versions where c.component = component AND c.version = version),0), \n" + \
" c.datadir, p.is_extension, \n" + \
" coalesce((select parent from versions where c.component = component and c.version = version),'') as parent, \n" + \
" coalesce((select release_date from versions where c.component = component and c.version = version),'20200101'), \n" + \
" c.install_dt, r.disp_name, \n" + \
" coalesce((select release_date from versions where c.component = component and is_current = 1),'20200101') \n" + \
" FROM components c, releases r, projects p, categories g \n" + \
" WHERE c.component = r.component AND r.project = p.project \n" + \
" AND p.category = g.category \n" + \
" AND " + r_sup_plat + installed_category_conditions + ext_component
available = \
"SELECT c.category, c.description, c.short_desc as short_cat_desc, v.component, v.version, 0, 'NotInstalled', \n" + \
" r.stage, v.is_current, '', p.is_extension, v.parent as parent, v.release_date, '', \n" + \
" r.disp_name, \n" + \
" coalesce((select release_date from versions where v.component = component and is_current = 1),'20200101') \n" + \
" FROM versions v, releases r, projects p, categories c \n" + \
" WHERE v.component = r.component AND r.project = p.project \n" + \
" AND p.category = c.category \n" + \
" AND " + util.like_pf("v.platform") + " \n" + \
" AND " + r_sup_plat + exclude_comp + available_category_conditions + ext_component
extensions = \
"SELECT c.category, c.description, c.short_desc as short_cat_desc, v.component, v.version, 0, 'NotInstalled', \n" + \
" r.stage, v.is_current, '', p.is_extension, v.parent as parent, v.release_date, '', \n" + \
" r.disp_name, \n" + \
" coalesce((select release_date from versions where v.component = component and is_current = 1),'20200101') \n" + \
" FROM versions v, releases r, projects p, categories c \n" + \
" WHERE v.component = r.component AND r.project = p.project \n" + \
" AND p.is_extension = 1 AND p.category = c.category \n" + \
" AND " + util.like_pf("v.platform") + " \n" + \
" AND v.parent in (select component from components) AND " + r_sup_plat + exclude_comp
if p_isExtensions:
sql = installed + "\n UNION \n" + available + "\n ORDER BY 1, 3, 4, 6"
else:
sql = installed + "\n UNION \n" + available + "\n UNION \n" + extensions + "\n ORDER BY 1, 3, 4, 6"
try:
c = con.cursor()
c.execute(sql)
data = c.fetchall()
headers = [
'Category', 'Component', 'Version', 'ReleaseDt', 'Stage', 'Status',
'Updates'
]
keys = [
'short_cat_desc', 'component', 'version', 'release_date', 'stage',
'status', 'current_version'
]
jsonList = []
kount = 0
previous_version = None
previous_comp = None
for row in data:
compDict = {}
kount = kount + 1
category = str(row[0])
category_desc = str(row[1])
short_cat_desc = str(row[2])
comp = str(row[3])
version = str(row[4])
port = str(row[5])
if previous_comp and previous_version:
if previous_comp == comp and previous_version == version:
continue
previous_version = version
previous_comp = comp
if str(row[6]) == "Enabled":
status = "Installed"
else:
status = str(row[6])
if status == "NotInstalled" and p_isJSON == False:
status = ""
stage = str(row[7])
if stage in ("soon", "bring-own", "included"):
continue
is_current = str(row[8])
if is_current == "0" and status in ("", "NotInstalled"):
if not p_isOLD:
continue
current_version = get_current_version(comp)
is_update_available = 0
cv = Version.coerce(current_version)
iv = Version.coerce(version)
if cv > iv:
is_update_available = 1
if is_update_available == 0:
updates = 0
current_version = ""
else:
updates = 1
if (port == "0") or (port == "1"):
port = ""
datadir = row[9]
if row[9] is None:
datadir = ""
else:
datadir = str(row[9]).strip()
is_extension = row[10]
parent = row[11]
disp_name = row[14]
release_desc = ''
release_date = '1970-01-01'
curr_rel_date = '1970-01-01'
curr_rel_dt = str(row[15])
rel_dt = str(row[12])
if len(rel_dt) == 8:
release_date = rel_dt[0:4] + "-" + rel_dt[4:6] + "-" + rel_dt[
6:8]
curr_rel_date = curr_rel_dt[0:4] + "-" + curr_rel_dt[
4:6] + "-" + curr_rel_dt[6:8]
compDict['is_new'] = 0
try:
rd = datetime.datetime.strptime(release_date, '%Y-%m-%d')
today_date = datetime.datetime.today()
date_diff = (today_date - rd).days
if date_diff <= 30:
compDict['is_new'] = 1
if p_showLATEST and date_diff > 30:
continue
except Exception as e:
pass
if util.is_postgres(comp):
if port > "" and status == "Installed" and datadir == "":
status = "NotInitialized"
port = ""
ins_date = str(row[13])
install_date = ""
compDict['is_updated'] = 0
if ins_date:
install_date = ins_date[0:4] + "-" + ins_date[
5:7] + "-" + ins_date[8:10]
try:
insDate = datetime.datetime.strptime(
install_date, '%Y-%m-%d')
today_date = datetime.datetime.today()
date_diff = (today_date - insDate).days
if date_diff <= 30:
compDict['is_updated'] = 1
except Exception as e:
pass
if p_relnotes and p_isJSON:
rel_version = version
if current_version != "":
rel_version = current_version
rel_notes = str(util.get_relnotes(comp, rel_version))
markdown_text = unicode(rel_notes,
sys.getdefaultencoding(),
errors='ignore').strip()
html_text = mistune.markdown(markdown_text)
compDict['rel_notes'] = html_text
compDict['category'] = category
compDict['category_desc'] = category_desc
compDict['short_cat_desc'] = short_cat_desc
compDict['component'] = comp
compDict['version'] = version
compDict['is_extension'] = is_extension
compDict['disp_name'] = disp_name
compDict['release_desc'] = release_desc
compDict['port'] = port
compDict['release_date'] = release_date
compDict['install_date'] = install_date
compDict['curr_release_date'] = curr_rel_date
compDict['status'] = status
compDict['stage'] = stage
compDict['updates'] = updates
compDict['is_current'] = is_current
compDict['current_version'] = current_version
compDict['parent'] = parent
jsonList.append(compDict)
if p_return:
return jsonList
if p_isJSON:
print(json.dumps(jsonList, sort_keys=True, indent=2))
else:
if len(jsonList) >= 1:
if p_showLATEST:
print("New components released in the last 30 days.")
print(api.format_data_to_table(jsonList, keys, headers))
except Exception as e:
fatal_error(e, sql, "meta.get_list()")
sys.exit(0)
def rdslist(p_isJSON,
p_isVERBOSE,
p_region="",
p_instance="",
p_email="",
p_engines=["postgres"]):
if p_region is None:
p_region = ""
if p_instance is None:
p_instance = ""
try:
rds_regions = []
available_rds_regions = boto3.session.Session().get_available_regions(
"rds")
if p_region > "":
if p_region in available_rds_regions:
rds_regions = [p_region]
else:
msg = str(p_region) + " is not a valid region for rds."
util.exit_message(msg, 1, p_isJSON)
else:
rds_regions = available_rds_regions
# get all of the postgres db instances
pg_list = []
for region in rds_regions:
msg = "Searching " + region + "..."
util.message(msg, "info", p_isJSON)
rds = boto3.client('rds', region_name=region)
dbs = rds.describe_db_instances()
for db in dbs['DBInstances']:
if db['Engine'] in p_engines:
ec2 = boto3.client('ec2', region_name=region)
pg_dict = {}
pg_dict['engine'] = get_field(db, 'Engine')
pg_dict['region'] = region
pg_dict['instance'] = get_field(db, 'DBInstanceIdentifier')
if p_instance > "":
if p_instance != pg_dict['instance']:
continue
pg_dict['arn'] = get_field(db, 'DBInstanceArn')
pg_dict['master_user'] = get_field(db, 'MasterUsername')
pg_dict['status'] = get_field(db, 'DBInstanceStatus')
pg_dict['address'] = get_field(db, 'Endpoint', 'Address')
pg_dict['port'] = get_field(db, 'Endpoint', 'Port')
pg_dict['dbname'] = get_field(db, 'DBName')
pg_dict['db_class'] = get_field(db, 'DBInstanceClass')
pg_dict['engine_version'] = get_field(db, 'EngineVersion')
pg_dict['auto_minor_upgrade'] = get_field(
db, 'AutoMinorVersionUpgrade')
try:
pg_dict['create_time'] = convert_tz(
get_field(db, 'InstanceCreateTime'))
except Exception as e:
pg_dict['create_time'] = ""
pg_dict['iops'] = get_field(db, 'Iops')
pg_dict['storage_allocated'] = get_field(
db, 'AllocatedStorage')
storage_type = get_field(db, 'StorageType')
if storage_type == 'standard':
pg_dict['storage_type'] = storage_type + ": Magnetic"
elif storage_type == 'gp2':
pg_dict[
'storage_type'] = storage_type + ": General Purpose (SSD)"
elif storage_type == 'io2':
pg_dict[
'storage_type'] = storage_type + ": Provisioned IOPS (SSD)"
else:
pg_dict['storage_type'] = storage_type
pg_dict['storage_encrypted'] = get_field(
db, 'StorageEncrypted')
pg_dict['maint_window'] = get_field(
db, 'PreferredMaintenanceWindow')
pg_dict['backup_window'] = get_field(
db, 'PreferredBackupWindow')
pg_dict['backup_retention'] = get_field(
db, 'BackupRetentionPeriod')
try:
pg_dict['latest_restorable'] = convert_tz(
get_field(db, 'LatestRestorableTime'))
except Exception as e:
pg_dict['latest_restorable'] = ""
pg_dict['az_is_multi'] = get_field(db, 'MultiAZ')
pg_dict['az_primary'] = get_field(db, 'AvailabilityZone')
pg_dict['az_secondary'] = get_field(
db, 'SecondaryAvailabilityZone')
pg_dict['publicly_accessible'] = get_field(
db, 'PubliclyAccessible')
pg_dict['monitoring_interval'] = get_field(
db, 'MonitoringInterval')
pg_dict['monitoring_resource_arn'] = get_field(
db, 'EnhancedMonitoringResourceArn')
pg_dict['monitoring_role_arn'] = get_field(
db, 'MonitoringRoleArn')
pg_dict['subnet_group'] = get_field(
db, 'DBSubnetGroup', 'DBSubnetGroupName')
pg_dict['vpc'] = ""
try:
vpc_id = get_field(db, 'DBSubnetGroup', 'VpcId')
pg_dict['vpc_id'] = vpc_id
pg_dict['vpc'] = vpc_id
my_vpcs = ec2.describe_vpcs(VpcIds=[
vpc_id,
])
for my_vpc in my_vpcs['Vpcs']:
for tag in my_vpc['Tags']:
if tag['Key'] == "Name":
vpc_name = tag['Value']
pg_dict['vpc'] = vpc_name
break
except Exception as e:
pass
try:
pg_dict['is_in_pglist'] = is_in_pglist(
p_email, pg_dict['region'], pg_dict['instance'],
pg_dict['address'], pg_dict['port'],
pg_dict['dbname'], pg_dict['master_user'])
except Exception as e:
pass
pg_list.append(pg_dict)
except KeyboardInterrupt as e:
util.exit_message("Keyboard Interrupt", 1, p_isJSON)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
msg = "Unable to run rds.describe_db_instances(). \n" + str(e)
util.exit_message(msg, 1, p_isJSON)
if p_isJSON:
json_dict = {}
json_dict['data'] = pg_list
json_dict['state'] = 'completed'
print(json.dumps([json_dict]))
return (0)
if p_isVERBOSE:
print_verbose(pg_list)
return (0)
keys = [
'region', 'instance', 'status', 'dbname', 'db_class', 'vpc',
'az_is_multi'
]
headers = [
'Region', 'Instance', 'Status', 'DBName', 'Class', 'VPC', 'MultiAZ'
]
print("")
print(api.format_data_to_table(pg_list, keys, headers))
return (0)
