def get_modules_in_order():
""" Return a mapping between module_name and the order of the module.
Returns:
Dict. The output is of the form:
{'module_a': 1, 'module_b': 2, ...}
"""
portspecs = configyml.get_modules("../config")
# get_modules returns a dictionary in following form:
# { 'modules': [{'name': 'array_ops'}, {'name': 'bayes'}, ...,
# {'depends': ['array_ops'], 'name': 'stats'}, ... ]
# }
# The list of modules is pre-sorted using topological sort
module_order = dict()
for i, moduleinfo in enumerate(portspecs['modules']):
module_order[moduleinfo['name']] = i
return module_order
# Validate that db platform is correct
if __check_db_port(portid) == False:
__error("Invalid database platform specified.", True)
# Adjust MADlib directories for this port (if they exist)
global maddir_conf
if os.path.isdir(maddir + "/ports/" + portid + "/" + dbver + "/config"):
maddir_conf = maddir + "/ports/" + portid + "/" + dbver + "/config"
global maddir_lib
if os.path.isfile(maddir + "/ports/" + portid + "/" + dbver + \
"/lib/libmadlib.so"):
maddir_lib = maddir + "/ports/" + portid + "/" + dbver + \
"/lib/libmadlib.so"
# Get the list of modules for this port
global portspecs
portspecs = configyml.get_modules(maddir_conf)
else:
con_args = None
dbrev = None
##
# Parse COMMAND argument and compare with Ports.yml
##
# Debugging...
# print "OS rev: " + str(rev) + " > " + str(__get_rev_num(rev))
# print "DB rev: " + str(dbrev) + " > " + str(__get_rev_num(dbrev))
# Make sure we have the necessary paramaters to continue
if args.command[0] != 'version':
if not portid:
def main(argv):
parser = argparse.ArgumentParser(
description='MADlib package manager (' + rev + ')',
argument_default=False,
formatter_class=argparse.RawTextHelpFormatter,
epilog="""Example:
$ madpack install -s madlib -p greenplum -c gpadmin@mdw:5432/testdb
This will install MADlib objects into a Greenplum database called TESTDB
running on server MDW:5432. Installer will try to login as GPADMIN
and will prompt for password. The target schema will be MADLIB.
""")
parser.add_argument(
'command', metavar='COMMAND', nargs=1,
choices=['install','update','uninstall','reinstall','version','install-check'],
help = "One of the following options:\n"
+ " install/update : run sql scripts to load into DB\n"
+ " uninstall : run sql scripts to uninstall from DB\n"
+ " reinstall : performs uninstall and install\n"
+ " version : compare and print MADlib version (binaries vs database objects)\n"
+ " install-check : test all installed modules\n"
)
parser.add_argument(
'-c', '--conn', metavar='CONNSTR', nargs=1, dest='connstr', default=None,
help= "Connection string of the following syntax:\n"
+ " [user[/password]@][host][:port][/database]\n"
+ "If not provided default values will be derived for PostgerSQL and Greenplum:\n"
+ "- user: PGUSER or USER env variable or OS username\n"
+ "- pass: PGPASSWORD env variable or runtime prompt\n"
+ "- host: PGHOST env variable or 'localhost'\n"
+ "- port: PGPORT env variable or '5432'\n"
+ "- db: PGDATABASE env variable or OS username\n"
)
parser.add_argument('-s', '--schema', nargs=1, dest='schema',
metavar='SCHEMA', default='madlib',
help="Target schema for the database objects.")
parser.add_argument('-p', '--platform', nargs=1, dest='platform',
metavar='PLATFORM', choices=portid_list,
help="Target database platform, current choices: " + str(portid_list))
parser.add_argument('-v', '--verbose', dest='verbose',
action="store_true", help="Verbose mode.")
parser.add_argument('-l', '--keeplogs', dest='keeplogs', default=False,
action="store_true", help="Do not remove installation log files.")
##
# Get the arguments
##
args = parser.parse_args()
global verbose
verbose = args.verbose
__info("Arguments: " + str(args), verbose);
global keeplogs
keeplogs = args.keeplogs
##
# Parse SCHEMA
##
if len(args.schema[0]) > 1:
schema = args.schema[0].lower()
else:
schema = args.schema.lower()
##
# Parse DB Platform (== PortID) and compare with Ports.yml
##
global portid
if args.platform:
try:
# Get the DB platform name == DB port id
portid = args.platform[0].lower()
# Loop through available db ports
for port in ports['ports']:
if portid == port['id']:
# Adjust MADlib directories for this port (if they exist)
global maddir_conf
if os.path.isdir(maddir + "/ports/" + portid + "/config"):
maddir_conf = maddir + "/ports/" + portid + "/config"
global maddir_lib
if os.path.isfile(maddir + "/ports/" + portid + \
"/lib/libmadlib_" + portid + ".so"):
maddir_lib = maddir + "/ports/" + portid + \
"/lib/libmadlib_" + portid + ".so"
# Get the list of modules for this port
global portspecs
portspecs = configyml.get_modules(maddir_conf)
# print portspecs
except:
portid = None
__error("Can not find specs for port %s" % (args.platform[0]), True)
else:
portid = None
##
# Parse CONNSTR (only if PLATFORM and DBAPI2 are defined)
##
if portid:
connStr = "" if args.connstr is None else args.connstr[0]
(c_user, c_pass, c_host, c_port, c_db) = parseConnectionStr(connStr)
# Find the default values for PG and GP
if portid == 'postgres' or portid == 'greenplum':
if c_user is None:
c_user = os.environ.get('PGUSER', getpass.getuser())
if c_pass is None:
c_pass = os.environ.get('PGPASSWORD', None)
if c_host is None:
c_host = os.environ.get('PGHOST', 'localhost')
if c_port is None:
c_port = os.environ.get('PGPORT', '5432')
if c_db is None:
c_db = os.environ.get('PGDATABASE', c_user)
##
# Try connecting to the database
##
__info("Testing database connection...", verbose)
# Get password
if c_pass is None:
c_pass = getpass.getpass("Password for user %s: " % c_user)
# Set connection variables
global con_args
con_args['host'] = c_host + ':' + c_port
con_args['database'] = c_db
con_args['user'] = c_user
con_args['password'] = c_pass
# Get MADlib version in DB
dbrev = __get_madlib_dbver(schema)
# Validate that db platform is correct
if __check_db_port(portid) == False:
__error("Invalid database platform specified.", True)
else:
con_args = None
dbrev = None
##
# Parse COMMAND argument and compare with Ports.yml
##
# Debugging...
# print "OS rev: " + str(rev) + " > " + str(__get_rev_num(rev))
# print "DB rev: " + str(dbrev) + " > " + str(__get_rev_num(dbrev))
# Make sure we have the necessary paramaters to continue
if args.command[0] != 'version':
if not portid:
__error("Missing -p/--platform parameter.", True)
if not con_args:
__error("Unknown problem with database connection string: %s" % con_args, True)
###
# COMMAND: version
###
if args.command[0] == 'version':
__print_revs(rev, dbrev, con_args, schema)
###
# COMMAND: uninstall/reinstall
###
if args.command[0] == 'uninstall' or args.command[0] == 'reinstall':
if __get_rev_num(dbrev) == ['0']:
__info("Nothing to uninstall. No version found in schema %s." % schema.upper(), True)
return
# Find any potential data to lose
affected_objects = __run_sql_query("""
SELECT
n1.nspname AS schema,
relname AS relation,
attname AS column,
typname AS type
FROM
pg_attribute a,
pg_class c,
pg_type t,
pg_namespace n,
pg_namespace n1
WHERE
n.nspname = '%s'
AND t.typnamespace = n.oid
AND a.atttypid = t.oid
AND c.oid = a.attrelid
AND c.relnamespace = n1.oid
AND c.relkind = 'r'
ORDER BY
n1.nspname, relname, attname, typname""" % schema.lower()
, True
);
__info("*** Uninstalling MADlib ***", True)
__info("***********************************************************************************", True)
__info("* Schema %s and all database objects depending on it will be dropped!" % schema.upper(), True)
if affected_objects:
__info("* If you continue the following data will be lost (schema : table.column : type):", True)
for ao in affected_objects:
__info ( '* - ' + ao['schema'] + ' : ' + ao['relation'] + '.' + ao['column'] + ' : ' + ao['type'], True);
__info("***********************************************************************************", True)
__info("Would you like to continue? [Y/N]", True)
go = raw_input('>>> ').upper()
while go != 'Y' and go != 'N':
go = raw_input('Yes or No >>> ').upper()
# 2) Do the uninstall/drop
if go == 'N':
__info('No problem. Nothing dropped.', True)
return
elif go == 'Y':
__info("> dropping schema %s" % schema.upper(), verbose)
try:
__run_sql_query("DROP SCHEMA %s CASCADE;" % (schema), True)
except:
__error("Cannot drop schema %s." % schema.upper(), True)
__info('Schema %s (and all dependent objects) has been dropped.' % schema.upper(), True)
__info('MADlib uninstalled successfully.', True)
else:
return
###
# COMMAND: install/update/reinstall
###
if ( args.command[0] == 'install'
or args.command[0] == 'update'
or args.command[0] == 'reinstall'):
# Refresh MADlib version in DB
if args.command[0] == 'reinstall':
dbrev = __get_madlib_dbver(schema)
print ""
__info("*** Installing MADlib ***", True)
# 1) Compare OS and DB versions. Continue if OS > DB.
__print_revs(rev, dbrev, con_args, schema)
if __get_rev_num(dbrev) >= __get_rev_num(rev):
__info("Current MADlib version already up to date.", True)
return
# 2) Run installation
try:
__plpy_check(py_min_ver)
__db_install(schema, dbrev)
except:
__error("MADlib installation failed.", True)
###
# COMMAND: install-check
###
if args.command[0] == 'install-check':
# 1) Compare OS and DB versions. Continue if OS = DB.
if __get_rev_num(dbrev) != __get_rev_num(rev):
__print_revs(rev, dbrev, con_args, schema)
__info("Versions do not match. Install-check stopped.", True)
return
# Create install-check user
test_user = '******' + rev.replace('.','') + '_installcheck'
try:
__run_sql_query("DROP USER IF EXISTS %s;" % (test_user), False)
except:
__run_sql_query("DROP OWNED BY %s CASCADE;" % (test_user), True)
__run_sql_query("DROP USER IF EXISTS %s;" % (test_user), True)
__run_sql_query("CREATE USER %s;" % (test_user), True)
# TO DO:
# Change ALL to USAGE in the below GRANT command
# and fix the failing modules which still write to MADLIB schema.
__run_sql_query("GRANT ALL ON SCHEMA %s TO %s;"
% (schema, test_user), True)
# 2) Run test SQLs
__info("> Running test scripts for:", verbose)
# Loop through all modules
for moduleinfo in portspecs['modules']:
# Get module name
module = moduleinfo['name']
__info("> - %s" % module, verbose)
# Make a temp dir for this module (if doesn't exist)
cur_tmpdir = tmpdir + '/' + module + '/test'
__make_dir(cur_tmpdir)
# Find the module dir (platform specific or generic)
if os.path.isdir(maddir + "/ports/" + portid + "/modules/" + module):
maddir_mod = maddir + "/ports/" + portid + "/modules"
else:
maddir_mod = maddir + "/modules"
# Prepare test schema
test_schema = "madlib_installcheck_%s" % (module)
__run_sql_query("DROP SCHEMA IF EXISTS %s CASCADE; CREATE SCHEMA %s;"
% (test_schema, test_schema), True)
__run_sql_query("GRANT ALL ON SCHEMA %s TO %s;"
% (test_schema, test_user), True)
# Switch to test user and prepare the search_path
pre_sql = '-- Switch to test user:\n' \
'SET ROLE %s;\n' \
'-- Set SEARCH_PATH for install-check:\n' \
'SET search_path=%s,%s;\n' \
% (test_user, test_schema, schema)
# Loop through all test SQL files for this module
sql_files = maddir_mod + '/' + module + '/test/*.sql_in'
for sqlfile in sorted(glob.glob(sql_files)):
result = 'PASS'
# Set file names
tmpfile = cur_tmpdir + '/' + os.path.basename(sqlfile) + '.tmp'
logfile = cur_tmpdir + '/' + os.path.basename(sqlfile) + '.log'
# If there is no problem with the SQL file
milliseconds = 0
# Run the SQL
run_start = datetime.datetime.now()
retval = __run_sql_file(schema, maddir_mod, module, sqlfile, tmpfile, logfile, pre_sql)
# Runtime evaluation
run_end = datetime.datetime.now()
milliseconds = round((run_end - run_start).seconds * 1000 + (run_end - run_start).microseconds / 1000)
# Check the exit status
if retval != 0:
__error("Failed executing %s" % tmpfile, False)
__error("Check the log at %s" % logfile, False)
result = 'FAIL'
keeplogs = True
# Since every single statement in the test file gets logged,
# an empty log file indicates an empty or a failed test
elif os.path.isfile(logfile) and os.path.getsize(logfile) > 0:
result = 'PASS'
# Otherwise
else:
result = 'ERROR'
# Spit the line
print "TEST CASE RESULT|Module: " + module + \
"|" + os.path.basename(sqlfile) + "|" + result + \
"|Time: %d milliseconds" % (milliseconds)
# Cleanup test schema for the module
__run_sql_query( "DROP SCHEMA IF EXISTS %s CASCADE;" % (test_schema), True)
# Drop install-check user
__run_sql_query( "DROP OWNED BY %s CASCADE;" % (test_user), True)
__run_sql_query( "DROP USER %s;" % (test_user), True)
def main(argv):
parser = argparse.ArgumentParser(
description="MADlib package manager (" + rev + ")",
argument_default=False,
formatter_class=argparse.RawTextHelpFormatter,
epilog="""Example:
$ madpack install -s madlib -p greenplum -c gpadmin@mdw:5432/testdb
This will install MADlib objects into a Greenplum database called TESTDB
running on server MDW:5432. Installer will try to login as GPADMIN
and will prompt for password. The target schema will be MADLIB.
""",
)
parser.add_argument(
"command",
metavar="COMMAND",
nargs=1,
choices=["install", "update", "uninstall", "version", "install-check"],
help="One of the following options:\n"
+ " install/update : run sql scripts to load into DB\n"
+ " uninstall : run sql scripts to uninstall from DB\n"
+ " version : compare and print MADlib version (binaries vs database objects)\n"
+ " install-check : test all installed modules\n",
)
parser.add_argument(
"-c",
"--conn",
metavar="CONNSTR",
nargs=1,
dest="connstr",
default=None,
help="Connection string of the following syntax:\n"
+ " [user[/password]@][host][:port][/database]\n"
+ "If not provided default values will be derived for PostgerSQL and Greenplum:\n"
+ "- user: PGUSER or USER env variable or OS username\n"
+ "- pass: PGPASSWORD env variable or runtime prompt\n"
+ "- host: PGHOST env variable or 'localhost'\n"
+ "- port: PGPORT env variable or '5432'\n"
+ "- db: PGDATABASE env variable or OS username\n",
)
parser.add_argument(
"-s",
"--schema",
nargs=1,
dest="schema",
metavar="SCHEMA",
default="madlib",
help="Target schema for the database objects.",
)
parser.add_argument(
"-p",
"--platform",
nargs=1,
dest="platform",
metavar="PLATFORM",
choices=portid_list,
help="Target database platform, current choices: " + str(portid_list),
)
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose mode.")
parser.add_argument(
"-l",
"--keeplogs",
dest="keeplogs",
default=False,
action="store_true",
help="Do not remove installation log files.",
)
##
# Get the arguments
##
args = parser.parse_args()
global verbose
verbose = args.verbose
__info("Arguments: " + str(args), verbose)
global keeplogs
keeplogs = args.keeplogs
##
# Parse SCHEMA
##
if len(args.schema[0]) > 1:
schema = args.schema[0]
else:
schema = args.schema
##
# Parse DB Platform (== PortID) and compare with Ports.yml
##
global portid
if args.platform:
try:
# Get the DB platform name == DB port id
portid = args.platform[0].lower()
# Loop through available db ports
for port in ports["ports"]:
if portid == port["id"]:
# Adjust MADlib directories for this port (if they exist)
global maddir_conf
if os.path.isdir(maddir + "/ports/" + portid + "/config"):
maddir_conf = maddir + "/ports/" + portid + "/config"
global maddir_lib
if os.path.isfile(maddir + "/ports/" + portid + "/lib/libmadlib_" + portid + ".so"):
maddir_lib = maddir + "/ports/" + portid + "/lib/libmadlib_" + portid + ".so"
# Get the list of modules for this port
global portspecs
portspecs = configyml.get_modules(maddir_conf)
# print portspecs
except:
portid = None
__error("Can not find specs for port %s" % (args.platform[0]), True)
else:
portid = None
##
# Parse CONNSTR (only if PLATFORM and DBAPI2 are defined)
##
if portid:
connStr = "" if args.connstr is None else args.connstr[0]
(c_user, c_pass, c_host, c_port, c_db) = parseConnectionStr(connStr)
# Find the default values for PG and GP
if portid == "postgres" or portid == "greenplum":
if c_user is None:
c_user = os.environ.get("PGUSER", getpass.getuser())
if c_pass is None:
c_pass = os.environ.get("PGPASSWORD", None)
if c_host is None:
c_host = os.environ.get("PGHOST", "localhost")
if c_port is None:
c_port = os.environ.get("PGPORT", "5432")
if c_db is None:
c_db = os.environ.get("PGDATABASE", c_user)
##
# Try connecting to the database
##
__info("Testing database connection...", verbose)
# Get password
if c_pass is None:
c_pass = getpass.getpass("Password for user %s: " % c_user)
# Set connection variables
global con_args
con_args["host"] = c_host + ":" + c_port
con_args["database"] = c_db
con_args["user"] = c_user
con_args["password"] = c_pass
# Get MADlib version in DB
dbrev = __get_madlib_dbver(schema)
# Validate that db platform is correct
if __check_db_port(portid) == False:
__error("Invalid database platform specified.", True)
else:
con_args = None
dbrev = None
##
# Parse COMMAND argument and compare with Ports.yml
##
# Debugging...
# print "OS rev: " + str(rev) + " > " + str(__get_rev_num(rev))
# print "DB rev: " + str(dbrev) + " > " + str(__get_rev_num(dbrev))
# Make sure we have the necessary paramaters to continue
if args.command[0] != "version":
if not portid:
__error("Missing -p/--platform parameter.", True)
if not con_args:
__error("Unknown problem with database connection string: %s" % con_args, True)
###
# COMMAND: version
###
if args.command[0] == "version":
__print_revs(rev, dbrev, con_args, schema)
###
# COMMAND: install/update
###
elif args.command[0] == "install" or args.command[0] == "update":
# 1) Compare OS and DB versions. Continue if OS > DB.
__print_revs(rev, dbrev, con_args, schema)
if __get_rev_num(dbrev) >= __get_rev_num(rev):
__info("Current MADlib version already up to date.", True)
return
# 2) Run installation
try:
__plpy_check(py_min_ver)
__db_install(schema, dbrev)
except:
__error("MADlib installation failed.", True)
###
# COMMAND: uninstall (drops the schema)
###
if args.command[0] == "uninstall":
if __get_rev_num(dbrev) == ["0"]:
__info("Nothing to uninstall. No version found in schema %s." % schema.upper(), True)
return
__info("***************************************************************************", True)
__info("* Schema %s and all database objects depending on it will be dropped!" % schema.upper(), True)
__info("* This is potentially very dangerous operation! ", True)
__info("***************************************************************************", True)
__info("Would you like to continue? [Y/N]", True)
go = raw_input(">>> ").upper()
while go != "Y" and go != "N":
go = raw_input("Yes or No >>> ").upper()
# 2) Do the uninstall/drop
if go == "N":
__info("No problem. Nothing dropped.", True)
return
elif go == "Y":
__info("> dropping schema %s" % schema.upper(), verbose)
try:
__run_sql_query("DROP SCHEMA %s CASCADE;" % (schema), True)
except:
__error("Cannot drop schema %s." % schema.upper(), True)
__info("Schema %s (and all dependent objects) has been dropped." % schema.upper(), True)
__info("MADlib uninstalled successfully.", True)
else:
return
###
# COMMAND: install-check
###
if args.command[0] == "install-check":
# 1) Compare OS and DB versions. Continue if OS = DB.
if __get_rev_num(dbrev) != __get_rev_num(rev):
__print_revs(rev, dbrev, con_args, schema)
__info("Versions do not match. Install-check stopped.", True)
return
# 2) Run test SQLs
__info("> Running test scripts for:", verbose)
# Loop through all modules
for moduleinfo in portspecs["modules"]:
# Get module name
module = moduleinfo["name"]
__info("> - %s" % module, verbose)
# Make a temp dir for this module (if doesn't exist)
cur_tmpdir = tmpdir + "/" + module + "/test"
__make_dir(cur_tmpdir)
# Find the module dir (platform specific or generic)
if os.path.isdir(maddir + "/ports/" + portid + "/modules/" + module):
maddir_mod = maddir + "/ports/" + portid + "/modules"
else:
maddir_mod = maddir + "/modules"
# Loop through all test SQL files for this module
sql_files = maddir_mod + "/" + module + "/test/*.sql_in"
for sqlfile in glob.glob(sql_files):
# Set file names
tmpfile = cur_tmpdir + "/" + os.path.basename(sqlfile) + ".tmp"
logfile = cur_tmpdir + "/" + os.path.basename(sqlfile) + ".log"
# Run the SQL
run_start = datetime.datetime.now()
retval = __run_sql_file(schema, maddir_mod, module, sqlfile, tmpfile, logfile)
# Runtime evaluation
run_end = datetime.datetime.now()
milliseconds = round((run_end - run_start).seconds * 1000 + (run_end - run_start).microseconds / 1000)
# Analyze the output
# If DB cmd line returned error
if retval == 3:
__error("Failed executing: %s" % tmpfile, False)
__error("For details check: %s" % logfile, False)
result = "FAIL"
# If error log file exists
elif os.path.getsize(logfile) > 0:
result = "PASS"
# Otherwise
else:
result = "ERROR"
# Spit the line
print "TEST CASE RESULT|Module: " + module + "|" + os.path.basename(
sqlfile
) + "|" + result + "|Time: %d milliseconds" % (milliseconds)
"""
sort-module.py
Sort input strings based on the module name in them such that
the module dependencies are resolved. Example:
$ sort-module.py a/regress/b a/svec/b a/array_ops/b
> a/svec/b a/regress/b a/array_opts/b
Note this script assumes to be run at the exact current directory,
so you need change directory first to run it.
"""
import configyml
portspecs = configyml.get_modules("../config")
def find_order(path):
for i, moduleinfo in enumerate(portspecs['modules']):
modname = moduleinfo['name']
if modname in path:
return i
# return as the last if not found.
return len(portspecs['modules'])
def main(args):
print " ".join(sorted(args, key = find_order))
if __name__ == '__main__':
import sys
main(sys.argv[1:])
def main(argv):
parser = argparse.ArgumentParser(
description='MADlib package manager (' + rev + ')',
argument_default=False,
formatter_class=argparse.RawTextHelpFormatter,
epilog="""Example:
$ madpack install -s madlib -p greenplum -c gpadmin@mdw:5432/testdb
This will install MADlib objects into a Greenplum database called TESTDB
running on server MDW:5432. Installer will try to login as GPADMIN
and will prompt for password. The target schema will be MADLIB.
""")
parser.add_argument(
'command',
metavar='COMMAND',
nargs=1,
choices=[
'install', 'update', 'uninstall', 'reinstall', 'version',
'install-check'
],
help="One of the following options:\n" +
" install/update : run sql scripts to load into DB\n" +
" uninstall : run sql scripts to uninstall from DB\n" +
" reinstall : performs uninstall and install\n" +
" version : compare and print MADlib version (binaries vs database objects)\n"
+ " install-check : test all installed modules\n")
parser.add_argument(
'-c',
'--conn',
metavar='CONNSTR',
nargs=1,
dest='connstr',
default=None,
help="Connection string of the following syntax:\n" +
" [user[/password]@][host][:port][/database]\n" +
"If not provided default values will be derived for PostgerSQL and Greenplum:\n"
+ "- user: PGUSER or USER env variable or OS username\n" +
"- pass: PGPASSWORD env variable or runtime prompt\n" +
"- host: PGHOST env variable or 'localhost'\n" +
"- port: PGPORT env variable or '5432'\n" +
"- db: PGDATABASE env variable or OS username\n")
parser.add_argument('-s',
'--schema',
nargs=1,
dest='schema',
metavar='SCHEMA',
default='madlib',
help="Target schema for the database objects.")
parser.add_argument('-p',
'--platform',
nargs=1,
dest='platform',
metavar='PLATFORM',
choices=portid_list,
help="Target database platform, current choices: " +
str(portid_list))
parser.add_argument('-v',
'--verbose',
dest='verbose',
action="store_true",
help="Verbose mode.")
parser.add_argument('-l',
'--keeplogs',
dest='keeplogs',
default=False,
action="store_true",
help="Do not remove installation log files.")
##
# Get the arguments
##
args = parser.parse_args()
global verbose
verbose = args.verbose
__info("Arguments: " + str(args), verbose)
global keeplogs
keeplogs = args.keeplogs
##
# Parse SCHEMA
##
if len(args.schema[0]) > 1:
schema = args.schema[0].lower()
else:
schema = args.schema.lower()
##
# Parse DB Platform (== PortID) and compare with Ports.yml
##
global portid
if args.platform:
try:
# Get the DB platform name == DB port id
portid = args.platform[0].lower()
# Loop through available db ports
for port in ports['ports']:
if portid == port['id']:
# Adjust MADlib directories for this port (if they exist)
global maddir_conf
if os.path.isdir(maddir + "/ports/" + portid + "/config"):
maddir_conf = maddir + "/ports/" + portid + "/config"
global maddir_lib
if os.path.isfile(maddir + "/ports/" + portid + \
"/lib/libmadlib_" + portid + ".so"):
maddir_lib = maddir + "/ports/" + portid + \
"/lib/libmadlib_" + portid + ".so"
# Get the list of modules for this port
global portspecs
portspecs = configyml.get_modules(maddir_conf)
# print portspecs
except:
portid = None
__error("Can not find specs for port %s" % (args.platform[0]),
True)
else:
portid = None
##
# Parse CONNSTR (only if PLATFORM and DBAPI2 are defined)
##
if portid:
connStr = "" if args.connstr is None else args.connstr[0]
(c_user, c_pass, c_host, c_port, c_db) = parseConnectionStr(connStr)
# Find the default values for PG and GP
if portid == 'postgres' or portid == 'greenplum':
if c_user is None:
c_user = os.environ.get('PGUSER', getpass.getuser())
if c_pass is None:
c_pass = os.environ.get('PGPASSWORD', None)
if c_host is None:
c_host = os.environ.get('PGHOST', 'localhost')
if c_port is None:
c_port = os.environ.get('PGPORT', '5432')
if c_db is None:
c_db = os.environ.get('PGDATABASE', c_user)
##
# Try connecting to the database
##
__info("Testing database connection...", verbose)
# Get password
if c_pass is None:
c_pass = getpass.getpass("Password for user %s: " % c_user)
# Set connection variables
global con_args
con_args['host'] = c_host + ':' + c_port
con_args['database'] = c_db
con_args['user'] = c_user
con_args['password'] = c_pass
# Get MADlib version in DB
dbrev = __get_madlib_dbver(schema)
# Validate that db platform is correct
if __check_db_port(portid) == False:
__error("Invalid database platform specified.", True)
else:
con_args = None
dbrev = None
##
# Parse COMMAND argument and compare with Ports.yml
##
# Debugging...
# print "OS rev: " + str(rev) + " > " + str(__get_rev_num(rev))
# print "DB rev: " + str(dbrev) + " > " + str(__get_rev_num(dbrev))
# Make sure we have the necessary paramaters to continue
if args.command[0] != 'version':
if not portid:
__error("Missing -p/--platform parameter.", True)
if not con_args:
__error(
"Unknown problem with database connection string: %s" %
con_args, True)
###
# COMMAND: version
###
if args.command[0] == 'version':
__print_revs(rev, dbrev, con_args, schema)
###
# COMMAND: uninstall/reinstall
###
if args.command[0] == 'uninstall' or args.command[0] == 'reinstall':
if __get_rev_num(dbrev) == ['0']:
__info(
"Nothing to uninstall. No version found in schema %s." %
schema.upper(), True)
return
# Find any potential data to lose
affected_objects = __run_sql_query(
"""
SELECT
n1.nspname AS schema,
relname AS relation,
attname AS column,
typname AS type
FROM
pg_attribute a,
pg_class c,
pg_type t,
pg_namespace n,
pg_namespace n1
WHERE
n.nspname = '%s'
AND t.typnamespace = n.oid
AND a.atttypid = t.oid
AND c.oid = a.attrelid
AND c.relnamespace = n1.oid
AND c.relkind = 'r'
ORDER BY
n1.nspname, relname, attname, typname""" % schema.lower(),
True)
__info("*** Uninstalling MADlib ***", True)
__info(
"***********************************************************************************",
True)
__info(
"* Schema %s and all database objects depending on it will be dropped!"
% schema.upper(), True)
if affected_objects:
__info(
"* If you continue the following data will be lost (schema : table.column : type):",
True)
for ao in affected_objects:
__info(
'* - ' + ao['schema'] + ' : ' + ao['relation'] + '.' +
ao['column'] + ' : ' + ao['type'], True)
__info(
"***********************************************************************************",
True)
__info("Would you like to continue? [Y/N]", True)
go = raw_input('>>> ').upper()
while go != 'Y' and go != 'N':
go = raw_input('Yes or No >>> ').upper()
# 2) Do the uninstall/drop
if go == 'N':
__info('No problem. Nothing dropped.', True)
return
elif go == 'Y':
__info("> dropping schema %s" % schema.upper(), verbose)
try:
__run_sql_query("DROP SCHEMA %s CASCADE;" % (schema), True)
except:
__error("Cannot drop schema %s." % schema.upper(), True)
__info(
'Schema %s (and all dependent objects) has been dropped.' %
schema.upper(), True)
__info('MADlib uninstalled successfully.', True)
else:
return
###
# COMMAND: install/update/reinstall
###
if (args.command[0] == 'install' or args.command[0] == 'update'
or args.command[0] == 'reinstall'):
# Refresh MADlib version in DB
if args.command[0] == 'reinstall':
dbrev = __get_madlib_dbver(schema)
print ""
__info("*** Installing MADlib ***", True)
# 1) Compare OS and DB versions. Continue if OS > DB.
__print_revs(rev, dbrev, con_args, schema)
if __get_rev_num(dbrev) >= __get_rev_num(rev):
__info("Current MADlib version already up to date.", True)
return
# 2) Run installation
try:
__plpy_check(py_min_ver)
__db_install(schema, dbrev)
except:
__error("MADlib installation failed.", True)
###
# COMMAND: install-check
###
if args.command[0] == 'install-check':
# 1) Compare OS and DB versions. Continue if OS = DB.
if __get_rev_num(dbrev) != __get_rev_num(rev):
__print_revs(rev, dbrev, con_args, schema)
__info("Versions do not match. Install-check stopped.", True)
return
# Create install-check user
test_user = '******' + rev.replace('.', '') + '_installcheck'
try:
__run_sql_query("DROP USER IF EXISTS %s;" % (test_user), False)
except:
__run_sql_query("DROP OWNED BY %s CASCADE;" % (test_user), True)
__run_sql_query("DROP USER IF EXISTS %s;" % (test_user), True)
__run_sql_query("CREATE USER %s;" % (test_user), True)
# TO DO:
# Change ALL to USAGE in the below GRANT command
# and fix the failing modules which still write to MADLIB schema.
__run_sql_query("GRANT ALL ON SCHEMA %s TO %s;" % (schema, test_user),
True)
# 2) Run test SQLs
__info("> Running test scripts for:", verbose)
# Loop through all modules
for moduleinfo in portspecs['modules']:
# Get module name
module = moduleinfo['name']
__info("> - %s" % module, verbose)
# Make a temp dir for this module (if doesn't exist)
cur_tmpdir = tmpdir + '/' + module + '/test'
__make_dir(cur_tmpdir)
# Find the module dir (platform specific or generic)
if os.path.isdir(maddir + "/ports/" + portid + "/modules/" +
module):
maddir_mod = maddir + "/ports/" + portid + "/modules"
else:
maddir_mod = maddir + "/modules"
# Prepare test schema
test_schema = "madlib_installcheck_%s" % (module)
__run_sql_query(
"DROP SCHEMA IF EXISTS %s CASCADE; CREATE SCHEMA %s;" %
(test_schema, test_schema), True)
__run_sql_query(
"GRANT ALL ON SCHEMA %s TO %s;" % (test_schema, test_user),
True)
# Switch to test user and prepare the search_path
pre_sql = '-- Switch to test user:\n' \
'SET ROLE %s;\n' \
'-- Set SEARCH_PATH for install-check:\n' \
'SET search_path=%s,%s;\n' \
% (test_user, test_schema, schema)
# Loop through all test SQL files for this module
sql_files = maddir_mod + '/' + module + '/test/*.sql_in'
for sqlfile in sorted(glob.glob(sql_files)):
result = 'PASS'
# Set file names
tmpfile = cur_tmpdir + '/' + os.path.basename(sqlfile) + '.tmp'
logfile = cur_tmpdir + '/' + os.path.basename(sqlfile) + '.log'
# If there is no problem with the SQL file
milliseconds = 0
# Run the SQL
run_start = datetime.datetime.now()
retval = __run_sql_file(schema, maddir_mod, module, sqlfile,
tmpfile, logfile, pre_sql)
# Runtime evaluation
run_end = datetime.datetime.now()
milliseconds = round((run_end - run_start).seconds * 1000 +
(run_end - run_start).microseconds / 1000)
# Check the exit status
if retval != 0:
__error("Failed executing %s" % tmpfile, False)
__error("Check the log at %s" % logfile, False)
result = 'FAIL'
keeplogs = True
# Since every single statement in the test file gets logged,
# an empty log file indicates an empty or a failed test
elif os.path.isfile(logfile) and os.path.getsize(logfile) > 0:
result = 'PASS'
# Otherwise
else:
result = 'ERROR'
# Spit the line
print "TEST CASE RESULT|Module: " + module + \
"|" + os.path.basename(sqlfile) + "|" + result + \
"|Time: %d milliseconds" % (milliseconds)
# Cleanup test schema for the module
__run_sql_query(
"DROP SCHEMA IF EXISTS %s CASCADE;" % (test_schema), True)
# Drop install-check user
__run_sql_query("DROP OWNED BY %s CASCADE;" % (test_user), True)
__run_sql_query("DROP USER %s;" % (test_user), True)
# Validate that db platform is correct
if __check_db_port(portid) == False:
__error("Invalid database platform specified.", True)
# Adjust MADlib directories for this port (if they exist)
global maddir_conf
if os.path.isdir(maddir + "/ports/" + portid + "/" + dbver + "/config"):
maddir_conf = maddir + "/ports/" + portid + "/" + dbver + "/config"
global maddir_lib
if os.path.isfile(maddir + "/ports/" + portid + "/" + dbver + \
"/lib/libmadlib.so"):
maddir_lib = maddir + "/ports/" + portid + "/" + dbver + \
"/lib/libmadlib.so"
# Get the list of modules for this port
global portspecs
portspecs = configyml.get_modules(maddir_conf)
else:
con_args = None
dbrev = None
##
# Parse COMMAND argument and compare with Ports.yml
##
# Debugging...
# print "OS rev: " + str(rev) + " > " + str(__get_rev_num(rev))
# print "DB rev: " + str(dbrev) + " > " + str(__get_rev_num(dbrev))
# Make sure we have the necessary paramaters to continue
if args.command[0] != 'version':
if not portid:
"""
sort-module.py
Sort input strings based on the module name in them such that
the module dependencies are resolved. Example:
$ sort-module.py a/regress/b a/svec/b a/array_ops/b
> a/svec/b a/regress/b a/array_opts/b
Note this script assumes to be run at the exact current directory,
so you need change directory first to run it.
"""
import configyml
portspecs = configyml.get_modules("../config")
def find_order(path):
for i, moduleinfo in enumerate(portspecs['modules']):
modname = moduleinfo['name']
if modname in path:
return i
# return as the last if not found.
return len(portspecs['modules'])
def main(args):
print " ".join(sorted(args, key=find_order))
def main( argv):
parser = argparse.ArgumentParser( description='MADlib package manager (' + rev + ')',
argument_default=False,
formatter_class=argparse.RawTextHelpFormatter,
epilog="""Example:
$ madpack install -s madlib -p greenplum -c gpadmin@mdw:5432/testdb
This will install MADlib objects into a Greenplum database called TESTDB
running on server MDW:5432. Installer will try to login as GPADMIN
and will prompt for password. The target schema will be MADLIB.
""")
parser.add_argument( 'command', metavar='COMMAND', nargs=1,
choices=['install','update','uninstall','version','install-check'],
help="""One of the following options:
install/update : run sql scripts to load into DB
uninstall : run sql scripts to uninstall from DB
version : compare and print MADlib version (binaries vs database objects)
install-check : test all installed modules""")
parser.add_argument( '-a', '--api', nargs=1, dest='api', metavar='API',
help="Python module for your database platform (DBAPI2 compliant)")
parser.add_argument( '-c', '--conn', metavar='CONNSTR', nargs=1, dest='connstr', default=None,
help="connection string of the following syntax: user[/pass]@host:port/dbname")
parser.add_argument( '-s', '--schema', nargs=1, dest='schema',
metavar='SCHEMA', default='madlib',
help="target schema for the database objects")
parser.add_argument( '-p', '--platform', nargs=1, dest='platform',
metavar='PLATFORM', choices=portid_list,
help="target database platform, current choices: " + str(portid_list))
parser.add_argument( '-v', '--verbose', dest='verbose',
action="store_true", help="more output")
##
# Get the arguments
##
args = parser.parse_args()
global verbose
verbose = args.verbose
__info( "Arguments: " + str(args), verbose);
##
# Parse DBAPI2 overwrite
##
try:
# Get the module name
api = args.api[0]
except:
api = None
# And try importing it
if api != None:
try:
dbapi2 = __import_dbapi( api)
__info( "Imported user defined dbapi2 module (%s)." % (api), verbose)
except:
__error( "cannot import dbapi2 module: %s. Try using the deafult one from Ports.yml file." % (args.api[0]), True)
##
# Parse SCHEMA
##
if len(args.schema[0]) > 1:
schema = args.schema[0]
else:
schema = args.schema
##
# Parse DB PLATFORM and compare with Ports.yml
##
if args.platform:
try:
platform = args.platform[0]
# Loop through available db ports
for port in ports['ports']:
if args.platform[0] == port['id']:
# Get the DB name
platform = args.platform
# Get the Python DBAPI2 module
portapi = port['dbapi2']
# Get the DB id
global portid
portid = port['id']
# Adjust MADlib directories for this port (if they exist)
global maddir_conf
if os.path.isdir( maddir + "/ports/" + portid + "/config"):
maddir_conf = maddir + "/ports/" + portid + "/config"
global maddir_lib
if os.path.isfile( maddir + "/ports/" + portid + \
"/lib/libmadlib_" + portid + ".so"):
maddir_lib = maddir + "/ports/" + portid + \
"/lib/libmadlib_" + portid + ".so"
# Get the list of modules for this port
global portspecs
portspecs = configyml.get_modules( maddir_conf)
# print portspecs
except:
platform = None
__error( "Can not find specs for port %s" % (args.platform[0]), True)
else:
platform = None
portapi = None
##
# If no API defined yet (try the default API - from Ports.yml)
##
if api is None and portapi != None:
# For POSTGRES import Pygresql.pgdb from madpack package
if portid.upper() == 'POSTGRES':
try:
sys.path.append( maddir + "/ports/postgres/madpack")
dbapi2 = __import_dbapi( portapi)
__info( "Imported dbapi2 module (%s) defined in Ports.yml." % (portapi), verbose)
except:
__error( "cannot import dbapi2 module: %s. You can try specifying a different one (see --help)." % (portapi), True)
# For GREENPLUM use the GP one: $GPHOME/lib/python/pygresql
else:
try:
dbapi2 = __import_dbapi( portapi)
__info( "Imported dbapi2 module (%s) defined in Ports.yml." % (portapi), verbose)
except:
__error( "cannot import dbapi2 module: %s. You can try specifying a different one (see --help)." % (portapi), True)
##
# Parse CONNSTR (only if PLATFORM and DBAPI2 are defined)
##
if platform != None and dbapi2 and args.connstr != None:
try:
(c_user, c_dsn) = args.connstr[0].split('@')
except:
__error( "invalid connection string: missing '@' separator (see '-h' for help)", True)
try:
(c_user, c_pass) = c_user.split('/')
except:
pass_prompt = True
c_pass = None
if c_user == '':
__error( "invalid connection string: missing 'user' parameter (see '-h' for help)", True)
try:
(c_dsn, c_db) = c_dsn.split('/')
except:
__error( "invalid connection string: missing '/' separator (see '-h' for help)", True)
try:
(c_host, c_port) = c_dsn.split(':')
except:
__error( "invalid connection string: missing ':' separator (see '-h' for help)", True)
##
# Try connecting to the database
##
__info( "Testing database connection...", verbose)
# get password
if c_pass == None:
c_pass = getpass.getpass( "Password for user %s: " % c_user)
# set connection variables
global con_args
con_args['host'] = c_dsn
con_args['database'] = c_db
con_args['user'] = c_user
con_args['password'] = c_pass
# Open connection
global dbconn
dbconn = dbapi2.connect( **con_args)
__info( 'Database connection successful: %s@%s/%s' % (c_user, c_dsn, c_db), verbose)
# Get MADlib version in DB
dbrev = __get_madlib_dbver( schema)
# Close connection
dbconn.close()
else:
con_args = None
dbrev = None
##
# Parse COMMAND argument and compare with Ports.yml
##
# Debugging...
# print "OS rev: " + str(rev) + " > " + str(__get_rev_num( rev))
# print "DB rev: " + str(dbrev) + " > " + str(__get_rev_num( dbrev))
###
# COMMAND: version
###
if args.command[0] == 'version':
__print_revs( rev, dbrev, con_args, schema)
###
# COMMAND: install/update
###
elif args.command[0] == 'install' or args.command[0] == 'update':
# 1) Compare OS and DB versions. Continue if OS > DB.
__print_revs( rev, dbrev, con_args, schema)
if __get_rev_num( dbrev) >= __get_rev_num( rev):
__info( "Current MADlib version already up to date.", True)
return
# 2) Run installation
dbconn = dbapi2.connect( **con_args)
try:
__plpy_check( py_min_ver)
__db_install( schema, dbrev)
except:
__error( "MADlib installation failed.", True)
dbconn.close()
###
# COMMAND: uninstall (drops the schema)
###
if args.command[0] == 'uninstall':
# 1) Check versions and confirm deletion
# __print_revs( rev, dbrev, con_args, schema)
if __get_rev_num( dbrev) == ['0']:
__info( "Nothing to uninstall. No version found in schema %s." % schema.upper(), True)
return
__info( "***************************************************************************", True)
__info( "* Schema %s and all database objects depending on it will be dropped!" % schema.upper(), True)
__info( "* This is potentially very dangerous operation! ", True)
__info( "***************************************************************************", True)
__info( "Would you like to continue? [Y/N]", True)
go = raw_input( '>>> ').upper()
while go != 'Y' and go != 'N':
go = raw_input( 'Yes or No >>> ').upper()
# 2) Do the uninstall/drop
if go == 'N':
__info( 'No problem. Nothing dropped.', True)
return
elif go == 'Y':
dbconn = dbapi2.connect( **con_args)
__info( "> dropping schema %s" % schema.upper(), verbose)
try:
cur = dbconn.cursor()
cur.execute( "SET client_min_messages=WARNING;")
cur.execute( "DROP SCHEMA %s CASCADE;" % (schema))
dbconn.commit();
except:
__error( "Cannot drop schema %s." % schema.upper(), True)
dbconn.close()
__info( 'Schema %s (and all dependent objects) has been dropped.' % schema.upper(), True)
__info( 'MADlib is uninstalled.', True)
else:
return
###
# COMMAND: install-check
###
if args.command[0] == 'install-check':
# 1) Compare OS and DB versions. Continue if OS = DB.
if __get_rev_num( dbrev) != __get_rev_num( rev):
__print_revs( rev, dbrev, con_args, schema)
__info( "Versions do not match. Install-check stopped.", True)
return
# 2) Run test SQLs
__info( "> Running test scripts for:", verbose)
# Loop through all modules
for moduleinfo in portspecs['modules']:
# Get module name
module = moduleinfo['name']
__info("> - %s" % module, verbose)
# Make a temp dir for this module (if doesn't exist)
cur_tmpdir = tmpdir + '/' + module + '/test'
__make_log_dir( cur_tmpdir)
# Find the module dir (platform specific or generic)
if os.path.isdir( maddir + "/ports/" + portid + "/modules/" + module):
maddir_mod = maddir + "/ports/" + portid + "/modules"
else:
maddir_mod = maddir + "/modules"
# Loop through all test SQL files for this module
sql_files = maddir_mod + '/' + module + '/test/*.sql_in'
for sqlfile in glob.glob( sql_files):
# Set file names
tmpfile = cur_tmpdir + '/' + os.path.basename(sqlfile) + '.tmp'
logfile = cur_tmpdir + '/' + os.path.basename(sqlfile) + '.log'
# Run the SQL
run_start = datetime.datetime.now()
retval = __db_run_sql( schema, maddir_mod, module, sqlfile, tmpfile, logfile)
# Runtime evaluation
run_end = datetime.datetime.now()
milliseconds = round( (run_end - run_start).seconds * 1000 + (run_end - run_start).microseconds / 1000)
# Analyze the output
# If DB cmd line returned error
if retval == 3:
__error( "Failed executing: %s" % tmpfile, False)
__error( "For details check: %s" % logfile, False)
result = 'FAIL'
# If error log file exists
elif os.path.getsize( logfile) > 0:
result = 'PASS'
# Otherwise
else:
result = 'ERROR'
# Spit the line
print "TEST CASE RESULT|Module: " + module + \
"|" + os.path.basename(sqlfile) + "|" + result + \
"|Time: %d milliseconds" % (milliseconds)