def connect(self):
if self.is_valid_postgres_schema() is False:
return False
if self.conn is not None and self.conn.closed == 0:
return True
try:
svr = ACHelpers.resolve_vault_references(
self.config_dict[POSTGRESQL_SCHEMA_HOSTNAME])
prt = ACHelpers.resolve_vault_references(
self.config_dict[POSTGRESQL_SCHEMA_PORT])
usr = ACHelpers.resolve_vault_references(
self.config_dict[POSTGRESQL_SCHEMA_USERNAME])
pwd = ACHelpers.resolve_vault_references(
self.config_dict[POSTGRESQL_SCHEMA_PASSWORD])
db = ACHelpers.resolve_vault_references(
self.config_dict[POSTGRESQL_SCHEMA_DATABASE])
self.conn = psycopg2.connect(host=svr,
port=prt,
user=usr,
password=pwd,
database=db)
except Exception, e:
self.conn = None
ACLogger().get_logger().error('Failed PostgreSQL login: %s' %
str(e))
return False
def create_from_files(self, file_manifest_json):
try:
with open(file_manifest_json) as data_file:
self.config_dict = json.load(data_file,
object_pairs_hook=OrderedDict)
except (IOError, ValueError), ex:
ACLogger().get_logger().error(
'invalid json loaded to ACPentahoDI %s', ex.message)
self.config_dict = None
return False
def handle_exceptions(cursor,
the_key_name,
ee,
the_query_set,
commit=False):
if POSTGRESQL_IGNORE_EXCEPTIONS in query_set:
ignore_exceptions = ACHelpers.string_is_true(
the_query_set[POSTGRESQL_IGNORE_EXCEPTIONS])
else:
ignore_exceptions = False
if ignore_exceptions is False:
ACLogger().get_logger().error(
'PostgreSql exception for key %s (%s)' %
(the_key_name, ee))
if commit is True:
try:
self.conn.commit()
except Exception, exe:
ACLogger().get_logger().info(
'PostgreSql unable to commit post exception for key %s (%s)'
% (the_key_name, exe))
cursor.close()
return False
def is_valid_schema(self):
if self.config_dict is None:
return False
error_list = list()
if PYTHON_ANACONDA_KEYS not in self.config_dict or \
isinstance(self.config_dict[PYTHON_ANACONDA_KEYS], dict) is False:
error_list.append(PYTHON_ANACONDA_KEYS)
if self.config_dict[PYTHON_ANACONDA_KEYS] is not None and \
isinstance(self.config_dict[PYTHON_ANACONDA_KEYS], dict) is True:
for key, item in self.config_dict[TABLEAU_SERVER_KEYS].iteritems():
if isinstance(item, dict) is False:
error_list.append('key: ' + key +
' must be in bracket format {}')
else:
if PYTHON_ANACONDA_FILENAME not in item:
error_list.append('key: ' + key + ': ' +
PYTHON_ANACONDA_FILENAME)
if PYTHON_ANACONDA_FILE_PARAMETERS in item:
if isinstance(item[PYTHON_ANACONDA_FILE_PARAMETERS],
dict) is False:
error_list.append('must be dict for key: ' + key +
': ' +
PYTHON_ANACONDA_FILE_PARAMETERS)
else:
for param, param_val in item[
PYTHON_ANACONDA_FILE_PARAMETERS].iteritems(
):
if isinstance(
param,
basestring) is False or isinstance(
param_val, basestring) is False:
error_list.append(
'must be strings for {k: val }: ' +
key + ': ' +
PYTHON_ANACONDA_FILE_PARAMETERS)
if len(error_list) > 0:
ACLogger().get_logger().error(
'invalid ACPythonAnaconda json, errors in: %s' % error_list)
return False
else:
return True
def is_valid_postgres_schema(self):
if self.config_dict is None:
return False
error_list = list()
if POSTGRESQL_SCHEMA_USERNAME not in self.config_dict:
error_list.append(POSTGRESQL_SCHEMA_USERNAME)
if POSTGRESQL_SCHEMA_PASSWORD not in self.config_dict:
error_list.append(POSTGRESQL_SCHEMA_PASSWORD)
if POSTGRESQL_SCHEMA_HOSTNAME not in self.config_dict:
error_list.append(POSTGRESQL_SCHEMA_HOSTNAME)
if POSTGRESQL_SCHEMA_DATABASE not in self.config_dict:
error_list.append(POSTGRESQL_SCHEMA_DATABASE)
if POSTGRESQL_SCHEMA_PORT not in self.config_dict:
error_list.append(POSTGRESQL_SCHEMA_PORT)
if POSTGRESQL_SCHEMA_KEY not in self.config_dict:
error_list.append(POSTGRESQL_SCHEMA_KEY)
if POSTGRESQL_STATEMENTS not in self.config_dict:
error_list.append(POSTGRESQL_STATEMENTS)
else:
query_dict = self.config_dict[POSTGRESQL_STATEMENTS]
if isinstance(query_dict, dict) is False:
error_list.append(POSTGRESQL_STATEMENTS)
else:
for key, val in query_dict.iteritems():
if POSTGRESQL_SQL_STRING not in val:
error_list.append(key + ':' + POSTGRESQL_SQL_STRING)
if POSTGRESQL_TYPE not in val:
error_list.append(key + ':' + POSTGRESQL_TYPE)
else:
query_type = val[POSTGRESQL_TYPE]
if query_type not in POSTGRE_EXEC_TYPES:
error_list.append(key + ':' + POSTGRESQL_TYPE)
if len(error_list) > 0:
ACLogger().get_logger().error(
'invalid PostgreSQL json, errors in: %s' % error_list)
return False
else:
return True
def is_valid_pentahodi_schema(self):
if self.config_dict is None:
return False
error_list = list()
if PENTAHODI_LOGLEVEL not in self.config_dict:
error_list.append(PENTAHODI_LOGLEVEL)
if PENTAHODI_KEYS not in self.config_dict or \
isinstance(self.config_dict[PENTAHODI_KEYS], dict) is False:
error_list.append(PENTAHODI_KEYS)
if self.config_dict[PENTAHODI_KEYS] is not None and \
isinstance(self.config_dict[PENTAHODI_KEYS], dict) is True:
for key, item in self.config_dict[TABLEAU_SERVER_KEYS].iteritems():
if isinstance(item,
dict) is False or PENTAHODI_FILENAME not in item:
error_list.append('key: ' + key + ': ' +
PENTAHODI_FILENAME)
if PENTAHODI_FILE_PARAMETERS in item:
if isinstance(item[PENTAHODI_FILE_PARAMETERS],
dict) is False:
error_list.append('must be dict for key: ' + key +
': ' + PENTAHODI_FILE_PARAMETERS)
else:
for param, param_val in item[
PENTAHODI_FILE_PARAMETERS].iteritems():
if isinstance(param,
basestring) is False or isinstance(
param_val, basestring) is False:
error_list.append(
'must be strings for {k: val }: ' + key +
': ' + PENTAHODI_FILE_PARAMETERS)
if len(error_list) > 0:
ACLogger().get_logger().error(
'invalid PentahoDI json, errors in: %s' % error_list)
return False
else:
return True
def _complete_creation(self):
if self.is_valid_pentahodi_schema() is False:
ACLogger().get_logger().error('invalid Tableau json schema %s',
self.config_dict)
return False
return True
def execute(self, key_values_dict, pdi_path, pdi_file_path=None):
rv = True
if PENTAHODI_LOGLEVEL in self.config_dict:
loglevel = '%s' % self.config_dict[PENTAHODI_LOGLEVEL]
else:
loglevel = 'Minimal'
shellret = ''
if self.config_dict[PENTAHODI_KEYS] is not None and \
isinstance(self.config_dict[PENTAHODI_KEYS], dict) is True:
for key, item in self.config_dict[TABLEAU_SERVER_KEYS].iteritems():
key_values_dict[key] = dict()
key_values_dict[key]['output'] = ACHelpers.compress('')
key_values_dict[key]['status'] = False
pdi = os.path.join(pdi_path, 'pan.sh')
if PENTAHODI_INTEGRATION_OUTPUT_FILENAME in item:
output_file_path = os.path.join(
os.getcwd(),
item[PENTAHODI_INTEGRATION_OUTPUT_FILENAME])
else:
output_file_path = None
if pdi_file_path is None:
pdi_file = os.path.join(os.getcwd(), 'docker-share',
item[PENTAHODI_FILENAME])
else:
pdi_file = os.path.join(pdi_file_path,
item[PENTAHODI_FILENAME])
try:
# pan.sh -file="/PRD/Customer Dimension.ktr" -level=Minimal -param:MASTER_HOST=192.168.1.3 -param:MASTER_PORT=8181
params = [
pdi,
'-file=%s' % pdi_file,
'-level=%s' % loglevel
]
if PENTAHODI_FILE_PARAMETERS in item:
for p, pval in item[
PENTAHODI_FILE_PARAMETERS].iteritems():
params.append('-param:%s=%s' % (p, pval))
shellret = subprocess.check_output(params)
ACLogger().get_logger().info("calling pan by: %s" % params)
key_values_dict[key]['output'] = ACHelpers.compress('')
if shellret is not None and shellret != '':
key_values_dict[key]['status'] = True
if output_file_path is not None:
ACLogger().get_logger().info(
"looking for file @: %s" % output_file_path)
try:
statinfo = os.stat(output_file_path)
except:
pass
else:
if statinfo.st_size > 0:
with open(output_file_path, 'rb') as f:
d = f.read()
if d is not None and len(d) > 0:
key_values_dict[key][
'output'] = ACHelpers.compress(
d)
ACLogger().get_logger().info(
" got outuput data for key %s: %s"
% (key, d))
else:
key_values_dict[key]['status'] = False
rv = False
except subprocess.CalledProcessError, e:
ACLogger().get_logger().error(
'PentahoDI execture exception: %s' % str(e))
key_values_dict[key]['status'] = False
finally:
def execute(self, key_values_dict, jupyter_dir, notebook_file_dir=None):
rv = True
shellret = ''
if self.config_dict[PYTHON_ANACONDA_KEYS] is not None and \
isinstance(self.config_dict[PYTHON_ANACONDA_KEYS], dict) is True:
for key, item in self.config_dict[TABLEAU_SERVER_KEYS].iteritems():
key_values_dict[key] = dict()
output_file_path = None
if jupyter_dir is not None:
jupyter = os.path.join(jupyter_dir, 'jupyter')
else:
jupyter = '/opt/conda/bin/jupyter'
if notebook_file_dir is None:
notebook_file = os.path.join(os.getcwd(), 'docker-share', item[PYTHON_ANACONDA_FILENAME])
if PYTHON_ANACONDA_NOTEBOOK_OUTPUT_FILENAME in item:
output_file_path = os.path.join(os.getcwd(), 'docker-share',
item[PYTHON_ANACONDA_NOTEBOOK_OUTPUT_FILENAME])
else:
notebook_file = os.path.join(notebook_file_dir, item[PYTHON_ANACONDA_FILENAME])
if PYTHON_ANACONDA_NOTEBOOK_OUTPUT_FILENAME in item:
output_file_path = os.path.join(notebook_file_dir,
item[PYTHON_ANACONDA_NOTEBOOK_OUTPUT_FILENAME])
try:
try:
statinfo = os.stat(notebook_file)
except:
pass
else:
if statinfo is None or statinfo.st_size == 0:
ACLogger().get_logger().error(
'ACPythonAnaconda unable to open notebook_file %s' % notebook_file)
key_values_dict[key]['status'] = False
return False
params = [jupyter, 'nbconvert', '--execute' , '%s' % notebook_file]
my_env = os.environ.copy()
if PYTHON_ANACONDA_FILE_PARAMETERS in item:
for p, pval in item[PYTHON_ANACONDA_FILE_PARAMETERS].iteritems():
my_env[p] = pval
#ACLogger.log_and_print("calling jupyter with env variables of: %s" % my_env)
shellret = subprocess.check_output(params, env=my_env, stderr=subprocess.STDOUT)
key_values_dict[key]['output'] = ACHelpers.compress('')
ACLogger.log_and_print("calling jupyter by: %s" % params)
if shellret is not None:
key_values_dict[key]['status'] = True
if output_file_path is not None:
try:
statinfo = os.stat(output_file_path)
except:
pass
else:
if statinfo.st_size > 0:
with open(output_file_path, 'rb') as f:
d = f.read()
if d is not None and len(d) > 0:
key_values_dict[key]['output'] = ACHelpers.compress(d)
ACLogger().get_logger().info(" got outuput data for key %s: %s" % (key,d))
else:
key_values_dict[key]['status'] = False
rv = False
except Exception, e:
ACLogger().get_logger().error('ACPythonAnaconda execute exception: %s shellret(%s)' % (str(e), shellret))
key_values_dict[key]['status'] = False
finally:
def postgres_execute(self, key_name, the_file):
if self.config_dict is None or key_name is None or self.connect(
) is False or the_file is None:
return False
if isinstance(the_file, file) is False or the_file.closed is True:
ACLogger().get_logger().error(
'ACPostgreSQL: must have open file object')
return False
try:
stmnts = self.config_dict[POSTGRESQL_STATEMENTS]
query_set = stmnts[key_name]
sql = query_set[POSTGRESQL_SQL_STRING]
sql = ACHelpers.resolve_vault_references(sql)
query_type = query_set[POSTGRESQL_TYPE]
except KeyError:
ACLogger().get_logger().error(
'ACPostgreSQL: cannot get statements from config for key %s',
key_name)
return False
insert_column_names = False
try:
if ACHelpers.string_is_true(
query_set[POSTGRESQL_INSERT_COLUMN_NAMES]) is True:
insert_column_names = True
except KeyError:
pass
to_json_parser = False
try:
if POSTGRESQL_SQL_PARSER in query_set:
if POSTGRESQL_SQL_TO_JSON_PARSER in query_set[
POSTGRESQL_SQL_PARSER]:
to_json_parser = True
if insert_column_names is False:
ACLogger().get_logger().error(
'ACPostgreSQL: key %s must be true when %s present, resetting'
% (POSTGRESQL_INSERT_COLUMN_NAMES,
POSTGRESQL_SQL_TO_JSON_PARSER))
insert_column_names = True
except KeyError:
pass
def handle_exceptions(cursor,
the_key_name,
ee,
the_query_set,
commit=False):
if POSTGRESQL_IGNORE_EXCEPTIONS in query_set:
ignore_exceptions = ACHelpers.string_is_true(
the_query_set[POSTGRESQL_IGNORE_EXCEPTIONS])
else:
ignore_exceptions = False
if ignore_exceptions is False:
ACLogger().get_logger().error(
'PostgreSql exception for key %s (%s)' %
(the_key_name, ee))
if commit is True:
try:
self.conn.commit()
except Exception, exe:
ACLogger().get_logger().info(
'PostgreSql unable to commit post exception for key %s (%s)'
% (the_key_name, exe))
cursor.close()
return False
else:
def _complete_creation(self):
if self.is_valid_postgres_schema() is False:
ACLogger().get_logger().error('invalid PostgreSQL json schema %s',
self.config_dict)
return False
return True
try:
self.conn.commit()
except Exception, exe:
ACLogger().get_logger().info(
'PostgreSql unable to commit post exception for key %s (%s)'
% (the_key_name, exe))
cursor.close()
return False
else:
#ACLogger().get_logger().info('PostgreSql ignored exception for key %s (%s)' % (key_name, e))
if commit is True:
try:
self.conn.commit()
except Exception, exe:
ACLogger().get_logger().info(
'PostgreSql unable to commit post exception for key %s (%s)'
% (key_name, exe))
cursor.close()
return True
if query_type == POSTGRESQL_TYPE_EXECUTE_SCALAR:
curs = self.conn.cursor()
try:
curs.execute(sql)
result = curs.fetchone()
self.conn.commit()
if isinstance(result, tuple):
the_file.write(str(result[0]))
else:
the_file.write('ERROR')
except Exception, e:
def _complete_creation(self):
if self.is_valid_schema() is False:
ACLogger().get_logger().error(
'invalid ACPythonAnaconda json schema %s', self.config_dict)
return False
return True
def execute(self, key_values_dict, jupyter_dir, notebook_file_dir=None):
rv = True
shellret = ''
if self.config_dict[PYTHON_ANACONDA_KEYS] is not None and \
isinstance(self.config_dict[PYTHON_ANACONDA_KEYS], dict) is True:
for key, item in self.config_dict[TABLEAU_SERVER_KEYS].iteritems():
key_values_dict[key] = dict()
output_file_path = None
if jupyter_dir is not None:
jupyter = os.path.join(jupyter_dir, 'jupyter')
else:
jupyter = '/opt/conda/bin/jupyter'
if notebook_file_dir is None:
notebook_file = os.path.join(
os.getcwd(), 'docker-share',
item[PYTHON_ANACONDA_FILENAME])
if PYTHON_ANACONDA_NOTEBOOK_OUTPUT_FILENAME in item:
output_file_path = os.path.join(
os.getcwd(), 'docker-share',
item[PYTHON_ANACONDA_NOTEBOOK_OUTPUT_FILENAME])
else:
notebook_file = os.path.join(
notebook_file_dir, item[PYTHON_ANACONDA_FILENAME])
if PYTHON_ANACONDA_NOTEBOOK_OUTPUT_FILENAME in item:
output_file_path = os.path.join(
notebook_file_dir,
item[PYTHON_ANACONDA_NOTEBOOK_OUTPUT_FILENAME])
try:
try:
statinfo = os.stat(notebook_file)
except:
pass
else:
if statinfo is None or statinfo.st_size == 0:
ACLogger().get_logger().error(
'ACPythonAnaconda unable to open notebook_file %s'
% notebook_file)
key_values_dict[key]['status'] = False
return False
params = [
jupyter, 'nbconvert', '--execute',
'%s' % notebook_file
]
my_env = os.environ.copy()
if PYTHON_ANACONDA_FILE_PARAMETERS in item:
for p, pval in item[
PYTHON_ANACONDA_FILE_PARAMETERS].iteritems():
my_env[p] = pval
#ACLogger.log_and_print("calling jupyter with env variables of: %s" % my_env)
shellret = subprocess.check_output(
params, env=my_env, stderr=subprocess.STDOUT)
key_values_dict[key]['output'] = ACHelpers.compress('')
ACLogger.log_and_print("calling jupyter by: %s" % params)
if shellret is not None:
key_values_dict[key]['status'] = True
if output_file_path is not None:
try:
statinfo = os.stat(output_file_path)
except:
pass
else:
if statinfo.st_size > 0:
with open(output_file_path, 'rb') as f:
d = f.read()
if d is not None and len(d) > 0:
key_values_dict[key][
'output'] = ACHelpers.compress(
d)
ACLogger().get_logger().info(
" got outuput data for key %s: %s"
% (key, d))
else:
key_values_dict[key]['status'] = False
rv = False
except Exception, e:
ACLogger().get_logger().error(
'ACPythonAnaconda execute exception: %s shellret(%s)' %
(str(e), shellret))
key_values_dict[key]['status'] = False
finally:
