def test(self):
"""
Test connector connection to database
Return None if OK
"""
api_instance = self.__api(self.__engine.api_client)
try:
self.__logger.debug("test connector id %s" %
self.database_connector_id)
# body added to workaround an issue with API
response = api_instance.test_database_connector(
self.database_connector_id,
body=self.obj,
_request_timeout=self.__engine.get_timeout())
if "Connection Failed" in response.response:
print_error(
"Connector test {} failed. Error message is: {}".format(
self.connector_name, response.response))
return 1
else:
print_message("Connector test {} succeeded".format(
self.connector_name))
return 0
except self.__apiexc as e:
print_error(e.body)
self.__logger.error(e)
return 1
def init_metadata(self):
if self.__conn:
self.__logger.debug("Creating table in config")
sql_dxm_engine_info = \
"""CREATE TABLE IF NOT EXISTS dxm_engine_info
(
engine_name varchar(60),
ip_address varchar(15),
username varchar(60),
password varchar(60),
protocol char(6),
port integer,
auth_id varchar(30),
defengine char(1),
proxy_url varchar(60),
proxy_user varchar(60),
unique(engine_name, username)
)"""
try:
if self.is_table():
if self.get_database_version()==0:
# we need to add proxy info
self.__logger.debug("Adding columns to the table")
self.__cursor.execute('alter table dxm_engine_info add proxy_url varchar(60)')
self.__cursor.execute('alter table dxm_engine_info add proxy_user varchar(60)')
self.set_database_version(1)
else:
self.__cursor.execute(sql_dxm_engine_info)
self.set_database_version(1)
except lite.Error as e:
self.__logger.debug("Error %s" % e.args[0])
print_error("Error %s" % e.args[0])
sys.exit(-1)
def update(self):
"""
Update file field data to Masking engine and print status message
return a None if non error
return 1 in case of error
"""
if (self.obj.file_field_metadata_id is None):
print_error("file_field_metadata_id is required")
self.__logger.error("file_field_metadata_id is required")
return 1
try:
if self.obj.date_format == '':
self.date_format = None
self.__logger.debug("create field input %s" % str(self))
api_instance = self.__api(self.__engine.api_client)
response = api_instance.update_file_field_metadata(
self.obj.file_field_metadata_id, self.obj)
self.__logger.debug("field response %s" % str(response))
print_message("Field %s updated" % self.obj.field_name)
return None
except self.__apiexc as e:
print_error(e.body)
self.__logger.error(e)
return 1
def LoadDomains(self):
"""
Load list of domains
Return None if OK
"""
self.__domainList.clear()
try:
api_instance = DomainApi(self.__engine.api_client)
domain_list = paginator(api_instance, "get_all_domains")
if domain_list.response_list:
for c in domain_list.response_list:
dom = DxDomain(self.__engine)
dom.from_domain(c)
self.__domainList[c.domain_name] = dom
else:
print_error("No domain found")
self.__logger.error("No domain found")
return 1
return None
except ApiException as e:
print_error(e.body)
self.__logger.error(e.body)
return 1
def user_delete(p_engine, username, force):
"""
Delete user from Engine
param1: p_engine: engine name from configuration
param2: username: user name to delete
param3: force: force deletion of user with admin privilege
return 0 if user deleted
"""
ret = 0
logger = logging.getLogger()
enginelist = get_list_of_engines(p_engine)
if enginelist is None:
return 1
for engine_tuple in enginelist:
engine_obj = DxMaskingEngine(engine_tuple)
if engine_obj.get_session():
continue
userlist = DxUserList()
userref = userlist.get_userId_by_name(username)
if userref is not None:
if userlist.delete(userref, force) is not None:
print_error("Problem with deleting user %s" % username)
logger.debug("Problem with deleting user %s" % username)
ret = ret + 1
continue
else:
print_error("User %s not found" % username)
logger.debug("User %s not found" % username)
def ruleset_export(p_engine, rulesetname, envname, outputfile, exportmeta,
metaname):
"""
Delete ruleset from Masking engine
param1: p_engine: engine name from configuration
param2: rulesetname: ruleset name
param3: envname: environment name
param4: exportmeta: export metadata with ruleset
param5: metaname: limit export to single meta object
return 0 if added, non 0 for error
"""
exp = []
ret = ruleset_worker(p_engine=p_engine,
rulesetname=rulesetname,
envname=envname,
function_to_call="do_export",
exportout=exp,
exportmeta=exportmeta,
metaname=metaname)
if ret == 0:
try:
json.dump(exp, outputfile, indent=4)
outputfile.close()
print_message("Ruleset exported to file %s" % outputfile.name)
return 0
except Exception as e:
print_error("Problem with file %s Error: %s" %
(outputfile.name, str(e)))
return 1
else:
return 1
def engine_add(p_engine, p_ip, p_username, p_password, p_protocol, p_port,
p_default, p_proxyurl, p_proxyuser, p_proxypassword):
"""
Add engine to a configuration
param1: p_engine: name of Masking engine
param2: p_ip: IP of Masking engine
param3: p_username: username
param4: p_password: password
param5: p_protocol: protocol (http/https)
param6: p_port: port
param7: p_default: is engine default - Y/N - default value N
param8: p_proxyurl: Proxy URL
param9: p_proxyuser: proxy username
param10: p_proxypassword: proxy password
return None if OK or integer with error
"""
config = DxConfig()
config.init_metadata()
if config.insert_engine_info(p_engine, p_ip, p_username, p_password,
p_protocol, p_port, p_default, p_proxyurl,
p_proxyuser, p_proxypassword):
print_error("Problem with adding engine to database")
config.close()
return -1
else:
print_message("Engine added to configuration")
config.close()
return None
def do_check(**kwargs):
"""
Compare
"""
ruleref = kwargs.get('ruleref')
rulelist = kwargs.get('rulelist')
envlist = DxEnvironmentList
envname = kwargs.get('envname')
ruleset = kwargs.get('ruleset')
rulesetname = kwargs.get('rulesetname')
p_engine = kwargs.get('p_engine')
connname = ruleset["Connector name"]
ruleobj = rulelist.get_by_ref(ruleref)
connobj = DxConnectorsList.get_by_ref(ruleobj.connectorId)
if connobj:
envobj = envlist.get_by_ref(connobj.environment_id)
connector_name = connobj.connector_name
environment_name = envobj.environment_name
else:
connector_name = 'N/A'
environment_name = 'N/A'
retcol = 0
metalist = DxMetaList()
metalist.LoadMeta(ruleobj.ruleset_id)
rettab = 0
for meta in ruleset["Metadata"]:
metalist_ref = metalist.get_MetadataId_by_name(meta["meta_name"], 1)
if metalist_ref:
rettab = rettab + 1
else:
print_error("Missing meta %s" % meta["meta_name"])
for col in ruleset["Columns"]:
count = [
x for x in ruleset["Columns"]
if col["Metadata name"] == x["Metadata name"]
]
rc = column_check(p_engine, rulesetname, envname, col, len(count))
if rc != 0:
retcol = retcol + 1
if (ruleobj.ruleset_name == rulesetname) and \
(connector_name == connname) and \
(environment_name == envname) and \
(retcol == 0) and \
(rettab == len(ruleset["Metadata"])):
print_message("Ruleset definition in engine is matching import file")
return 0
else:
print_error("There are difference between engine and import file")
return 1
def update(self):
"""
Update connector on engine
Return None if OK
"""
api_instance = DatabaseConnectorApi(self.__engine.api_client)
body = DatabaseConnector()
for k in self.attribute_map.keys():
if getattr(self, k) is not None:
setattr(body, k, getattr(self, k))
try:
self.__logger.debug("update connector input %s" % str(self))
response = api_instance.update_database_connector(
self.database_connector_id,
body,
_request_timeout=self.__engine.get_timeout())
self.__logger.debug("update connector response %s" % str(response))
self.database_connector_id = response.database_connector_id
print_message("Connector %s updated" % self.connector_name)
except ApiException as e:
print_error(e.body)
self.__logger.error(e)
return 1
def cancel(self):
"""
Cancel running job in Engine
return a 0 if non error
return 1 in case of error
"""
try:
execid = self.__lastExec.execution_id
exec_api = self.__apiexec(self.__engine.api_client)
self.__logger.debug("Stopping execution %s" % str(execid))
execjob = exec_api.cancel_execution(execid)
self.__logger.debug("Stopping execution response %s" %
str(execjob))
while execjob.status == 'RUNNING':
time.sleep(1)
execjob = exec_api.get_execution_by_id(execid)
print_message(execjob)
return 0
except self.__apiexc as e:
print_error(e.body)
self.__logger.error(e)
return 1
def list_execution_component(self, execid):
"""
List an execution detalis ( tables, rows, etc)
:param1 execid: execution id to display
return a None if non error
return 1 in case of error
"""
if (execid is None):
print_error("Execution id is required")
self.__logger.error("Execution id is required")
return 1
try:
self.__logger.debug("execute component")
api_instance = self.__apicomponent(self.__engine.api_client)
execomponents = paginator(
api_instance,
"get_all_execution_components",
execution_id=execid,
_request_timeout=self.__engine.get_timeout())
return execomponents.response_list
except self.__apiexc as e:
print_error(e.body)
self.__logger.error(e)
return None
def LoadApplications(self):
"""
Load a list of applications into memory
return None if OK
return 1 if not OK
"""
self.__applicationList.clear()
try:
self.__api = ApplicationApi
self.__apiexc = ApiException
api_instance = self.__api(self.__engine.api_client)
a = paginator(
api_instance,
"get_all_applications",
_request_timeout=self.__engine.get_timeout())
if a.response_list:
for c in a.response_list:
application = DxApplication(self.__engine)
application.from_obj(c)
if self.__engine.version_ge("6.0.0.0") and c.application_id is not None:
self.__applicationList[c.application_id] = application
else:
self.__applicationList[c.application_name] = application
else:
print_error("No applications found")
return 1
except self.__apiexc as e:
print_error(e.body)
self.__logger.error(e.body)
return 1
def LoadDrivers(self):
"""
Load list of drivers
Return None if OK
"""
self.__api = JdbcDriverApi
self.__apiexc = ApiException
try:
api_instance = self.__api(self.__engine.api_client)
drivers = paginator(
api_instance,
"get_all_jdbc_drivers")
if drivers.response_list:
for c in drivers.response_list:
driver = DxJDBC(self.__engine)
driver.from_driver(c)
self.__driverList[c.jdbc_driver_id] = driver
else:
print_error("No JDBC drivers found")
self.__logger.error("No JDBC drivers found")
except self.__apiexc as e:
print_error("Can't load JDBC drivers %s" % e.body)
return None
def test(self):
"""
Test connector connection to database
Return None if OK
"""
api_instance = self.__api(self.__engine.api_client)
try:
self.__logger.debug("test connector id %s" % self.connectorId)
if (self.__engine.version_ge('6.0.0')):
response = api_instance.test_file_connector(
self.connectorId,
body=self.obj,
_request_timeout=self.__engine.get_timeout())
else:
response = api_instance.test_file_connector(
self.connectorId,
_request_timeout=self.__engine.get_timeout())
if response.response == "Connection Failed":
print_error("Connector test %s failed" % self.connector_name)
return 1
else:
print_message("Connector test %s succeeded" %
self.connector_name)
return 0
except self.__apiexc as e:
print_error(e.body)
self.__logger.error(e)
return 1
def get_properties(self):
"""
Return dict with properties required for connector type
"""
props = {
'port': self.port,
'host': self.host,
'username': self.username,
'schema_name': self.schema_name,
'connector_name': self.connector_name,
'password': self.password,
'database_type': self.database_type,
'environment_id': self.environment_id,
'database_name': self.database_name
}
empty = 0
for k in props.keys():
if (props[k] is None):
print_error("Property %s can't be empty" % k)
empty = empty + 1
if empty == 0:
return props
else:
return None
def add(self):
"""
Add connector to engine
Return 0 if OK
"""
payload_dict = self.get_properties()
if payload_dict is None:
print_error('Some required property not found')
return 1
try:
self.__logger.debug("create connector input %s" % str(self))
api_instance = DatabaseConnectorApi(self.__engine.api_client)
response = api_instance.create_database_connector(
self, _request_timeout=self.__engine.get_timeout())
self.database_connector_id = response.database_connector_id
self.__logger.debug("connector response %s" % str(response))
print_message("Connector %s added" % self.connector_name)
return 0
except ApiException as e:
print_error(e.body)
self.__logger.error(e)
return 1
def export(self, name, path=None):
"""
Export algorithm into file
:param path: path to save algorithm
"""
api_sync = SyncApi(self.__engine.api_client)
self.__logger.debug("Export input %s" % self)
export_list = []
export_list.append(self)
api_response = api_sync.export(export_list)
self.__logger.debug("Export response (without blob) %s" %
str(api_response.export_response_metadata))
print(path)
filename = os.path.join(path, '{0}.bin'.format(name))
print(filename)
self.__logger.debug("saving to %s" % filename)
dependencylist = [
b["objectType"] for b in
api_response.export_response_metadata["exportedObjectList"]
]
try:
binary_file = open(filename, mode='wb')
pickle.dump(api_response, binary_file)
binary_file.close()
print_message("Exported object types: ")
print_message(",".join(dependencylist))
print_message("Object saved to %s" % filename)
return 0
except Exception as e:
self.__logger.debug("Problem with saving object to %s" % filename)
self.__logger.debug(str(e))
print_error("Problem with saving object to %s" % filename)
print_error(str(e))
return 1
def LoadAppSettings(self):
"""
Load application settings list from Engine into global group list
return None if OK
return 1 if error
"""
self.__appSettingGroupList.clear()
try:
api_instance = ApplicationSettingsApi(self.__engine.api_client)
applist = paginator(api_instance,
"get_all_application_settings",
_request_timeout=self.__engine.get_timeout())
if applist.response_list:
for c in applist.response_list:
setting = DxAppSetting(self.__engine)
setting.from_role(c)
if c.setting_group not in self.__appSettingGroupList:
self.__appSettingGroupList[c.setting_group] = {}
self.__appSettingGroupList[c.setting_group][
c.setting_name] = setting
else:
self.__logger.error("No application settings found")
print_error("No application settings found")
except ApiException as e:
self.__logger.error("Can't load application settings %s" % e.body)
print_error("Can't load application settings %s" % e.body)
return 1
def LoadUsers(self):
"""
Load users list from Engine into global list
return None if OK
return 1 if error
"""
self.__api = UserApi
self.__apiexc = ApiException
self.__userList.clear()
try:
api_instance = self.__api(self.__engine.api_client)
userlist = paginator(api_instance,
"get_all_users",
_request_timeout=self.__engine.get_timeout())
if userlist.response_list:
for c in userlist.response_list:
user = DxUser(self.__engine)
user.from_user(c)
self.__userList[c.user_id] = user
else:
self.__logger.error("No users found")
print_error("No users found")
except self.__apiexc as e:
self.__logger.error("Can't load users %s" % e.body)
print_error("Can't load users %s" % e.body)
return 1
def wait_for_task(self):
"""
"""
api_instance = self.__api(self.__engine.api_client)
try:
running = True
while (running):
self.__logger.debug("wait async input %s" % str(self))
response = api_instance.get_async_task(
self.obj.async_task_id,
_request_timeout=self.__engine.get_timeout())
self.__logger.debug("wait async response %s" % str(response))
if response.status != "RUNNING":
running = False
sleep(1)
print_message("Waiting for task %s to complete " %
self.obj.async_task_id)
if response.status == "SUCCEEDED":
print_message("Task finished sucesfully")
return 0
else:
print_error("Task finished with status %s" % response.status)
return 1
except self.__apiexc as e:
print_error(e.body)
self.__logger.error(e)
return 1
def do_print_meta(**kwargs):
connref = kwargs.get('connref')
connlist = kwargs.get('connlist')
engine_obj = kwargs.get('engine_obj')
format = kwargs.get('format')
connobj = connlist.get_by_ref(connref)
if connobj.is_database:
metaname = 'Table name'
else:
metaname = 'File name'
data = DataFormatter()
data_header = [("Engine name", 30), ("Connector name", 30), (metaname, 40)]
data.create_header(data_header)
data.format_type = format
metalist = connobj.fetch_meta()
if metalist:
for meta_item in metalist:
data.data_insert(engine_obj.get_name(), connobj.connector_name,
meta_item)
else:
print_error("List of tables/files is empty")
return 1
print("")
print(data.data_output(False))
print("")
return 0
def add(self):
"""
Add table to Masking engine and print status message
return 0 if non error
return 1 in case of error
"""
if (self.table_name is None):
print "Table name is required"
self.__logger.error("Table name is required")
return 1
if (self.ruleset_id is None):
print "ruleset_id is required"
self.__logger.error("ruleset_id is required")
return 1
try:
self.__logger.debug("create table input %s" % str(self))
api_instance = TableMetadataApi(self.__engine.api_client)
self.__logger.debug("API instance created")
response = api_instance.create_table_metadata(self)
self.table_metadata_id = response.table_metadata_id
self.__logger.debug("table response %s" % str(response))
print_message("Table %s added" % self.table_name)
return 0
except ApiException as e:
print_error(e.body)
self.__logger.error(e)
return 1
def LoadRoles(self):
"""
Load roles list from Engine into global list
return None if OK
return 1 if error
"""
self.__roleList.clear()
try:
api_instance = RoleApi(self.__engine.api_client)
rolelist = paginator(api_instance,
"get_all_roles",
_request_timeout=self.__engine.get_timeout())
if rolelist.response_list:
for c in rolelist.response_list:
role = DxRole(self.__engine)
role.from_role(c)
self.__roleList[c.role_id] = role
else:
self.__logger.error("No roles found")
print_error("No roles found")
except ApiException as e:
self.__logger.error("Can't load roles %s" % e.body)
print_error("Can't load roles %s" % e.body)
return 1
def add(self):
"""
Add ruleset to Masking engine and print status message
return a None if non error
return 1 in case of error
"""
if (self.ruleset_name is None):
print "Ruleset name is required"
self.__logger.error("Ruleset name is required")
return 1
if (self.file_connector_id is None):
print "File connector Id is required"
self.__logger.error("File connector ID is required")
return 1
try:
self.__logger.debug("create database ruleset input %s" % str(self))
api_instance = FileRulesetApi(self.__engine.api_client)
response = api_instance.create_file_ruleset(self)
self.file_ruleset_id = response.file_ruleset_id
self.__logger.debug("ruleset response %s" % str(response))
print_message("Ruleset %s added" % self.ruleset_name)
except ApiException as e:
print_error(e.body)
self.__logger.error(e)
return 1
def decrypt_password(self, password):
"""
Encrypt password and hash with SHA265
:param password: password
return encryted password
"""
hashhex = password[-64:]
enc = password[0:-64]
try:
dehex = bytes.fromhex(enc)
msg_nonce = dehex[:12]
ciphertext = dehex[12:]
cipher = ChaCha20Poly1305(secret)
password = cipher.decrypt(msg_nonce, ciphertext, None)
except InvalidTag:
print_error("Wrong encryption key")
sys.exit(1)
hash_object = hashes.Hash(hashes.SHA256(), backend=default_backend())
hash_object.update(password)
dhash = hash_object.finalize()
hashhex_check = dhash.hex()
if hashhex == hashhex_check:
return password.decode('utf-8')
else:
print_error("Password SHA265 value after decrypt is wrong")
return None
def get_session(self):
"""
Create a session with a Masking engine
:return autorization key for a session
"""
if "masking_api_53.api" in sys.modules.keys():
is_api5 = True
version = 5
else:
is_api5 = False
if "masking_api_60.api" in sys.modules.keys():
is_api6 = True
version = 6
else:
is_api6 = False
self.get_session_worker(version=version)
if self.get_version()<"6.0.0.0":
if is_api5:
self.__logger.debug("Change API from 6 to 5")
self.get_session_worker(version=5)
else:
print_error("Delphix Engine version is not supported by compiled API. Please generate SDK with proper version")
sys.exit(1)
if self.get_version()>="6.0.0.0" and is_api6 == False:
print_error("Delphix Engine version is not supported by compiled API. Please generate SDK with proper version")
sys.exit(1)
def get_connectorId_by_name_worker(self, name, check_uniquness, verbose):
"""
Get a list of connectors by name
:param1 name: name of connector
:param2 check_uniqueness: check uniqueness put None if skip this check
:param3 verbose: set if output should be printed
return list of connectors
"""
connectors = get_objref_by_val_and_attribute(name, self,
'connector_name')
if len(connectors) < 1:
if verbose:
print_error("Connector %s not found" % name)
self.__logger.error("Connector %s not found " % name)
return None
if check_uniquness:
if len(connectors) > 1:
if verbose:
print_error("Connector name %s is not unique" % name)
self.__logger.error("Connector %s is not unique" % name)
return None
return connectors
def delete(self, force):
"""
Delete user from Masking engine and print status message
:param force: if True, change user to non-admin and delete
return a None if non error
return 1 in case of error
"""
if self.is_admin and force:
self.is_admin = False
nap = self.__modelnap(environment_ids=[], role_id=1)
self.non_admin_properties = nap
if self.update() != 0:
print_error("Can't switch to non-admin in force mode")
self.__logger.debug("Can't switch to non-admin in force mode")
return 1
api_instance = self.__api(self.__engine.api_client)
try:
self.__logger.debug("delete user id %s" % self.user_id)
response = api_instance.delete_user_by_id(
self.user_id, _request_timeout=self.__engine.get_timeout())
self.__logger.debug("delete user response %s" % str(response))
print_message("User %s deleted" % self.user_name)
return 0
except self.__apiexc as e:
print_error(e.body)
self.__logger.error(e)
return 1
def LoadFileFormats(self):
"""
Load list of rule sets
Return None if OK
"""
if (self.__engine.version_ge('6.0.0')):
from masking_api_60.api.file_format_api import FileFormatApi
from masking_api_60.rest import ApiException
else:
from masking_api_53.api.file_format_api import FileFormatApi
from masking_api_53.rest import ApiException
self.__api = FileFormatApi
self.__apiexc = ApiException
try:
api_instance = self.__api(self.__engine.api_client)
fileformats = paginator(
api_instance,
"get_all_file_formats")
if fileformats.response_list:
for c in fileformats.response_list:
fileformat = DxFileFormat(self.__engine)
fileformat.from_filetype(c)
self.__filetypeList[c.file_format_id] = fileformat
else:
print_error("No file formats found")
self.__logger.error("No file formats found")
except self.__apiexc as e:
print_error("Can't load file formats %s" % e.body)
return None
def LoadProfileExt(self):
"""
Load list of profiles expressions
Return None if OK
"""
try:
api_instance = ProfileExpressionApi(self.__engine.api_client)
profileexts = paginator(
api_instance,
"get_all_profile_expressions",
_request_timeout=self.__engine.get_timeout())
if profileexts.response_list:
for pe in profileexts.response_list:
proext = DxProfileExt()
proext.from_profilesetext(pe)
self.__profileexp_list[pe.profile_expression_id] = proext
else:
print_error("No Profile expression found")
self.__logger.error("No Profile expression found")
self.__logger.debug("All Profile expression loaded")
except ApiException as e:
print_error("Can't load Profile expression list %s" % e.body)
return 1
