def ruleset_add(p_engine, rulesetname, connectorname, envname): """ Add ruleset to Masking engine param1: p_engine: engine name from configuration param2: rulesetname: ruleset name param3: connectorname: connectorname name param4: envname: environment name return 0 if added, non 0 for error """ ret = 0 logger = logging.getLogger() enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue envlist = DxEnvironmentList() envlist.LoadEnvironments() rulelist = DxRulesetList() connlist = DxConnectorsList() connlist.LoadConnectors(envname) logger.debug("Connector is %s" % connectorname) connref = connlist.get_connectorId_by_name(connectorname) connobj = connlist.get_by_ref(connref) if connobj: if connobj.is_database: ruleset = DxDatabaseRuleset(engine_obj) ruleset.ruleset_name = rulesetname ruleset.database_connector_id = connobj.connectorId else: ruleset = DxFileRuleset(engine_obj) ruleset.ruleset_name = rulesetname ruleset.file_connector_id = connobj.connectorId if rulelist.add(ruleset): ret = ret + 1 else: ret = ret + 1 return ret
def expression_worker(**kwargs): """ Expression worker - run a dynamic action on expression object p_engine: engine name from configuration expname: expression name to list function_to_call: function name to call for every expression object kwargs: passed to dynamic action return sum of return codes of dynamic action run on all selected expressions """ p_engine = kwargs.get('p_engine') expname = kwargs.get('expname') function_to_call = kwargs.get('function_to_call') enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 ret = 0 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue # load all objects profileexplist = DxProfileExpList() expressions = [] if expname is None: expressions = profileexplist.get_allref() else: expression = profileexplist.get_profileExpId_by_name(expname) expressions.append(expression) if expression is None: ret = ret + 1 continue for peref in expressions: proexpobj = profileexplist.get_by_ref(peref) dynfunc = globals()[function_to_call] ret = ret + dynfunc(proexpobj=proexpobj, profileexplist=profileexplist, engine_obj=engine_obj, **kwargs) return ret
def ruleset_addmeta(p_engine, params, inputfile): """ Add matadata to Masking engine param1: p_engine: engine name from configuration param2: params: set of required parameters to add meta param3: inputfile: file with table/file definition return 0 if added, non 0 for error """ ret = 0 rulesetname = params["rulesetname"] envname = params["envname"] enginelist = get_list_of_engines(p_engine) if (params["metaname"] is None) and (inputfile is None): print_error("Option metaname or inputfile is required") return 1 if (params["metaname"]) and (inputfile): print_error("Option metaname and inputfile are mutally exclusive") return 1 if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue envlist = DxEnvironmentList() envlist.LoadEnvironments() rulelist = DxRulesetList() rulelist.LoadRulesets(envname) ruleref = rulelist.get_rulesetId_by_name(rulesetname) if ruleref: ruleobj = rulelist.get_by_ref(ruleref) if (params["metaname"]): ret = ret + ruleobj.addmeta(params) else: ret = ret + ruleobj.addmetafromfile(inputfile) else: ret = ret + 1 return ret
def application_list(p_engine, format, appname): """ Print list of applications param1: p_engine: engine name from configuration param2: format: output format param3: appname: application name to list, all if None return 0 if application found """ ret = 0 enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 data = DataFormatter() data_header = [ ("Engine name", 30), ("Application name", 30), ] data.create_header(data_header) data.format_type = format for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue applist = DxApplicationList() # load all objects applist.LoadApplications() if appname is None: applications = applist.get_allref() else: applications = applist.get_applicationId_by_name(appname) if len(applications) == 0: ret = ret + 1 for appref in applications: appobj = applist.get_by_ref(appref) data.data_insert( engine_tuple[0], appobj.application_name ) print("") print (data.data_output(False)) print("") return ret
def sync_import(p_engine, p_username, envname, inputfile, inputpath, force): """ Load algorithm from file param1: p_engine: engine name from configuration param2: inputfile: input file param3: force: overwrite object return 0 if OK """ ret = 0 enginelist = get_list_of_engines(p_engine, p_username) if inputfile is None and inputpath is None: print_error("Inputfile or inputpath parameter is required") return 1 list_of_opened_files = [] if inputpath: for f in os.listdir(inputpath): fullpath = os.path.join(inputpath, f) if os.path.isfile(fullpath): fh = open(fullpath, "rb") list_of_opened_files.append(fh) else: if inputfile: list_of_opened_files = [inputfile] if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue envlist = DxEnvironmentList() if envname: environment_id = envlist.get_environmentId_by_name(envname) else: environment_id = None for i in list_of_opened_files: syncobj = DxSync(engine_obj) ret = ret + syncobj.importsync(i, environment_id, force) return ret
def environment_list(p_engine, format, envname): """ Print list of environments param1: p_engine: engine name from configuration param2: format: output format param3: envname: environemnt name to list, all if None return 0 if environment found """ ret = 0 enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 data = DataFormatter() data_header = [("Engine name", 30), ("Environment name", 30), ("Application name", 30)] data.create_header(data_header) data.format_type = format for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple[0], engine_tuple[1], engine_tuple[2], engine_tuple[3]) if engine_obj.get_session(): continue envlist = DxEnvironmentList() # load all objects envlist.LoadEnvironments() if envname is None: environments = envlist.get_allref() else: environment = envlist.get_environmentId_by_name(envname) if environment is None: ret = ret + 1 continue environments = [environment] for envref in environments: envobj = envlist.get_by_ref(envref) data.data_insert(engine_tuple[0], envobj.environment_name, envobj.application) print("") print(data.data_output(False)) print("") return ret
def job_selector(**kwargs): """ Select unique job from Masking engine and run function on it param1: p_engine: engine name from configuration param2: jobname: job name param3: envname: environment name param4: function_to_call: name of function to call on connector return 0 if added, non 0 for error """ p_engine = kwargs.get('p_engine') jobname = kwargs.get('jobname') envname = kwargs.get('envname') function_to_call = kwargs.get('function_to_call') joblist_class = kwargs.get('joblist_class') ret = 0 enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue envlist = DxEnvironmentList() envlist.LoadEnvironments() joblist = globals()[joblist_class]() joblist.LoadJobs(envname) jobref = joblist.get_jobId_by_name(jobname) if jobref: dynfunc = globals()[function_to_call] ret = ret + dynfunc(jobref=jobref, engine_obj=engine_obj, joblist=joblist, **kwargs) else: ret = ret + 1 continue return ret
def connector_selector(p_engine, connectorname, envname, function_to_call, format='fixed'): """ Select unique connector from Masking engine and run function on it param1: p_engine: engine name from configuration param2: connectorname: connectorname name param3: envname: environment name param4: function_to_call: name of function to call on connector param5: format: format of output, set to fixed return 0 if added, non 0 for error """ ret = 0 enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue envlist = DxEnvironmentList() envlist.LoadEnvironments() connlist = DxConnectorsList() connlist.LoadConnectors(envname) connref = connlist.get_connectorId_by_name(connectorname) if connref: dynfunc = globals()[function_to_call] ret = ret + dynfunc(connref=connref, engine_obj=engine_obj, connlist=connlist, format=format) else: ret = ret + 1 continue return ret
def role_list(p_engine, p_username, format, rolename): """ Print list of roles param1: p_engine: engine name from configuration param2: format: output format param3: rolename: role name to list, all if None return 0 if role found """ ret = 0 enginelist = get_list_of_engines(p_engine, p_username) if enginelist is None: return 1 data = DataFormatter() data_header = [("Engine name", 30), ("Role name", 30)] data.create_header(data_header) data.format_type = format for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue rolelist = DxRoleList() # load all objects if rolename is None: roles = rolelist.get_allref() else: role = rolelist.get_roleId_by_name(rolename) if role is None: ret = ret + 1 continue roles = [role] for roleref in roles: roleobj = rolelist.get_by_ref(roleref) data.data_insert(engine_tuple[0], roleobj.role_name) print("") print(data.data_output(False)) print("") return ret
def profile_add(p_engine, p_username, profilename, expname, description): """ Add profile to engine param1: p_engine: engine name from configuration param2: profilename: profile name param3: expname: expression list names param4: description: profile description return 0 if profile was added """ ret = 0 logger = logging.getLogger() enginelist = get_list_of_engines(p_engine, p_username) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue # load all objects profilelist = DxProfilesList() probj = DxProfile() explist = probj.set_expressions_using_names(expname) if explist: probj.create_profile( profile_set_name = profilename, created_by = engine_obj.get_username(), description = description, profile_expression_ids=explist ) if profilelist.add(probj): ret = ret + 1 else: ret = ret + 1 return ret
def ruleset_worker(**kwargs): """ Delete ruleset from Masking engine param1: p_engine: engine name from configuration param2: rulesetname: ruleset name param3: function_to_call: function to call for particual ruleset return 0 if added, non 0 for error """ p_engine = kwargs.get('p_engine') rulesetname = kwargs.get('rulesetname') envname = kwargs.get('envname') function_to_call = kwargs.get('function_to_call') ret = 0 enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple[0], engine_tuple[1], engine_tuple[2], engine_tuple[3]) if engine_obj.get_session(): continue # envlist = DxEnvironmentList() # envlist.LoadEnvironments() # rulelist = DxRulesetList() rulelist = DxRulesetList(envname) ruleref = rulelist.get_rulesetId_by_name(rulesetname) if ruleref: dynfunc = globals()[function_to_call] if dynfunc(ruleref=ruleref, rulelist=rulelist, engine_obj=engine_obj, **kwargs): ret = ret + 1 else: ret = ret + 1 return ret
def expression_add(p_engine, expname, domainname, level, regex): """ Add expression to engine param1: p_engine: engine name from configuration param2: expname: expression name param3: domainname: domain name param4: datalevel: data level param5: regex: regular expression return 0 if expression was added """ ret = 0 logger = logging.getLogger() enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue # load all objects profileexplist = DxProfileExpList() peobj = DxProfileExt() if level: if level == 'data': data_level_profiling = True else: data_level_profiling = False else: data_level_profiling = False peobj.create_profile_expression( expression_name=expname, domain_name=domainname, regular_expression=regex, data_level_profiling=data_level_profiling) if profileexplist.add(peobj): ret = ret + 1 return ret
def ruleset_deletemeta(p_engine, rulesetname, metaname, envname): """ Delete meta (file, table) from ruleset to Masking engine param1: p_engine: engine name from configuration param2: rulesetname: ruleset name param3: metaname: metaname to delete param4: envname: environment name return 0 if added, non 0 for error """ ret = 0 enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple[0], engine_tuple[1], engine_tuple[2], engine_tuple[3]) if engine_obj.get_session(): continue envlist = DxEnvironmentList() envlist.LoadEnvironments() rulelist = DxRulesetList() rulelist.LoadRulesets(envname) ruleref = rulelist.get_rulesetId_by_name(rulesetname) metalist = DxMetaList() metalist.LoadMeta(ruleset_id=ruleref) metaref = metalist.get_MetadataId_by_name(metaname) if metaref: if metalist.delete(metaref): ret = ret + 1 else: ret = ret + 1 return ret
def domain_update(p_engine, p_username, domain_name, classification, default_algname): """ Update the domain to the Masking Engine param1: p_engine: engine name from configuration param2: domain_name: domain name param3: classification: domain classification param4: default_algname: default algorithm name return 0 if added, non 0 for error """ ret = 0 enginelist = get_list_of_engines(p_engine, p_username) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue domainlist = DxDomainList() domobj = domainlist.get_domain_by_name(domain_name) if not domobj: print_error("Domain {} not found".format(domain_name)) return 1 # set required properties if classification: domobj.classification = classification if default_algname: domobj.default_algorithm_code = default_algname if domobj.update(): ret = ret + 1 return ret
def algorithm_worker(p_engine, algname, **kwargs): """ Select an algorithm and run action on it param1: p_engine: engine name from configuration param2: algname: algorithm name kwargs: parameters to pass including function name to call return 0 if algname found """ ret = 0 function_to_call = kwargs.get('function_to_call') enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple[0], engine_tuple[1], engine_tuple[2], engine_tuple[3]) if engine_obj.get_session(): continue domainlist = DxDomainList() domainlist.LoadDomains() alglist = DxAlgorithmList() algref_list = [] algobj = alglist.get_by_ref(algname) if algobj is None: ret = ret + 1 continue dynfunc = globals()[function_to_call] if dynfunc(algobj=algobj, engine_obj=engine_obj, **kwargs): ret = ret + 1 return ret
def driver_add(p_engine, p_username, driver_name, driver_class_name, driver_file_name): """ Add application to Masking engine param1: p_engine: engine name from configuration param2: driver_file_name: driver file name return 0 if added, non 0 for error """ ret = 0 enginelist = get_list_of_engines(p_engine, p_username) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue if not feature_support(engine_obj, "5.3.9"): ret = ret + 1 continue files_obj = DxEngineFiles() uploadid = files_obj.upload_file(driver_file_name) driverList = DxJDBCList() driver = DxJDBC(engine_obj) driver.create_driver(driver_name=driver_name, driver_class_name=driver_class_name, file_reference_id=uploadid) if driverList.add(driver): ret = ret + 1 return ret
def algorithm_import(p_engine, p_username, inputfile): """ Load algorithm from file param1: p_engine: engine name from configuration param2: inputfile: input file return 0 if OK """ ret = 0 enginelist = get_list_of_engines(p_engine, p_username) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue algobj = DxAlgorithm(engine_obj) algobj.importalg(None)
def tab_selector(p_engine, rulesetname, envname, metaname, function_to_call, params): """ List details of tables/file from ruleset param1: p_engine: engine name from configuration param2: p_format: output format param3: rulesetname: ruleset name to display metadata from param4: envname: environemnt name to display metadata from param5: metamame: name of table/file to display param6: what - Database/File return 0 if added, non zero for error """ ret = 0 update = False enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple[0], engine_tuple[1], engine_tuple[2], engine_tuple[3]) if engine_obj.get_session(): continue envlist = DxEnvironmentList() envlist.LoadEnvironments() rulelist = DxRulesetList() rulelist.LoadRulesets(envname) if rulesetname: ruleref = rulelist.get_rulesetId_by_name(rulesetname) else: ruleref = None metalist = DxMetaList() metalist.LoadMeta(ruleset_id=ruleref) metaref = metalist.get_MetadataId_by_name(metaname) if metaref: metaobj = metalist.get_by_ref(metaref) else: ret = ret + 1 continue param_map = { "custom_sql": "custom_sql", "where_clause": "where_clause", "having_clause": "having_clause", "key_column": "key_column", "file_format": "file_format_id", "file_delimiter": "delimiter", "file_eor": "end_of_record", "file_enclosure": "enclosure", "file_name_regex": "name_is_regular_expression" } eor = params["file_eor"] if eor == 'custom': if params["file_eor_custom"]: params["file_eor"] = params["file_eor_custom"] else: print_error("Custom End of record is unknown") return 1 for p in param_map.keys(): if params[p]: if hasattr(metaobj, param_map[p]): update = True value = params[p] if value == '': value = None setattr(metaobj, param_map[p], value) if update: ret = ret + metaobj.update() return ret
def user_update(p_engine, username, firstname, lastname, email, password, user_type, user_environments, user_role): """ Update user in Engine param1: p_engine: engine name from configuration param2: username: user name to add param3: firstname: user first name to add param4: lastname: user last name to add param5: email: user email to add param6: password: user password to add param7: user_type: user type (admin / nonadmin) param8: user_environments: list of comma separated environments param9: user_role: user role name return 0 if user updated """ ret = 0 update = 0 logger = logging.getLogger() enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue userlist = DxUserList() userref = userlist.get_userId_by_name(username) if userref is None: print_error("User %s not found" % username) logger.debug("User %s not found" % username) ret = ret + 1 continue userobj = userlist.get_by_ref(userref) if user_type is not None: update = 1 if user_type == 'nonadmin': if user_role is None: print_error("User role is required for non-admin user") return 1 rolelist = DxRoleList() roleref = rolelist.get_roleId_by_name(user_role) if roleref is None: print_error("Role name %s not found" % user_role) logger.debug("Role name %s not found" % user_role) ret = ret + 1 continue envreflist = [] if user_environments is not None: envlist = DxEnvironmentList() envnamelist = user_environments.split(',') for envname in envnamelist: envref = envlist.get_environmentId_by_name(envname) if envref is None: ret = ret + 1 return 1 else: envreflist.append(envref) userobj.is_admin = False nap = NonAdminProperties() nap.role_id = roleref nap.environment_ids = envreflist userobj.non_admin_properties = nap else: userobj.is_admin = True userobj.delete_nap() print userobj if firstname is not None: update = 1 userobj.first_name = firstname if lastname is not None: update = 1 userobj.last_name = lastname if email is not None: update = 1 userobj.email = email if password is not None: update = 1 try: userobj.password = password except ValueError as e: print str(e) ret = ret + 1 return ret if update == 1: ret = ret + userobj.update() else: print_error("No values set for update.") ret = ret + 1 return ret
def user_list(p_engine, format, username): """ Print list of users param1: p_engine: engine name from configuration param2: format: output format param3: username: user name to list, all if None return 0 if role found """ ret = 0 enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 data = DataFormatter() data_header = [("Engine name", 30), ("User name", 30), ("First name", 30), ("Last name", 30), ("E-mail", 30), ("Auth type", 10), ("Principal", 30), ("Role name", 30), ("Locked", 6), ("Environment list", 30)] data.create_header(data_header) data.format_type = format for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue userlist = DxUserList() rolelist = DxRoleList() envlist = DxEnvironmentList() # check if ldap is configured appsettingslist = DxAppSettingList() ldapobj = appsettingslist.get_appSetting_by_group_and_name( 'ldap', 'Enable') if ldapobj.setting_value == 'true': msadobj = appsettingslist.get_appSetting_by_group_and_name( 'ldap', 'MsadDomain') else: msadobj = None if username is None: users = userlist.get_allref() else: user = userlist.get_userId_by_name(username) if user is None: ret = ret + 1 continue users = [user] for userref in users: userobj = userlist.get_by_ref(userref) if not userobj.is_admin: roleobj = rolelist.get_by_ref( userobj.non_admin_properties.role_id) if roleobj is not None: rolename = roleobj.role_name else: rolename = 'Not Found' else: rolename = 'Administrator' if userobj.non_admin_properties is not None: if userobj.non_admin_properties.environment_ids: envs = ';'.join([ envlist.get_by_ref(x).environment_name for x in userobj.non_admin_properties.environment_ids ]) else: envs = '' else: envs = '' if userobj.is_locked: locked = 'Locked' else: locked = 'Open' if msadobj is None: authtype = 'NATIVE' principal = '' else: authtype = 'LDAP' principal = '{}@{}'.format(userobj.user_name, msadobj.setting_value) data.data_insert(engine_tuple[0], userobj.user_name, userobj.first_name, userobj.last_name, userobj.email, authtype, principal, rolename, locked, envs) print("") print(data.data_output(False)) print("") return ret
def jobs_report_worker(p_engine, p_username, jobname, envname, p_format, last, startdate, enddate, details, jobtype='masking'): """ Print report of jobs param1: p_engine: engine name from configuration param2: jobname: job name to list param3: envname: environemnt name to list jobs from param4: p_format: output format param5: last: display last job only param6: startdate: filter by start date param7: enddate: filter by end date param8: details param9: joblist_class - DxJobsList, DxProfileJobslist return 0 if environment found """ ret = 0 logger = logging.getLogger() enginelist = get_list_of_engines(p_engine, p_username) if enginelist is None: return 1 data = DataFormatter() if jobtype == 'masking': if details: data_header = [ ("Engine name", 30), ("Environment name", 30), ("Job name", 30), ("ExecId", 6), ("Meta name", 12), ("Masked Rows", 10), ("Started", 20), ("Completed", 20), ("Status", 20), ("Runtime", 20) ] else: data_header = [ ("Engine name", 30), ("Environment name", 30), ("Job name", 30), ("Job Id", 6), ("Min Memory", 10), ("Max Memory", 10), ("Streams", 7), ("On The Fly", 10), ("Ruleset Type", 12), ("ExecId", 6), ("Total Rows", 10), ("Masked Rows", 10), ("Started", 20), ("Completed", 20), ("Status", 20), ("Runtime", 20) ] else: data_header = [ ("Engine name", 30), ("Environment name", 30), ("Job name", 30), ("Ruleset Type", 12), ("ExecId", 6), ("Started", 20), ("Completed", 20), ("Status", 20), ("Runtime", 20) ] data.create_header(data_header) data.format_type = p_format for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue # load all objects envlist = DxEnvironmentList() envlist.LoadEnvironments() rulesetlist = DxRulesetList(envname) connectorlist = DxConnectorsList(envname) if jobtype == 'masking': joblist = DxJobsList() else: joblist = DxProfileJobsList() logger.debug("Envname is %s, job name is %s" % (envname, jobname)) joblist.LoadJobs(envname) #rulesetlist.LoadRulesets(envname) #connectorlist.LoadConnectors(envname) if jobname is None: jobs = joblist.get_allref() else: jobs = joblist.get_all_jobId_by_name(jobname) if jobs is None: ret = ret + 1 continue for jobref in jobs: jobobj = joblist.get_by_ref(jobref) rulesetobj = rulesetlist.get_by_ref(jobobj.ruleset_id) # those test are requierd for 5.X engies where API is not showing all types of connectors if rulesetobj is not None: ruleset_type = rulesetobj.type rulename = rulesetobj.ruleset_name connectorobj = connectorlist.get_by_ref(rulesetobj.connectorId) if connectorobj is not None: connectorname = connectorobj.connector_name envobj = envlist.get_by_ref(connectorobj.environment_id) if envobj is not None: envobjname = envobj.environment_name else: envobjname = "N/A" else: connectorname = "N/A" envobjname = "N/A" else: rulename = "N/A" connectorname = "N/A" envobjname = "N/A" ruleset_type = "N/A" if last: lastonly = True else: lastonly = False if lastonly: execlist = [ jobobj.lastExec ] else: if startdate or enddate: execlist = jobobj.filter_executions(startdate, enddate) else: execlist = jobobj.execList if execlist: for jobexec in execlist: if details == False: if jobtype == 'masking': if jobexec is not None: status = jobexec.status execid = jobexec.execution_id rowsmasked = jobexec.rows_masked rowstotal = jobexec.rows_total if jobexec.start_time is not None: starttime = jobexec.start_time.strftime("%Y-%m-%d %H:%M:%S") else: starttime = 'N/A' if (jobexec.end_time is not None) and \ (jobexec.start_time is not None): endtime = jobexec.end_time \ .strftime("%Y-%m-%d %H:%M:%S") runtimetemp = jobexec.end_time \ - jobexec.start_time runtime = str(runtimetemp) else: endtime = 'N/A' runtime = 'N/A' else: status = 'N/A' endtime = 'N/A' starttime = 'N/A' runtime = 'N/A' execid = 'N/A' rowsmasked = 'N/A' rowstotal = 'N/A' data.data_insert( engine_tuple[0], envobjname, jobobj.job_name, jobobj.masking_job_id, jobobj.min_memory, jobobj.max_memory, jobobj.num_input_streams, jobobj.on_the_fly_masking, ruleset_type, execid, rowstotal, rowsmasked, starttime, endtime, status, runtime ) else: if jobexec is not None: status = jobexec.status execid = jobexec.execution_id if jobexec.start_time is not None: starttime = jobexec.start_time.strftime("%Y-%m-%d %H:%M:%S") else: starttime = 'N/A' if (jobexec.end_time is not None) and \ (jobexec.start_time is not None): endtime = jobexec.end_time \ .strftime("%Y-%m-%d %H:%M:%S") runtimetemp = jobexec.end_time \ - jobexec.start_time runtime = str(runtimetemp) else: endtime = 'N/A' runtime = 'N/A' else: status = 'N/A' endtime = 'N/A' starttime = 'N/A' runtime = 'N/A' execid = 'N/A' data.data_insert( engine_tuple[0], envobjname, jobobj.job_name, ruleset_type, execid, starttime, endtime, status, runtime ) else: # details here if jobexec is not None: execid = jobexec.execution_id complist = jobobj.list_execution_component(execid) if complist is not None: for comp in complist: status = comp.status rowsmasked = comp.rows_masked metaname = comp.component_name if comp.start_time is not None: starttime = comp.start_time.strftime("%Y-%m-%d %H:%M:%S") else: starttime = 'N/A' if (comp.end_time is not None) and \ (comp.start_time is not None): endtime = comp.end_time \ .strftime("%Y-%m-%d %H:%M:%S") runtimetemp = comp.end_time \ - comp.start_time runtime = str(runtimetemp) else: endtime = 'N/A' runtime = 'N/A' data.data_insert( engine_tuple[0], envobjname, jobobj.job_name, execid, metaname, rowsmasked, starttime, endtime, status, runtime ) else: print("setting 1") ret = 1 else: # no executions ret = 1 else: # no executions ret = 1 print("") print (data.data_output(False)) print("") return ret
def jobs_list_worker(p_engine, p_username, jobname, envname, p_format, joblist_class): """ Print list of jobs param1: p_engine: engine name from configuration param2: jobname: job name to list param3: envname: environemnt name to list jobs from param4: p_format: output format param5: joblist_class - DxJobsList, DxProfileJobslist return 0 if environment found """ ret = 0 logger = logging.getLogger() enginelist = get_list_of_engines(p_engine, p_username) if enginelist is None: return 1 data = DataFormatter() data_header = [ ("Engine name", 30), ("Job name", 30), ("Ruleset name", 30), ("Connector name", 30), ("Environment name", 30), ("Completed", 20), ("Status", 20), ("Runtime", 20) ] data.create_header(data_header) data.format_type = p_format for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue # load all objects envlist = DxEnvironmentList() envlist.LoadEnvironments() rulesetlist = DxRulesetList(envname) connectorlist = DxConnectorsList(envname) joblist = globals()[joblist_class]() logger.debug("Envname is %s, job name is %s" % (envname, jobname)) joblist.LoadJobs(envname) #rulesetlist.LoadRulesets(envname) #connectorlist.LoadConnectors(envname) if jobname is None: jobs = joblist.get_allref() else: jobs = joblist.get_all_jobId_by_name(jobname) if jobs is None: ret = ret + 1 continue for jobref in jobs: jobobj = joblist.get_by_ref(jobref) rulesetobj = rulesetlist.get_by_ref(jobobj.ruleset_id) # those test are requierd for 5.X engies where API is not showing all types of connectors if rulesetobj is not None: rulename = rulesetobj.ruleset_name connectorobj = connectorlist.get_by_ref(rulesetobj.connectorId) if connectorobj is not None: connectorname = connectorobj.connector_name envobj = envlist.get_by_ref(connectorobj.environment_id) if envobj is not None: envobjname = envobj.environment_name else: envobjname = "N/A" else: connectorname = "N/A" envobjname = "N/A" else: rulename = "N/A" connectorname = "N/A" envobjname = "N/A" if jobobj.lastExec is not None: status = jobobj.lastExec.status if (jobobj.lastExec.end_time is not None) and \ (jobobj.lastExec.end_time is not None): endtime = jobobj.lastExec.end_time \ .strftime("%Y-%m-%d %H:%M:%S") runtimetemp = jobobj.lastExec.end_time \ - jobobj.lastExec.start_time runtime = str(runtimetemp) else: endtime = 'N/A' runtime = 'N/A' else: status = 'N/A' endtime = 'N/A' runtime = 'N/A' data.data_insert( engine_tuple[0], jobobj.job_name, rulename, connectorname, envobjname, endtime, status, runtime ) print("") print (data.data_output(False)) print("") return ret
def job_add(p_engine, p_username, params): """ Add masking job to Masking engine param1: p_engine: engine name from configuration param2: params: job parameters return 0 if added, non 0 for error """ ret = 0 enginelist = get_list_of_engines(p_engine, p_username) logger = logging.getLogger() envname = params['envname'] jobname = params['jobname'] rulesetname = params['rulesetname'] if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue joblist = DxJobsList() envlist = DxEnvironmentList() envlist.LoadEnvironments() logger.debug("Envname is %s, job name is %s" % (envname, jobname)) rulesetlist = DxRulesetList(envname) rulesetref = rulesetlist.get_rulesetId_by_name(rulesetname) job = DxJob(engine_obj, None) job.create_job(job_name=jobname, ruleset_id=rulesetref) for p in optional_params_list: if params[p] is not None: if params[p] == 'Y': value = True elif params[p] == 'N': value = False else: value = params[p] setattr(job, p, value) dmo = DxDatabaseMaskingOptions() for p in optional_options_list: if params[p] is not None: if params[p] == 'Y': value = True elif params[p] == 'N': value = False else: value = params[p] setattr(dmo, p, value) if params["on_the_fly_masking"] == 'Y' : src_env = params["on_the_fly_src_envname"] src_con = params["on_the_fly_src_connector"] conlist = DxConnectorsList(src_env) conid = conlist.get_connectorId_by_name(src_con) if not conid : return 1 on_the_fly_maskking_srcobj = DxOnTheFlyJob() on_the_fly_maskking_srcobj.connector_id = conid[1:] conObj = conlist.get_by_ref(conid) if conObj.is_database : on_the_fly_maskking_srcobj.connector_type = "DATABASE" else: on_the_fly_maskking_srcobj.connector_type = "FILE" job.on_the_fly_masking_source = on_the_fly_maskking_srcobj if params["prescript"]: scriptname = os.path.basename(params["prescript"].name) prescript = DxMaskingScriptJob(name=scriptname, contents=''.join(params["prescript"].readlines())) dmo.prescript = prescript if params["postscript"]: scriptname = os.path.basename(params["postscript"].name) postscript = DxMaskingScriptJob(name=scriptname, contents = ''.join(params["postscript"].readlines())) dmo.postscript = postscript job.database_masking_options = dmo if joblist.add(job): ret = ret + 1 return ret
def connector_add(p_engine, params): """ Add application to Masking engine param1: p_engine: engine name from configuration param2: params: dict of parameters needed for connector to add return 0 if added, non 0 for error """ ret = 0 logger = logging.getLogger() enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 envname = params['envname'] schemaName = params['schemaName'] host = params['host'] port = params['port'] password = params['password'] username = params['username'] connname = params['connname'] for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue envlist = DxEnvironmentList() envlist.LoadEnvironments() logger.debug("Envname is %s" % envname) envref = envlist.get_environmentId_by_name(envname) if envref is None: ret = ret + 1 continue connlist = DxConnectorsList() if params['type'] in database_types: if params['type'] == 'oracle': connobj = OracleConnector(engine_obj) elif params['type'] == 'mssql': connobj = MSSQLConnector(engine_obj) elif params['type'] == 'sybase': connobj = SybaseConnector(engine_obj) else: connobj = DxConnector(engine_obj) connobj.database_type = params['type'].upper() connobj.connector_name = connname connobj.schema_name = schemaName connobj.username = username connobj.password = password connobj.host = host if port: connobj.port = port + 0 connobj.sid = params['sid'] connobj.jdbc = params['jdbc'] connobj.environment_id = envref connobj.instance_name = params['instancename'] connobj.database_name = params['databasename'] elif params['type'] in file_types: path = params['path'] connmode = params['servertype'] connobj = DxFileConnector(engine_obj) connobj.is_database = False connobj.connector_name = connname connobj.environment_id = envref connobj.file_type = params['type'].upper() ci = ConnectionInfo() ci.host = host ci.port = port ci.login_name = username ci.password = password ci.path = path ci.connection_mode = connmode.upper() connobj.connection_info = ci else: print_error('Wrong connector type %s' % params['type']) logger.error('Wrong connector type %s' % params['type']) return 1 if connlist.add(connobj): ret = ret + 1 return ret
def connector_list(p_engine, format, envname, connector_name, details): """ Print list of connectors param1: p_engine: engine name from configuration param2: format: output format param3: envname: environemnt name filter for connectors param4: connector_name: connector name to list param5: details: print connector details return 0 if connector found """ ret = 0 enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 data = DataFormatter() if details: data_header = [("Engine name", 30), ("Environment name", 30), ("Connector name", 30), ("Connector type", 15), ("Hostname", 30), ("Port", 5), ("Schema name", 30), ("Type depended", 100)] else: data_header = [("Engine name", 30), ("Environment name", 30), ("Connector name", 30), ("Connector type", 15)] data.create_header(data_header) data.format_type = format for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue # connlist = DxConnectorsList() # envlist = DxEnvironmentList() # envlist.LoadEnvironments() # connlist.LoadConnectors(envname) DxConnectorsList(envname) if connector_name is None: connectors = DxConnectorsList.get_allref() else: connectors = DxConnectorsList.get_all_connectorId_by_name( connector_name) if connectors is None: ret = ret + 1 continue for connref in connectors: connobj = DxConnectorsList.get_by_ref(connref) if details: rest = ''.join([ '%s = %s ' % (key, value) for (key, value) in connobj.get_type_properties().items() ]) data.data_insert( engine_tuple[0], DxEnvironmentList.get_by_ref( connobj.environment_id).environment_name, connobj.connector_name, connobj.connector_type, connobj.host, connobj.port, connobj.schema_name, rest) else: data.data_insert( engine_tuple[0], DxEnvironmentList.get_by_ref( connobj.environment_id).environment_name, connobj.connector_name, connobj.connector_type) print("") print(data.data_output(False)) print("") return ret
def connector_update(p_engine, params): """ Update connector from Masking engine param1: p_engine: engine name from configuration param2: connectorname: connectorname name param3: params: dict of parameters needed for connector to add return 0 if added, non 0 for error """ ret = 0 logger = logging.getLogger() enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 connectorname = params['connname'] envname = params['envname'] for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue DxConnectorsList(envname) connref = DxConnectorsList.get_connectorId_by_name(connectorname) if connref is None: ret = ret + 1 continue connobj = DxConnectorsList.get_by_ref(connref) if params['schemaName']: connobj.schema_name = params['schemaName'] if params['jdbc']: connobj.host = None connobj.port = None if hasattr(connobj, 'sid'): connobj.sid = None if hasattr(connobj, 'instance_name'): connobj.instance_name = None if hasattr(connobj, 'database_name'): connobj.database_name = None connobj.jdbc = params['jdbc'] else: connobj.jdbc = None if params['host']: connobj.host = params['host'] if params['port']: connobj.port = params['port'] if hasattr(connobj, 'sid'): if params['sid']: connobj.sid = params['sid'] if hasattr(connobj, 'instance_name'): if params['instancename']: connobj.instance_name = params['instancename'] if hasattr(connobj, 'database_name'): if params['databasename']: connobj.database_name = params['databasename'] if params['password']: connobj.password = params['password'] if params['username']: connobj.username = params['username'] if params['connname']: connobj.connector_name = params['connname'] if connobj.update(): ret = ret + 1 return ret
def fileformat_list(p_engine, format, fileformat_type, fileformat_name): """ Print list of file formats param1: p_engine: engine name from configuration param2: format: output format param3: fileformat_type: file format type param4: fileformat_name: file format name return 0 if environment found """ ret = 0 logger = logging.getLogger() logger.debug("fileformat type %s fileformat name %s" % (fileformat_type, fileformat_name)) enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 data = DataFormatter() data_header = [("Engine name", 30), ("File format type", 30), ("File format name", 30)] data.create_header(data_header) data.format_type = format for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple[0], engine_tuple[1], engine_tuple[2], engine_tuple[3]) if engine_obj.get_session(): continue fileformatList = DxFileFormatList() # load all objects if fileformat_name: fileformats_name = fileformatList.get_all_file_format_id_by_name( fileformat_name) if fileformats_name is None: ret = ret + 1 fileformats_name = [] else: fileformats_name = fileformatList.get_allref() if fileformat_type: fileformats_type = fileformatList.get_all_file_format_id_by_type( fileformat_type) if fileformats_type is None: ret = ret + 1 fileformats_type = [] else: fileformats_type = fileformats_name fileformats = list(set(fileformats_name) & set(fileformats_type)) if fileformats: for fileformatref in fileformats: fileformatobj = fileformatList.get_by_ref(fileformatref) data.data_insert(engine_tuple[0], fileformatobj.file_format_type, fileformatobj.file_format_name) else: if fileformat_type and fileformat_name: ret = ret + 1 print("") print(data.data_output(False)) print("") return ret
def ruleset_listmeta(p_engine, format, rulesetname, envname, metaname): """ List tables/file from ruleset param1: p_engine: engine name from configuration param2: format: output format param3: rulesetname: ruleset name to display metadata from param4: envname: environemnt name to display metadata from param5: metamame: name of table/file to display return 0 if added, non zero for error """ ret = 0 found = False data = DataFormatter() data_header = [ ("Engine name", 30), ("Environent name", 30), ("Ruleset name", 30), ("Metadata type", 15), ("Metadata name", 32) ] data.create_header(data_header) data.format_type = format enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple) if engine_obj.get_session(): continue envlist = DxEnvironmentList() envlist.LoadEnvironments() rulelist = DxRulesetList() rulelist.LoadRulesets(envname) connlist = DxConnectorsList() connlist.LoadConnectors(envname) if rulesetname: rulesetref_list = rulelist.get_all_rulesetId_by_name(rulesetname) if rulesetref_list is None: ret = ret + 1 continue else: rulesetref_list = rulelist.get_allref() if rulesetref_list is None: continue metalist = DxMetaList() for ruleref in rulesetref_list: ruleobj = rulelist.get_by_ref(ruleref) connobj = connlist.get_by_ref(ruleobj.connectorId) if connobj: envobj = envlist.get_by_ref(connobj.environment_id) environment_name = envobj.environment_name else: environment_name = 'N/A' metalist.LoadMeta(ruleobj.ruleset_id) if metaname: metalist_ref = metalist.get_all_MetadataId_by_name(metaname, 1) if metalist_ref is None: ret = ret + 1 continue found = True else: metalist_ref = metalist.get_allref() if metalist_ref is None: continue for metaid in metalist_ref: metaobj = metalist.get_by_ref(metaid) data.data_insert( engine_tuple[0], environment_name, ruleobj.ruleset_name, ruleobj.type, metaobj.meta_name ) print("") print (data.data_output(False)) print("") if found: return 0 else: if metaname: print_error("Table or file %s not found" % metaname) return ret
def tab_list_details(p_engine, p_format, rulesetname, envname, metaname, what): """ List details of tables/file from ruleset param1: p_engine: engine name from configuration param2: p_format: output format param3: rulesetname: ruleset name to display metadata from param4: envname: environemnt name to display metadata from param5: metamame: name of table/file to display param6: what - Database/File return 0 if added, non zero for error """ ret = 0 found = False data = DataFormatter() if what == 'Database': data_header = [("Engine name", 30), ("Environent name", 30), ("Ruleset name", 30), ("Table name", 32), ("Logical key", 32), ("Where clause", 50), ("Custom SQL", 50)] else: data_header = [("Engine name", 30), ("Environent name", 30), ("Ruleset name", 30), ("File name", 32), ("File type", 32), ("File format name", 32), ("Delimiter", 10), ("End of record", 10)] data.create_header(data_header) data.format_type = p_format enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple[0], engine_tuple[1], engine_tuple[2], engine_tuple[3]) if engine_obj.get_session(): continue envlist = DxEnvironmentList() envlist.LoadEnvironments() rulelist = DxRulesetList() rulelist.LoadRulesets(envname) connlist = DxConnectorsList() connlist.LoadConnectors(envname) if rulesetname: rulesetref_all = rulelist.get_all_rulesetId_by_name(rulesetname) if rulesetref_all is None: ret = ret + 1 continue rulesetref_list = [ x for x in rulesetref_all if rulelist.get_by_ref(x).type == what ] if rulesetref_list is None: ret = ret + 1 continue else: if what == 'Database': rulesetref_list = rulelist.get_all_database_rulesetIds() if rulesetref_list is None: continue else: rulesetref_list = rulelist.get_all_file_rulesetIds() if rulesetref_list is None: continue filetypelist = DxFileFormatList() metalist = DxMetaList() for ruleref in rulesetref_list: ruleobj = rulelist.get_by_ref(ruleref) connobj = connlist.get_by_ref(ruleobj.connectorId) if connobj: envobj = envlist.get_by_ref(connobj.environment_id) environment_name = envobj.environment_name else: environment_name = 'N/A' metalist.LoadMeta(ruleobj.ruleset_id) if metaname: metalist_ref = metalist.get_all_MetadataId_by_name(metaname, 1) if metalist_ref is None: ret = ret + 1 continue found = True else: metalist_ref = metalist.get_allref() if metalist_ref is None: continue for metaid in metalist_ref: metaobj = metalist.get_by_ref(metaid) if what == 'Database': data.data_insert(engine_tuple[0], environment_name, ruleobj.ruleset_name, metaobj.meta_name, metaobj.key_column, repr(metaobj.where_clause), repr(metaobj.custom_sql)) else: if metaobj.file_format_id is not None: fileformatobj = filetypelist.get_by_ref( metaobj.file_format_id) fileformatname = fileformatobj.file_format_name else: fileformatname = 'N/A' data.data_insert(engine_tuple[0], environment_name, ruleobj.ruleset_name, metaobj.meta_name, metaobj.file_type, fileformatname, metaobj.delimiter, repr(metaobj.end_of_record)) print("") print(data.data_output(False)) print("") if found: return 0 else: if metaname: print_error("Table %s not found" % metaname) return ret
def ruleset_list_worker(**kwargs): """ Print list of ruleset by ruleset name or environment name param1: p_engine: engine name from configuration param2: format: output format param2: ruleset: name of ruleset to display param3: envname: name of environment to list ruleset return 0 if environment found """ p_engine = kwargs.get('p_engine') format = kwargs.get('format') rulesetName = kwargs.get('rulesetName') envname = kwargs.get('envname') function_to_call = kwargs.get('function_to_call') data = kwargs.get('data') ret = 0 enginelist = get_list_of_engines(p_engine) if enginelist is None: return 1 for engine_tuple in enginelist: engine_obj = DxMaskingEngine(engine_tuple[0], engine_tuple[1], engine_tuple[2], engine_tuple[3]) if engine_obj.get_session(): continue #envlist = DxEnvironmentList() rulelist = DxRulesetList() rulelist.LoadRulesets(envname) if rulesetName is None: rulesets = rulelist.get_allref() if len(rulesets) == 0: ret = ret + 1 continue else: rulesets = rulelist.get_all_rulesetId_by_name(rulesetName) if rulesets is None: ret = ret + 1 continue # connlist = DxConnectorsList(envname) # connlist.LoadConnectors(envname) for ruleid in rulesets: ruleobj = rulelist.get_by_ref(ruleid) connobj = DxConnectorsList.get_by_ref(ruleobj.connectorId) dynfunc = globals()[function_to_call] if dynfunc( ruleobj=ruleobj, connobj=connobj, # envlist=envlist, engine_obj=engine_obj, **kwargs): ret = ret + 1 continue return ret