def cancel(self):
"""
Cancel running job in Engine
return a 0 if non error
return 1 in case of error
"""
try:
execid = self.__lastExec.execution_id
exec_api = ExecutionApi(self.__engine.api_client)
self.__logger.debug("Stopping execution %s" % str(execid))
execjob = exec_api.cancel_execution(execid)
self.__logger.debug("Stopping execution response %s" %
str(execjob))
while execjob.status == 'RUNNING':
time.sleep(1)
execjob = exec_api.get_execution_by_id(execid)
print execjob
return 0
except ApiException as e:
print_error(e.body)
self.__logger.error(e)
return 1
def start(self, target_connector_id, source_connector_id, nowait, posno,
lock):
"""
Start masking job
:param1 target_connector_id: target connector id for multinentant
:param2 source_connector_id: source connector for on the fly job
:param3 wait_for_finish: wait for job to finish
Return 0 if job started and finished OK, or was in nowait
Return 1 if errors
"""
exec_api = ExecutionApi(self.__engine.api_client)
execjob = Execution()
execjob.job_id = self.masking_job_id
if (self.multi_tenant):
# target is mandatory
if target_connector_id:
execjob.target_connector_id = target_connector_id
else:
print_error("Target connector is required for multitenant job")
return 1
if (self.on_the_fly_masking):
if not self.on_the_fly_masking_source:
if source_connector_id:
execjob.source_connector_id = source_connector_id
else:
print_error(
"Source connector is required for on the fly job")
return 1
try:
self.__logger.debug("start job input %s" % str(execjob))
response = exec_api.create_execution(
execjob, _request_timeout=self.__engine.get_timeout())
self.__logger.debug("start job response %s" % str(response))
if nowait:
return 0
else:
return self.wait_for_job(response, posno, lock)
except ApiException as e:
print_error(e.body)
self.__logger.error(e)
lock.acquire()
dxm.lib.DxJobs.DxJobCounter.ret = \
dxm.lib.DxJobs.DxJobCounter.ret + 1
lock.release()
self.__logger.error('return value %s' %
dxm.lib.DxJobs.DxJobCounter.ret)
return 1
def LoadJobs(self, environment_name=None):
"""
Load list of rule sets
Return None if OK
"""
try:
api_instance = MaskingJobApi(self.__engine.api_client)
execapi = ExecutionApi(self.__engine.api_client)
execList = paginator(
execapi,
"get_all_executions")
if execList.response_list:
for e in execList.response_list:
self.__executionList[e.job_id] = e
if environment_name:
environment_id = DxEnvironmentList.get_environmentId_by_name(
environment_name)
if environment_id:
jobs = paginator(
api_instance,
"get_all_masking_jobs",
environment_id=environment_id,
_request_timeout=self.__engine.get_timeout())
else:
return 1
else:
jobs = paginator(
api_instance,
"get_all_masking_jobs",
_request_timeout=self.__engine.get_timeout())
if jobs.response_list:
for c in jobs.response_list:
if c.masking_job_id in self.__executionList:
lastExec = self.__executionList[c.masking_job_id]
else:
lastExec = None
job = DxJob(self.__engine, lastExec)
job.from_job(c)
self.__jobsList[c.masking_job_id] = job
else:
if environment_name is None:
print_error("No jobs found")
self.__logger.error("No jobs found")
self.__logger.debug("All jobs loaded")
except ApiException as e:
print_error("Can't load job list %s" % e.body)
return 1
def wait_for_job(self, execjob, posno, lock):
"""
Wait for job to finish execution
:param1 execjob: Execution job response
Return 0 finished OK
Return 1 if errors
"""
execid = execjob.execution_id
exec_api = ExecutionApi(self.__engine.api_client)
self.__logger.debug('Waiting for profilejob %s to finish' %
self.job_name)
if not self.monitor:
print_message('Waiting for profilejob %s to finish' %
self.job_name)
while execjob.status == 'RUNNING':
time.sleep(10)
execjob = exec_api.get_execution_by_id(execid)
if execjob.status == 'SUCCEEDED':
if not self.monitor:
print_message('Profile job %s finished.' % self.job_name)
else:
self.__logger.debug('Profile job %s finished' % self.job_name)
self.__logger.debug('%s rows masked' % execjob.rows_masked)
return 0
else:
if not self.monitor:
print_error('Problem with profile job %s' % self.job_name)
else:
self.__logger.error('Problem with profile job %s' %
self.job_name)
lock.acquire()
dxm.lib.DxJobs.DxJobCounter.profileret = \
dxm.lib.DxJobs.DxJobCounter.profileret + 1
lock.release()
self.__logger.error('return value %s' %
dxm.lib.DxJobs.DxJobCounter.profileret)
return 1
def wait_for_job(self, execjob, posno, lock):
"""
Wait for job to finish execution
:param1 execjob: Execution job response
Return 0 finished OK
Return 1 if errors
"""
execid = execjob.execution_id
first = True
bar = None
exec_api = ExecutionApi(self.__engine.api_client)
last = 0
self.__logger.debug('Waiting for job %s to start processing rows' %
self.job_name)
if not self.monitor:
print_message('Waiting for job %s to start processing rows' %
self.job_name)
while execjob.status == 'RUNNING':
time.sleep(10)
execjob = exec_api.get_execution_by_id(execid)
if first and (execjob.rows_total is not None):
first = False
if self.monitor and (bar is None):
bar = tqdm(total=execjob.rows_total,
desc=self.job_name,
position=posno,
bar_format="{desc}: {percentage:3.0f}%|{bar}|"
" {n_fmt}/{total_fmt}")
else:
print_message('Job %s is processing rows' % self.job_name)
if execjob.rows_masked is not None:
if self.monitor and (bar is not None):
self.__logger.debug(execjob.rows_masked)
self.__logger.debug(last)
step = execjob.rows_masked - last
# if step == 0:
# step = 1
self.__logger.debug(step)
bar.update(step)
last = execjob.rows_masked
if execjob.status == 'SUCCEEDED':
if not self.monitor:
print_message('Masking job %s finished.' % self.job_name)
print_message('%s rows masked' % (execjob.rows_masked or 0))
else:
if bar:
bar.close()
self.__logger.debug('Masking job %s finished' % self.job_name)
self.__logger.debug('%s rows masked' % execjob.rows_masked)
return 0
else:
if not self.monitor:
print_error('Problem with masking job %s' % self.job_name)
print_error('%s rows masked' % (execjob.rows_masked or 0))
else:
if bar:
bar.close()
self.__logger.error('Problem with masking job %s' %
self.job_name)
self.__logger.error('%s rows masked' % execjob.rows_masked)
lock.acquire()
dxm.lib.DxJobs.DxJobCounter.ret = \
dxm.lib.DxJobs.DxJobCounter.ret + 1
lock.release()
self.__logger.error('return value %s' %
dxm.lib.DxJobs.DxJobCounter.ret)
return 1