def modify_job(self, operation_name: str, cron: str, start_date: datetime = None, end_date: datetime = None):
if cron is None or cron == '':
raise OperationalException("Cron required")
data_operation = self.data_operation_repository.first(IsDeleted=0, Code=operation_name)
if data_operation is None:
raise OperationalException("Data operation not found")
data_operation_jobs = self.data_operation_job_repository.filter_by(IsDeleted=0,
DataOperationId=data_operation.Id).all()
if data_operation_jobs is None or len(data_operation_jobs) == 0:
raise OperationalException("Job not found initialized with cron")
founded_cron_job: DataOperationJob = None
for pdi_job in data_operation_jobs:
if pdi_job.Cron is not None:
founded_cron_job = pdi_job
break
if founded_cron_job is None:
raise OperationalException("Job not found initialized with cron")
if start_date is not None or start_date != '':
start_date = datetime.strptime(start_date, "%Y-%m-%dT%H:%M:%S.%fZ").astimezone()
if end_date is not None or end_date != '':
end_date = datetime.strptime(end_date, "%Y-%m-%dT%H:%M:%S.%fZ").astimezone()
trigger = CronTrigger.from_crontab(cron)
trigger.start_date = start_date
trigger.end_date = end_date
if founded_cron_job.ApSchedulerJob.IsDeleted == 0:
self.job_scheduler.remove_job(job_id=founded_cron_job.ApSchedulerJob.JobId)
self.data_operation_job_repository.delete_by_id(founded_cron_job.Id)
ap_scheduler_job = self.add_job_with_cron(job_function=JobOperationService.job_start_data_operation,
cron=cron, start_date=start_date, end_date=end_date,
args=(None, data_operation.Id,))
data_operation_job = self.insert_data_operation_job(ap_scheduler_job, data_operation, cron, start_date, end_date)
return data_operation_job
def get_by_name(self, name) -> ConnectorType:
"""
Get
"""
entity = self.connector_type_repository.first(IsDeleted=0, Name=name)
if entity is None:
raise OperationalException("Connector Type Not Found")
return entity
def add_pdi_job_with_cron(self, operation_name: str, cron: str, start_date: datetime = None,
end_date: datetime = None):
if cron is None or cron == '':
raise OperationalException("Cron required")
data_operation = self.data_operation_repository.first(IsDeleted=0, Code=operation_name)
if data_operation is None:
raise OperationalException("Data Operation not found")
self.check_cron_initialized_jobs(data_operation.Id)
if start_date is not None or start_date != '':
start_date = datetime.strptime(start_date, "%Y-%m-%dT%H:%M:%S.%fZ").astimezone()
if end_date is not None or end_date != '':
end_date = datetime.strptime(end_date, "%Y-%m-%dT%H:%M:%S.%fZ").astimezone()
ap_scheduler_job = self.add_job_with_cron(job_function=JobOperationService.job_start_data_operation,
cron=cron, start_date=start_date, end_date=end_date,
args=(None, data_operation.Id,))
data_operation_job = self.insert_data_operation_job(ap_scheduler_job, data_operation, None, start_date, None)
return data_operation_job
def delete(self, id: int):
"""
Delete SecretSourceBasicAuthentication
"""
secret_source_basic_authentication = self.secret_source_basic_authentication_repository.first(Id=id,
IsDeleted=0)
if secret_source_basic_authentication is None:
raise OperationalException("Secret Source Basic Authentication Not Found")
self.secret_source_basic_authentication_repository.delete_by_id(secret_source_basic_authentication.Id)
def delete(self, id: int):
"""
Delete ConnectionSecret
"""
connection_secret = self.connection_secret_repository.first(
IsDeleted=0, Id=id)
if connection_secret is None:
raise OperationalException("Connection Secret Not Found")
self.connection_secret_repository.delete_by_id(id)
self.secret_service.delete_secret(connection_secret.SecretId)
def add_pdi_job_with_date(self, operation_name: str, run_date: datetime = None):
data_operation = self.data_operation_repository.first(IsDeleted=0, Name=operation_name)
if data_operation is None:
raise OperationalException("Data operation not found")
if run_date is not None or run_date != '':
start_date = datetime.strptime(run_date, "%Y-%m-%dT%H:%M:%S.%fZ").astimezone()
else:
start_date = datetime.now().astimezone()
ap_scheduler_job = self.add_job_with_date(job_function=JobOperationService.job_start_data_operation,
run_date=start_date, args=(None, data_operation.Id,))
data_operation_job = self.insert_data_operation_job(ap_scheduler_job, data_operation, None, start_date, None)
return data_operation_job
def update(self, secret_source: SecretSource, user: str, password: str) -> SecretSourceBasicAuthentication:
"""
Update SecretSourceBasicAuthentication
"""
secret_source_basic_authentication = self.secret_source_basic_authentication_repository.first(
IsDeleted=0, SecretSource=secret_source)
if secret_source_basic_authentication is None:
raise OperationalException("Secret Source Basic Authentication Not Found")
secret_source_basic_authentication.User = self.crypto_service.encrypt_code(user.encode()).decode()
secret_source_basic_authentication.Password = self.crypto_service.encrypt_code(password.encode()).decode()
return secret_source
def update_basic_authentication(self, name: str, user: str,
password: str) -> Secret:
"""
Create File connection
"""
secret = self.secret_repository.first(IsDeleted=0, Name=name)
if secret is None:
raise OperationalException("Secret Not Found")
else:
self.secret_source_service.update_basic_authentication(
secret=secret, user=user, password=password)
return secret
def update_basic_authentication(self, secret: Secret, user: str,
password: str) -> SecretSource:
"""
Create File connection
"""
secret_source = self.secret_source_repository.first(IsDeleted=0,
Secret=secret)
if secret_source is None:
raise OperationalException("Secret Source Not Found")
self.secret_source_basic_authentication_service.update(
secret_source=secret_source, user=user, password=password)
return secret_source
def delete_secret(self, id: int):
"""
Delete Database connection
"""
secret = self.secret_repository.first(Id=id, IsDeleted=0)
if secret is None:
raise OperationalException("Secret Not Found")
self.secret_repository.delete_by_id(secret.Id)
if secret.SecretSources is not None:
for secret_source in secret.SecretSources:
self.secret_source_service.delete(id=secret_source.Id)
def delete(self, id: int):
"""
Delete Secret Source
"""
secret_source = self.secret_source_repository.first(Id=id, IsDeleted=0)
if secret_source is None:
raise OperationalException("Secret Source Not Found")
self.secret_source_repository.delete_by_id(secret_source.Id)
if secret_source.SecretSourceBasicAuthentications is not None:
for secret_source_basic_authentication in secret_source.SecretSourceBasicAuthentications:
self.secret_source_basic_authentication_service.delete(
id=secret_source_basic_authentication.Id)
def update(self, connection: Connection, user: str,
password: str) -> ConnectionSecret:
"""
Update ConnectionSecret
"""
connection_secret = self.connection_secret_repository.first(
IsDeleted=0, Connection=connection)
if connection_secret is None:
raise OperationalException("Connection Secret Not Found")
self.secret_service.update_basic_authentication(name=connection.Name,
user=user,
password=password)
return connection_secret
def insert_data_operation(self,
data_operation_model: CreateDataOperationModel,
definition: Definition) -> DataOperation:
"""
Create Data Operation
"""
if self.check_data_operation_name(data_operation_model.Name):
raise OperationalException(
"Name already defined for other data operation")
data_operation = DataOperation(Name=data_operation_model.Name,
Definition=definition)
self.data_operation_repository.insert(data_operation)
if data_operation_model.Contacts is not None and len(
data_operation_model.Contacts) > 0:
for operation_contact in data_operation_model.Contacts:
data_operation_contact = DataOperationContact(
Email=operation_contact.Email,
DataOperation=data_operation)
self.data_operation_contact_repository.insert(
data_operation_contact)
order = 0
for operation_integration in data_operation_model.Integrations:
order = order + 1
data_integration = self.data_integration_service.create_data_integration(
data=operation_integration.Integration, definition=definition)
data_operation_integration = DataOperationIntegration(
Order=order,
Limit=operation_integration.Limit,
ProcessCount=operation_integration.ProcessCount,
DataIntegration=data_integration,
DataOperation=data_operation)
self.data_operation_integration_repository.insert(
data_operation_integration)
self.database_session_manager.commit()
data_operation = self.data_operation_repository.first(
Id=data_operation.Id)
return data_operation
def delete_connection(self, id: int):
"""
Delete Database connection
"""
connection = self.connection_repository.first(Id=id, IsDeleted=0)
if connection is None:
raise OperationalException("Connection Not Found")
self.connection_repository.delete_by_id(connection.Id)
if connection.Database is not None:
self.connection_database_service.delete(id=connection.Database.Id)
if connection.File is not None:
self.connection_file_service.delete(id=connection.File.Id)
if connection.ConnectionSecrets is not None:
for connection_secret in connection.ConnectionSecrets:
self.connection_secret_service.delete(id=connection_secret.Id)
message = f'{connection.Name} connection deleted'
self.sql_logger.info(message)
self.database_session_manager.commit()
def delete_data_operation(self, id: int):
"""
Delete Data operation
"""
data_operation = self.data_operation_repository.first(Id=id,
IsDeleted=0)
if data_operation is None:
raise OperationalException("Data Operation Not Found")
self.data_operation_repository.delete_by_id(data_operation.Id)
check_existing_integrations = self.data_operation_integration_repository.filter_by(
IsDeleted=0, DataOperationId=data_operation.Id)
for existing_integration in check_existing_integrations:
self.data_operation_integration_repository.delete_by_id(
existing_integration.Id)
self.data_integration_service.delete_data_integration(
existing_integration.DataIntegration.Code)
message = f'{data_operation.Name} data operation deleted'
self.sql_logger.info(message)
self.database_session_manager.commit()
def update_data_operation(self,
data_operation_model: UpdateDataOperationModel,
definition: Definition) -> DataOperation:
"""
Update Data Operation
"""
if not self.check_data_operation_name(data_operation_model.Name):
raise OperationalException("Data Operation not found")
data_operation = self.data_operation_repository.first(
Name=data_operation_model.Name)
# insert or update data_integration
data_operation.Definition = definition
if data_operation_model.Contacts is not None and len(
data_operation_model.Contacts) > 0:
for operation_contact in data_operation_model.Contacts:
data_operation_contact = self.data_operation_contact_repository.first(
IsDeleted=0,
DataOperationId=data_operation.Id,
Email=operation_contact.Email)
if data_operation_contact is None:
data_operation_contact = DataOperationContact(
Email=operation_contact.Email,
DataOperation=data_operation)
self.data_operation_contact_repository.insert(
data_operation_contact)
check_existing_contacts = self.data_operation_contact_repository.filter_by(
IsDeleted=0, DataOperationId=data_operation.Id).all()
for existing_contact in check_existing_contacts:
founded = False
if data_operation_model.Contacts is not None and len(
data_operation_model.Contacts) > 0:
for operation_contact in data_operation_model.Contacts:
if existing_contact.Email == operation_contact.Email:
founded = True
if not founded:
self.data_operation_contact_repository.delete_by_id(
existing_contact.Id)
order = 0
for operation_integration in data_operation_model.Integrations:
order = order + 1
data_integration = self.data_integration_repository.first(
IsDeleted=0, Code=operation_integration.Integration.Code)
if data_integration is None:
data_integration = self.data_integration_service.create_data_integration(
data=operation_integration.Integration,
definition=definition)
else:
self.data_integration_service.update_data_integration(
data=operation_integration.Integration,
definition=definition)
data_operation_integration = self.data_operation_integration_repository.first(
DataOperationId=data_operation.Id,
DataIntegrationId=data_integration.Id)
if data_operation_integration is None:
new_data_operation_integration = DataOperationIntegration(
Order=order,
Limit=operation_integration.Limit,
ProcessCount=operation_integration.ProcessCount,
DataIntegration=data_integration,
DataOperation=data_operation)
self.data_operation_integration_repository.insert(
new_data_operation_integration)
else:
data_operation_integration.Order = order
data_operation_integration.Limit = operation_integration.Limit
data_operation_integration.ProcessCount = operation_integration.ProcessCount
data_operation_integration.IsDeleted = 0
check_existing_integrations = self.data_operation_integration_repository.filter_by(
IsDeleted=0, DataOperationId=data_operation.Id)
for existing_integration in check_existing_integrations:
founded = False
for operation_integration in data_operation_model.Integrations:
if existing_integration.DataIntegration.Code == operation_integration.Integration.Code:
founded = True
if not founded:
self.data_operation_integration_repository.delete_by_id(
existing_integration.Id)
self.data_integration_service.delete_data_integration(
data_operation_integration.DataIntegration.Code)
self.database_session_manager.commit()
data_operation = self.data_operation_repository.first(
Id=data_operation.Id)
return data_operation
def check_cron_initialized_jobs(self, data_operation_id):
data_operation_jobs = self.data_operation_job_repository.filter_by(
DataOperationId=data_operation_id).all()
for job in data_operation_jobs:
if job.Cron is not None and job.IsDeleted == 0:
raise OperationalException("Job already initialized with cron. ")