def parallel_data_test(process_id, process_name, tasks, results):
try:
print('[%s] evaluation routine starts' % process_name)
root_directory = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
os.pardir, os.pardir, os.pardir))
IocManager.configure_startup(root_directory)
sql_logger = IocManager.injector.get(SqlLogger)
while True:
new_value = tasks.get()
if new_value.IsFinished:
sql_logger.info(f"{process_name} process finished")
print('[%s] evaluation routine quits' % process_name)
# Indicate finished
results.put(new_value)
break
else:
new_value.Data.Result = TestParallelService.calculate_method(
sql_logger, process_name, new_value.Data.Value)
# Add result to the queue
results.put(new_value)
except Exception as ex:
data = TaskData(IsFinished=True)
results.put(data)
def __init__(self, methodName='TestConfigService'):
super(TestConfigService, self).__init__(methodName)
os.environ["PYTHON_ENVIRONMENT"] = 'test'
root_directory = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,
os.pardir, os.pardir))
IocManager.configure_startup(root_directory=root_directory,
app_wrapper=FlaskAppWrapper,
job_scheduler=JobScheduler)
def start():
from infrastructor.IocManager import IocManager
import os
root_directory = os.path.dirname(os.path.abspath(__file__))
from infrastructor.api.FlaskAppWrapper import FlaskAppWrapper
from infrastructor.scheduler.JobScheduler import JobScheduler
IocManager.configure_startup(root_directory=root_directory,
app_wrapper=FlaskAppWrapper,
job_scheduler=JobScheduler)
IocManager.run()
def __init__(self, methodName='runProcessTest'):
super(TestMultiProcess, self).__init__(methodName)
from infrastructor.api.FlaskAppWrapper import FlaskAppWrapper
from infrastructor.scheduler.JobScheduler import JobScheduler
os.environ["PYTHON_ENVIRONMENT"] = 'test'
root_directory = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,
os.pardir))
IocManager.configure_startup(root_directory=root_directory,
app_wrapper=FlaskAppWrapper,
job_scheduler=JobScheduler)
self.client = IocManager.app.test_client()
def __init__(self):
from infrastructor.api.FlaskAppWrapper import FlaskAppWrapper
from infrastructor.scheduler.JobScheduler import JobScheduler
os.environ["PYTHON_ENVIRONMENT"] = 'test'
root_directory = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,
os.pardir, os.pardir))
IocManager.configure_startup(root_directory=root_directory,
app_wrapper=FlaskAppWrapper,
job_scheduler=JobScheduler)
self.client = IocManager.app.test_client()
self.ioc_manager = IocManager
self.api_client = TestApiClient(self.client)
self.service_endpoints = TestServiceEndpoints(self.api_client)
self.service_scenarios = TestServiceScenarios(
service_endpoints=self.service_endpoints,
ioc_manager=self.ioc_manager)
def parallel_operation(process_id, job_id, sub_process_id, process_name,
tasks, results):
root_directory = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,
os.pardir, os.pardir))
IocManager.configure_startup(root_directory)
sql_logger = IocManager.injector.get(SqlLogger)
try:
print('[%s] evaluation routine starts' % process_name)
database_session_manager = IocManager.injector.get(
DatabaseSessionManager)
data_operation_job_service = IocManager.injector.get(
DataOperationJobService)
data_integration_repository: Repository[
DataIntegration] = Repository[DataIntegration](
database_session_manager)
data_integration_connection_repository: Repository[
DataIntegrationConnection] = Repository[
DataIntegrationConnection](database_session_manager)
data_integration = data_integration_repository.first(Id=process_id,
IsDeleted=0)
source_connection = data_integration_connection_repository.first(
IsDeleted=0,
DataIntegration=data_integration,
SourceOrTarget=0)
target_connection = data_integration_connection_repository.first(
IsDeleted=0,
DataIntegration=data_integration,
SourceOrTarget=1)
execute_operation_dto = data_operation_job_service.get_execute_operation_dto(
source_connection=source_connection,
target_connection=target_connection,
data_integration_columns=data_integration.Columns)
while True:
# waiting for new task
new_task: TaskData = tasks.get()
new_task.SubProcessId = sub_process_id
if new_task.IsFinished:
sql_logger.info(f"{process_name} process finished")
# Indicate finished
results.put(new_task)
break
else:
start = time()
sql_logger.info(
f"{process_name}-{sub_process_id} process got a new task.Id:{new_task.Data.Id} limits:{new_task.Data.SubLimit}-{new_task.Data.TopLimit} "
)
execute_operation_dto.limit_modifier = new_task.Data
data_operation_job_service.execute_operation(
execute_operation_dto=execute_operation_dto)
end = time()
sql_logger.info(
f"{process_name} process finished task. limits:{new_task.Data.SubLimit}-{new_task.Data.TopLimit}. time:{end - start}"
)
new_task.IsProcessed = True
results.put(new_task)
except Exception as ex:
sql_logger.error(f"{process_name} process getting error:{ex}",
job_id=job_id)
data = TaskData(SubProcessId=sub_process_id, IsFinished=True)
results.put(data)