def resource_for_job(cls, job_id, role, party_id, operation_type):
operate_status = False
engine_name, cores, memory = cls.calculate_job_resource(job_id=job_id, role=role, party_id=party_id)
try:
with DB.atomic():
updates = {
Job.f_engine_type: EngineType.COMPUTING,
Job.f_engine_name: engine_name,
Job.f_cores: cores,
Job.f_memory: memory,
}
filters = [
Job.f_job_id == job_id,
Job.f_role == role,
Job.f_party_id == party_id,
]
if operation_type == ResourceOperation.APPLY:
updates[Job.f_remaining_cores] = cores
updates[Job.f_remaining_memory] = memory
updates[Job.f_resource_in_use] = True
updates[Job.f_apply_resource_time] = base_utils.current_timestamp()
filters.append(Job.f_resource_in_use == False)
elif operation_type == ResourceOperation.RETURN:
updates[Job.f_resource_in_use] = False
updates[Job.f_return_resource_time] = base_utils.current_timestamp()
filters.append(Job.f_resource_in_use == True)
operate = Job.update(updates).where(*filters)
record_status = operate.execute() > 0
if not record_status:
raise RuntimeError(f"record job {job_id} resource {operation_type} failed on {role} {party_id}")
filters, updates = cls.update_resource_sql(resource_model=EngineRegistry,
cores=cores,
memory=memory,
operation_type=operation_type,
)
filters.append(EngineRegistry.f_engine_type == EngineType.COMPUTING)
filters.append(EngineRegistry.f_engine_name == engine_name)
operate = EngineRegistry.update(updates).where(*filters)
apply_status = operate.execute() > 0
if not apply_status:
raise RuntimeError(
f"{operation_type} resource from engine {engine_name} for job {job_id} resource {operation_type} failed on {role} {party_id}")
operate_status = True
except Exception as e:
schedule_logger(job_id=job_id).warning(e)
schedule_logger(job_id=job_id).warning(
f"{operation_type} job {job_id} resource(cores {cores} memory {memory}) on {role} {party_id} failed")
operate_status = False
finally:
remaining_cores, remaining_memory = cls.get_remaining_resource(EngineRegistry,
[
EngineRegistry.f_engine_type == EngineType.COMPUTING,
EngineRegistry.f_engine_name == engine_name])
operate_msg = "successfully" if operate_status else "failed"
schedule_logger(job_id=job_id).info(
f"{operation_type} job {job_id} resource(cores {cores} memory {memory}) on {role} {party_id} {operate_msg}, remaining cores: {remaining_cores} remaining memory: {remaining_memory}")
return operate_status
def bulk_insert_model_data(self, model, data_source):
with DB.connection_context():
try:
DB.create_tables([model])
batch_size = 50 if RuntimeConfig.USE_LOCAL_DATABASE else 1000
for i in range(0, len(data_source), batch_size):
with DB.atomic():
model.insert_many(data_source[i:i+batch_size]).execute()
return len(data_source)
except Exception as e:
schedule_logger(self.job_id).exception(e)
return 0
def bulk_insert_into_db(model, data_source, logger):
try:
try:
DB.create_tables([model])
except Exception as e:
logger.exception(e)
batch_size = 50 if RuntimeConfig.USE_LOCAL_DATABASE else 1000
for i in range(0, len(data_source), batch_size):
with DB.atomic():
model.insert_many(data_source[i:i + batch_size]).execute()
return len(data_source)
except Exception as e:
logger.exception(e)
return 0
