def start_task(task: Task):
print('start_task')
print(task.task_id)
tc = TaskController()
dm = DataManager()
try:
print('script path:' + get_script_path(task))
# codes from git which contain docker-compose.yml
if is_git_files(task):
sub = subprocess.Popen(
['docker-compose', 'up'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=get_script_work_path(task), bufsize=1)
# uploaded codes
else:
sub = subprocess.Popen(
['python3.6', get_script_path(task)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=get_script_work_path(task), bufsize=1)
for line in iter(sub.stdout.readline, b''):
if line:
print(line)
dm.add_custom_log(CustomLog(
task_id=task.task_id,
content=line,
time=int(time.time())
))
sub.stdout.close()
if sub.returncode == 0:
print('Subprogram success')
tc.finish_task(task.task_id)
else:
print('Subprogram failed')
tc.add_custom_log_callback(CustomLog(
task_id=task.task_id,
content=sub.stdout.read().decode('utf8'),
time=int(time.time())
))
tc.stop_task(task.task_id)
except Exception as e:
print('[error]' + str(e))
tc.add_custom_log_callback(CustomLog(
task_id=task.task_id,
content=str(e),
time=int(time.time())
))
from werkzeug.utils import secure_filename
from server.config.conf import STATIC_PATH
from task_controller.gen.task_controller_client import TaskController, Task
from data_manager.gen.data_manger_client import DataManager
from task_runtime.gen.task_runtime_client import TaskRuntime
from flask_cors import CORS
import json
import task_controller.gen.deploy_controller_client
import task_runtime.gen.deploy_runtime_client
import time
app = Flask(__name__)
cors = CORS(app, resources={r"/*": {"origins": "*"}})
tc = TaskController()
dm = DataManager()
tr = TaskRuntime()
dc = task_controller.gen.deploy_controller_client.TaskController()
dr = task_runtime.gen.deploy_runtime_client.TaskRuntime()
@app.route('/')
def hello():
return "Hello, It's CECL."
@app.route('/upload', methods=['POST'])
def upload():
f = request.files['train']
filename = secure_filename(f.filename)
def __init__(self):
self.db = DataManager()
self.runtime = TaskRuntime()
self.tasks = []
class TaskController(task_controller_pb2_grpc.TaskControllerServicer):
def __init__(self):
self.db = DataManager()
self.runtime = TaskRuntime()
self.tasks = []
def AddTaskByGit(self, request, context):
request_task: task_controller_pb2.Task = request.task
# wrap to task
task: Task = Task(
task_id=request_task.task_id,
name=request_task.name,
create_time=request_task.create_time,
start_time=request_task.start_time,
end_time=request_task.end_time,
union_train=request_task.union_train,
edge_nodes=request_task.edge_nodes,
file=request_task.file,
status=0,
)
# add to db
self.tasks.append(task)
resp = self.db.add_task(task).resp
print("add task by git:" + resp.message)
print(resp)
if resp.code != 0:
return task_controller_pb2.AddTaskByGitResp(
resp=task_controller_pb2.Response(
code=resp.code,
message="add task to db fail",
))
# get id from db
task.task_id = json.loads(resp.message)['id']
# clone
self.runtime.add_task_by_git(task)
# response
return task_controller_pb2.AddTaskByGitResp(
resp=task_controller_pb2.Response(
code=0,
message="start cloning",
))
def AddTaskByHTTP(self, request, context):
return super().AddTaskByHTTP(request, context)
def AddCustomLogCallback(self, request, context):
log: CustomLog = request.custom_log
resp = self.db.add_custom_log(log).resp
return task_controller_pb2.AddCustomLogCallbackResp(
resp=task_controller_pb2.Response(
code=resp.code,
message=resp.message,
))
def AddTask(self, request, context):
request_task = request.task
task: Task = Task(task_id=request_task.task_id,
name=request_task.name,
create_time=request_task.create_time,
start_time=request_task.start_time,
end_time=request_task.end_time,
union_train=request_task.union_train,
edge_nodes=request_task.edge_nodes,
file=request_task.file,
status=0)
self.tasks.append(task)
resp = self.db.add_task(task).resp
print("add task:" + resp.message)
return task_controller_pb2.AddTaskResp(
resp=task_controller_pb2.Response(
code=resp.code,
message=resp.message,
))
# if resp.code != 0:
# return task_controller_pb2.AddTaskResp(resp=task_controller_pb2.Response(
# code=resp.code,
# message=resp.message,
# ))
# resp = self.runtime.start_task(request_task.task_id).resp
# return task_controller_pb2.AddTaskResp(resp=task_controller_pb2.Response(
# code=resp.code,
# message=resp.message,
# ))
def StartTask(self, request, context):
task_id: int = request.task_id
print("start task task id:" + str(task_id))
resp = self.db.start_task(task_id, int(time.time())).resp
if resp.code != 0:
return task_controller_pb2.StartTaskResp(
resp=task_controller_pb2.Response(code=resp.code,
message=resp.message))
resp = self.runtime.start_task(task_id).resp
print("start task runtime code:" + str(resp.code))
if resp.code != 0:
return task_controller_pb2.StartTaskResp(
resp=task_controller_pb2.Response(code=resp.code,
message=resp.message))
return task_controller_pb2.StartTaskResp(
resp=task_controller_pb2.Response(code=resp.code,
message=resp.message))
def StopTask(self, request, context):
task_id: int = request.task_id
resp = self.runtime.stop_task(task_id).resp
if resp.code != 0:
return task_controller_pb2.StopTaskReq(
resp=task_controller_pb2.Response(code=resp.code,
message=resp.message))
resp = self.db.stop_task(task_id, int(time.time()))
return task_controller_pb2.StopTaskReq(
resp=task_controller_pb2.Response(code=resp.code,
message=resp.message))
def FinishTask(self, request, context):
task_id: int = request.task_id
resp = self.db.finish_task(task_id, int(time.time())).resp
return task_controller_pb2.FinishTaskResp(
resp=task_controller_pb2.Response(code=resp.code,
message=resp.message))
def GetAllTasks(self, request, context):
return super().GetAllTasks(request, context)
def GetTaskInfo(self, request, context):
return super().GetTaskInfo(request, context)
def UpdateTask(self, request, context):
return super().UpdateTask(request, context)
def SendMessage(self, request, context):
print("get message: " + request.message)
return task_controller_pb2.SendMessageResp(
resp=task_controller_pb2.Response(code=0, message="success"))
from git import Repo
from data_manager.gen.data_manger_client import DataManager
from common.task import Task
from common.custom_log import CustomLog
import time
db = DataManager()
def clone_from_url(url, path, task: Task):
try:
Repo.clone_from(url, path)
clone_finished(task)
except Exception as e:
clone_error(task, e)
def clone_error(request_task, error):
# add pulling log
db.add_pulling_log(task_log=CustomLog(task_id=request_task.task_id,
content=str(error),
time=int(time.time())))
# update status
request_task.status = Task.PULL_FAILURE
db.update_task(task=request_task)
def clone_finished(request_task):
# update status
def __init__(self):
self.tasks: dict = {}
self.db = DataManager()