def __enter__(self):
logger.debug("Connecting to " + self.server_addr + ":" + str(self.server_port))
server_auth_name = xdcs().config('server.auth.name')
server_auth_key = xdcs().config('server.auth.key', None)
server_auth_password = xdcs().config('server.auth.password', None)
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(
self.server_addr, self.server_port,
username=server_auth_name,
pkey=paramiko.RSAKey.from_private_key_file(server_auth_key),
password=server_auth_password,
look_for_keys=False,
)
logger.debug("Connected to server's sshd")
self.transport = client.get_transport()
remote_port = self.transport.request_port_forward("127.0.0.1", 0)
logger.debug("Now forwarding remote port %d to local port %d", remote_port, self.local_port)
if self.asynchronous:
self.thread = threading.Thread(target=self._transport_thread, args=())
self.thread.setDaemon(True)
self.thread.start()
else:
self._transport_thread()
def execute(self):
request = AgentRegistrationRequest()
request.displayName = xdcs().config('agent.name')
stub = AgentRegistrationStub(xdcs().channel())
response = stub.Register(request)
if not response.success:
raise RegistrationFailedException()
def execute(self):
stub = ObjectRepositoryStub(xdcs().channel())
req = DependencyResolutionRequest()
deployment_key = ObjectKey()
deployment_key.objectId = self._deployment_id
deployment_key.objectType = ObjectType.DEPLOYMENT
req.objectKeys.extend([deployment_key])
req.depth = 2**32 - 1
object_ids = stub.ResolveDependencies(req).objectIds
object_ids.append(self._deployment_id)
xdcs().execute(RetrieveObjectsCmd((ids for ids in [object_ids])))
def execute(self):
logger.info('Reporting task completion: ' + self._task_id)
stub = TaskReportingStub(xdcs().channel())
task_id = TaskId()
task_id.taskId = self._task_id
stub.ReportCompletion(task_id)
def execute(self):
deployment = self._deployment
script_path = path.join(self._workspace_path,
deployment['config'].get('scriptfile'))
if script_path is None:
raise TaskExecutionException('Script path is empty')
with UploadingLogHandler(self._task_id) as log_handler:
log_handler = log_handler.combine(PassThroughLogHandler(logger))
# TODO: this is currently needed only because we cannot set
# file permissions in GUI yet
st = os.stat(script_path)
os.chmod(script_path, st.st_mode | stat.S_IEXEC)
exec_cmd([script_path], log_handler)
xdcs().execute(ReportTaskCompletionCmd(self._task_id))
def execute(self):
deployment = self._deployment
dockerfile = deployment['config'].get('dockerfile', None)
if dockerfile is None or len(dockerfile) == 0:
dockerfile = 'Dockerfile'
dockerfile = path.join(self._workspace_path, dockerfile)
image_id = DockerCli().build(self._workspace_path, dockerfile)
logger.info('Docker built, image ID: ' + image_id)
with UploadingLogHandler(self._task_id) as log_handler:
log_handler = log_handler.combine(PassThroughLogHandler(logger))
DockerCli() \
.remove_container_after_finish() \
.nvidia_all_devices() \
.container_name('xdcs_' + self._task_id) \
.allocate_pseudo_tty() \
.run(image_id, log_handler)
xdcs().execute(ReportTaskCompletionCmd(self._task_id))
def _dump_tree(self, root_id, out_path):
entries = xdcs().object_repository().cat_json(root_id)
for entry in entries:
file_path = path.join(out_path, entry['name'])
mode = entry['mode']
object_id = entry['id']
object_type = mode[:2]
permissions = int(mode[2:], 8)
if object_type == '12':
content = xdcs().object_repository().cat_bytes(object_id)
os.symlink(content, file_path)
os.chmod(file_path, permissions)
elif object_type == '10':
content = xdcs().object_repository().cat_bytes(object_id)
with open(file_path, 'wb+') as fh:
fh.write(content)
os.chmod(file_path, permissions)
elif object_type == '04':
os.mkdir(file_path, permissions)
self._dump_tree(object_id, file_path)
def execute(self):
stub = ObjectRepositoryStub(xdcs().channel())
chunks = stub.RequestObjects(self.__generate_requests())
tmp_dir = tempfile.mkdtemp()
try:
zip_path = tmp_dir + '/zip'
# download the zip file
with open(zip_path, 'wb') as zipfile:
self.__write_chunks(zipfile, chunks)
# extract objects
with ZipFile(zip_path) as zipfile:
zipfile.extractall(tmp_dir)
objects = zipfile.namelist()
# remove the zip file
os.remove(zip_path)
obj_repo = xdcs().object_repository()
for obj in objects:
obj_repo.import_object(tmp_dir + '/' + obj, required_id=obj)
finally:
shutil.rmtree(tmp_dir)
def execute(self):
logger.info('Running a deployment: ' + self._deployment_id)
with tempfile.TemporaryDirectory() as workspace_path:
xdcs().execute(FetchDeploymentCmd(self._deployment_id))
deployment: dict = xdcs().object_repository().cat_json(
self._deployment_id)
root_id = deployment['root']
xdcs().execute(DumpObjectRepositoryTreeCmd(root_id,
workspace_path))
config_type = deployment['config']['type']
constructor_args = [
workspace_path, deployment, self._deployment_id, self._task_id
]
if config_type == 'docker':
xdcs().execute(RunDockerTaskCmd(*constructor_args))
elif config_type == 'script':
xdcs().execute(RunScriptTaskCmd(*constructor_args))
else:
raise Exception('Unsupported task type ' + config_type)
logger.info('Deployment finished: ' + self._deployment_id)
def _send_messages(self):
q = self._log_queue
stub = TaskReportingStub(xdcs().channel())
finished = False
while not finished:
lines = [q.get()]
while not q.empty():
lines.append(q.get())
if lines[-1] is None:
lines = lines[0:-1]
finished = True
if len(lines) > 0:
logs = Logs()
logs.taskId = self._task_id
logs.lines.extend(lines)
stub.UploadLogs(logs)
time.sleep(1)
def AcceptSecurityInformation(self, request: SecurityInformation, context):
token = request.tokenGrant.token
certificate = request.serverCertificate.bytes
xdcs().set_token(token)
token_interceptor = interceptors.header_adder_interceptor('authorization', token)
server_host = xdcs().config('server.host')
server_port = xdcs().config('server.port.grpc')
creds = grpc.ssl_channel_credentials(root_certificates=certificate)
channel = grpc.secure_channel(server_host + ':' + str(server_port), creds)
intercepted_channel = grpc.intercept_channel(channel, token_interceptor)
xdcs().set_channel(intercepted_channel)
self._register_async()
return SecurityInformationResponse()
def __execute_async(self, deployment_id, task_id):
xdcs().execute(RunTaskCmd(deployment_id, task_id))
import logging
from xdcs.app import xdcs
logging.basicConfig(level=logging.DEBUG)
if __name__ == '__main__':
xdcs().run()
def replacement(*args, **kwargs):
return xdcs().executor() \
.submit(lambda: task(*args, **kwargs))
def __init__(self) -> None:
self._docker_exec = xdcs().config('system.docker', 'docker')
def _register_async(self):
xdcs().execute(RegisterAgentCmd())
