def delete(self, sparkapp_name: str):
service = SparkAppService()
try:
sparkapp_info = service.delete_sparkapp(sparkapp_name)
return {'data': sparkapp_info.to_dict()}, HTTPStatus.OK
except NotFoundException:
return {'data': {'name': sparkapp_name}}, HTTPStatus.OK
def setUp(self) -> None:
super().setUp()
self._upload_path = os.path.join(BASE_DIR, 'test-spark')
os.makedirs(self._upload_path)
self._patch_upload_path = patch(
'fedlearner_webconsole.sparkapp.service.UPLOAD_PATH',
self._upload_path)
self._patch_upload_path.start()
self._sparkapp_service = SparkAppService()
def post(self):
service = SparkAppService()
data = request.json
try:
config = SparkAppConfig.from_dict(data)
if config.files:
config.files = base64.b64decode(config.files)
except ValueError as err:
raise InvalidArgumentException(details=err)
res = service.submit_sparkapp(config=config)
return {'data': res.to_dict()}, HTTPStatus.CREATED
def __init__(self, task_id: int) -> None:
self.task_id = task_id
self.task_type = None
self.files_dir = None
self.files_path = None
self.main_application = None
self.command = []
self.sparkapp_name = None
self.args = {}
self.started = False
self.error_msg = False
self.spark_service = SparkAppService()
class SparkAppServiceTest(unittest.TestCase):
def setUp(self) -> None:
super().setUp()
self._upload_path = os.path.join(BASE_DIR, 'test-spark')
os.makedirs(self._upload_path)
self._patch_upload_path = patch(
'fedlearner_webconsole.sparkapp.service.UPLOAD_PATH',
self._upload_path)
self._patch_upload_path.start()
self._sparkapp_service = SparkAppService()
def tearDown(self) -> None:
self._patch_upload_path.stop()
shutil.rmtree(self._upload_path)
return super().tearDown()
def _get_tar_file_path(self) -> str:
return os.path.join(
BASE_DIR, 'test/fedlearner_webconsole/test_data/sparkapp.tar')
def test_get_sparkapp_upload_path(self):
existable, sparkapp_path = self._sparkapp_service._get_sparkapp_upload_path(
'test')
self.assertFalse(existable)
os.makedirs(sparkapp_path)
existable, _ = self._sparkapp_service._get_sparkapp_upload_path('test')
self.assertTrue(existable)
def test_copy_files_to_target_filesystem(self):
_, sparkapp_path = self._sparkapp_service._get_sparkapp_upload_path(
'test')
self._sparkapp_service._clear_and_make_an_empty_dir(sparkapp_path)
files_path = self._get_tar_file_path()
with tempfile.TemporaryDirectory() as temp_dir:
file_name = files_path.rsplit('/', 1)[-1]
temp_file_path = os.path.join(temp_dir, file_name)
shutil.copy(files_path, temp_file_path)
self._sparkapp_service._copy_files_to_target_filesystem(
source_filesystem_path=temp_file_path,
target_filesystem_path=sparkapp_path)
self.assertTrue(
os.path.exists(os.path.join(sparkapp_path, 'convertor.py')))
@patch(
'fedlearner_webconsole.utils.k8s_client.k8s_client.create_sparkapplication'
)
def test_submit_sparkapp(self, mock_create_sparkapp: MagicMock):
mock_create_sparkapp.return_value = {
'apiVersion': 'sparkoperator.k8s.io/v1beta2',
'kind': 'SparkApplication',
'metadata': {
'creationTimestamp': '2021-05-18T08:59:16Z',
'generation': 1,
'name': 'fl-transformer-yaml',
'namespace': 'fedlearner',
'resourceVersion': '432649442',
'selfLink':
'/apis/sparkoperator.k8s.io/v1beta2/namespaces/fedlearner/sparkapplications/fl-transformer-yaml',
'uid': '52d66d27-b7b7-11eb-b9df-b8599fdb0aac'
},
'spec': {
'arguments': ['data.csv', 'data_tfrecords/'],
'driver': {
'coreLimit': '4000m',
'cores': 1,
'labels': {
'version': '3.0.0'
},
'memory': '512m',
'serviceAccount': 'spark',
},
'dynamicAllocation': {
'enabled': False
},
'executor': {
'cores': 1,
'instances': 1,
'labels': {
'version': '3.0.0'
},
'memory': '512m',
},
'image': 'dockerhub.com',
'imagePullPolicy': 'Always',
'mainApplicationFile': 'transformer.py',
'mode': 'cluster',
'pythonVersion': '3',
'restartPolicy': {
'type': 'Never'
},
'sparkConf': {
'spark.shuffle.service.enabled': 'false'
},
'sparkVersion': '3.0.0',
'type': 'Python',
},
'status': {
'applicationState': {
'state': 'COMPLETED'
},
'driverInfo': {
'podName': 'fl-transformer-yaml-driver',
'webUIAddress': '11.249.131.12:4040',
'webUIPort': 4040,
'webUIServiceName': 'fl-transformer-yaml-ui-svc'
},
'executionAttempts': 1,
'executorState': {
'fl-transformer-yaml-bdc15979a314310b-exec-1': 'PENDING',
'fl-transformer-yaml-bdc15979a314310b-exec-2': 'COMPLETED'
},
'lastSubmissionAttemptTime': '2021-05-18T10:31:13Z',
'sparkApplicationId': 'spark-a380bfd520164d828a334bcb3a6404f9',
'submissionAttempts': 1,
'submissionID': '5bc7e2e7-cc0f-420c-8bc7-138b651a1dde',
'terminationTime': '2021-05-18T10:32:08Z'
}
}
tarball_file_path = os.path.join(
BASE_DIR, 'test/fedlearner_webconsole/test_data/sparkapp.tar')
with open(tarball_file_path, 'rb') as f:
files_bin = f.read()
inputs = {
'name': 'fl-transformer-yaml',
'files': files_bin,
'image_url': 'dockerhub.com',
'driver_config': {
'cores': 1,
'memory': '200m',
'coreLimit': '4000m',
},
'executor_config': {
'cores': 1,
'memory': '200m',
'instances': 5,
},
'command': ['data.csv', 'data.rd'],
'main_application': '${prefix}/convertor.py'
}
config = SparkAppConfig.from_dict(inputs)
resp = self._sparkapp_service.submit_sparkapp(config)
self.assertTrue(
os.path.exists(
os.path.join(self._upload_path, 'sparkapp',
'fl-transformer-yaml', 'convertor.py')))
mock_create_sparkapp.assert_called_once()
self.assertTrue(resp.namespace, 'fedlearner')
@patch(
'fedlearner_webconsole.utils.k8s_client.k8s_client.get_sparkapplication'
)
def test_get_sparkapp_info(self, mock_get_sparkapp: MagicMock):
mock_get_sparkapp.return_value = {
'apiVersion': 'sparkoperator.k8s.io/v1beta2',
'kind': 'SparkApplication',
'metadata': {
'creationTimestamp': '2021-05-18T08:59:16Z',
'generation': 1,
'name': 'fl-transformer-yaml',
'namespace': 'fedlearner',
'resourceVersion': '432649442',
'selfLink':
'/apis/sparkoperator.k8s.io/v1beta2/namespaces/fedlearner/sparkapplications/fl-transformer-yaml',
'uid': '52d66d27-b7b7-11eb-b9df-b8599fdb0aac'
},
'spec': {
'arguments': ['data.csv', 'data_tfrecords/'],
'driver': {
'coreLimit': '4000m',
'cores': 1,
'labels': {
'version': '3.0.0'
},
'memory': '512m',
'serviceAccount': 'spark',
},
'dynamicAllocation': {
'enabled': False
},
'executor': {
'cores': 1,
'instances': 1,
'labels': {
'version': '3.0.0'
},
'memory': '512m',
},
'image': 'dockerhub.com',
'imagePullPolicy': 'Always',
'mainApplicationFile': 'transformer.py',
'mode': 'cluster',
'pythonVersion': '3',
'restartPolicy': {
'type': 'Never'
},
'sparkConf': {
'spark.shuffle.service.enabled': 'false'
},
'sparkVersion': '3.0.0',
'type': 'Python',
},
'status': {
'applicationState': {
'state': 'COMPLETED'
},
'driverInfo': {
'podName': 'fl-transformer-yaml-driver',
'webUIAddress': '11.249.131.12:4040',
'webUIPort': 4040,
'webUIServiceName': 'fl-transformer-yaml-ui-svc'
},
'executionAttempts': 1,
'executorState': {
'fl-transformer-yaml-bdc15979a314310b-exec-1': 'PENDING',
'fl-transformer-yaml-bdc15979a314310b-exec-2': 'COMPLETED'
},
'lastSubmissionAttemptTime': '2021-05-18T10:31:13Z',
'sparkApplicationId': 'spark-a380bfd520164d828a334bcb3a6404f9',
'submissionAttempts': 1,
'submissionID': '5bc7e2e7-cc0f-420c-8bc7-138b651a1dde',
'terminationTime': '2021-05-18T10:32:08Z'
}
}
resp = self._sparkapp_service.get_sparkapp_info('fl-transformer-yaml')
mock_get_sparkapp.assert_called_once()
self.assertTrue(resp.namespace, 'fedlearner')
@patch(
'fedlearner_webconsole.sparkapp.service.SparkAppService._get_sparkapp_upload_path'
)
@patch('fedlearner_webconsole.utils.file_manager.FileManager.remove')
@patch(
'fedlearner_webconsole.utils.k8s_client.k8s_client.delete_sparkapplication'
)
def test_delete_sparkapp(self, mock_delete_sparkapp: MagicMock,
mock_file_mananger_remove: MagicMock,
mock_upload_path: MagicMock):
mock_delete_sparkapp.return_value = {
'kind': 'Status',
'apiVersion': 'v1',
'metadata': {},
'status': 'Success',
'details': {
'name': 'fl-transformer-yaml',
'group': 'sparkoperator.k8s.io',
'kind': 'sparkapplications',
'uid': '52d66d27-b7b7-11eb-b9df-b8599fdb0aac'
}
}
mock_upload_path.return_value = (True, 'test')
resp = self._sparkapp_service.delete_sparkapp(
name='fl-transformer-yaml')
mock_delete_sparkapp.assert_called_once()
mock_file_mananger_remove.assert_called_once()
self.assertTrue(resp.name, 'fl-transformer-yaml')
def get(self, sparkapp_name: str):
service = SparkAppService()
return {
'data': service.get_sparkapp_info(sparkapp_name).to_dict()
}, HTTPStatus.OK
class DataPipelineRunner(IRunner):
TYPE_PARAMS_MAPPER = {
DataPipelineType.ANALYZER: {
'files_dir': 'fedlearner_webconsole/dataset/sparkapp/pipeline',
'main_application': 'pipeline/analyzer.py',
},
DataPipelineType.CONVERTER: {
'files_dir': 'fedlearner_webconsole/dataset/sparkapp/pipeline',
'main_application': 'pipeline/converter.py',
},
DataPipelineType.TRANSFORMER: {
'files_dir': 'fedlearner_webconsole/dataset/sparkapp/pipeline',
'main_application': 'pipeline/transformer.py',
}
}
SPARKAPP_STATE_TO_RUNNER_STATUS = {
'': RunnerStatus.RUNNING,
'SUBMITTED': RunnerStatus.RUNNING,
'PENDING_RERUN': RunnerStatus.RUNNING,
'RUNNING': RunnerStatus.RUNNING,
'COMPLETED': RunnerStatus.DONE,
'SUCCEEDING': RunnerStatus.DONE,
'FAILED': RunnerStatus.FAILED,
'SUBMISSION_FAILED': RunnerStatus.FAILED,
'INVALIDATING': RunnerStatus.FAILED,
'FAILING': RunnerStatus.FAILED,
'UNKNOWN': RunnerStatus.FAILED
}
def __init__(self, task_id: int) -> None:
self.task_id = task_id
self.task_type = None
self.files_dir = None
self.files_path = None
self.main_application = None
self.command = []
self.sparkapp_name = None
self.args = {}
self.started = False
self.error_msg = False
self.spark_service = SparkAppService()
def start(self, context: Context):
try:
self.started = True
self.args = deepcopy(context.data.get(str(self.task_id), {}))
self.task_type = DataPipelineType(self.args.pop('task_type'))
name = self.args.pop('sparkapp_name')
job_id = uuid4().hex
self.sparkapp_name = f'pipe-{self.task_type.value}-{job_id}-{name}'
params = self.__class__.TYPE_PARAMS_MAPPER[self.task_type]
self.files_dir = os.path.join(Envs.BASE_DIR, params['files_dir'])
self.files_path = Envs.SPARKAPP_FILES_PATH
self.main_application = params['main_application']
self.command = self.args.pop('input')
files = None
if self.files_path is None:
files_obj = io.BytesIO()
with tarfile.open(fileobj=files_obj, mode='w') as f:
f.add(self.files_dir)
files = files_obj.getvalue()
config = {
'name': self.sparkapp_name,
'files': files,
'files_path': self.files_path,
'image_url': Envs.SPARKAPP_IMAGE_URL,
'volumes': gen_sparkapp_volumes(Envs.SPARKAPP_VOLUMES),
'driver_config': {
'cores':
1,
'memory':
'4g',
'volume_mounts':
gen_sparkapp_volume_mounts(Envs.SPARKAPP_VOLUME_MOUNTS),
},
'executor_config': {
'cores':
2,
'memory':
'4g',
'instances':
1,
'volume_mounts':
gen_sparkapp_volume_mounts(Envs.SPARKAPP_VOLUME_MOUNTS),
},
'main_application': f'${{prefix}}/{self.main_application}',
'command': self.command,
}
config_dict = SparkAppConfig.from_dict(config)
resp = self.spark_service.submit_sparkapp(config=config_dict)
logging.info(
f'created spark app, name: {name}, '
f'config: {config_dict.__dict__}, resp: {resp.__dict__}')
except Exception as e: # pylint: disable=broad-except
self.error_msg = f'[composer] failed to run this item, err: {e}, \
trace: {traceback.format_exc()}'
def result(self, context: Context) -> Tuple[RunnerStatus, dict]:
if self.error_msg:
context.set_data(f'failed_{self.task_id}',
{'error': self.error_msg})
return RunnerStatus.FAILED, {}
if not self.started:
return RunnerStatus.RUNNING, {}
resp = self.spark_service.get_sparkapp_info(self.sparkapp_name)
logging.info(f'sparkapp resp: {resp.__dict__}')
if not resp.state:
return RunnerStatus.RUNNING, {}
return self.__class__.SPARKAPP_STATE_TO_RUNNER_STATUS.get(
resp.state, RunnerStatus.FAILED), resp.to_dict()