def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/commands.*') self.command = { "id": "test_command", "created": "2016-05-09T20:50:56.000Z", "updated": "2016-05-11T00:18:26.000Z", "tags": ["genie.id:test_command", "type:test", "ver:1", "genie.name:test"], "version": "1", "user": "******", "name": "presto", "description": None, "configs": [], "dependencies": [], "setupFile": None, "status": "ACTIVE", "executable": "sleep 1", "checkDelay": 5000, "page": { "totalPages": 1, "number": 1, }, }
class TestGenie(unittest.TestCase): "Test related to the bdp EMR library" def setUp(self): self.genie = Genie(GENIE_CONF) self.cluster_name = 'test' self.path = re.compile(GENIE_URL + '/api/v3/.*') def test_properties(self): "Testing that relative paths exists as methods" assert self.genie.path_application assert self.genie.path_command assert self.genie.path_job assert self.genie.path_cluster @responses.activate def test_get_clusters(self): "Testing that get_clusters returns a generator" responses.add(responses.GET, self.path) clusters = self.genie.get_clusters() assert isinstance(clusters, types.GeneratorType), "Did not return a generator" @responses.activate def test_get_applications(self): "Testing that get_applications returns a generator" responses.add(responses.GET, self.path) applications = self.genie.get_applications() assert isinstance(applications, types.GeneratorType), "Did not return a generator" @responses.activate def test_get_jobs(self): "Testing that get_jobs returns a generator" responses.add(responses.GET, self.path) jobs = self.genie.get_jobs() assert isinstance(jobs, types.GeneratorType), "Did not return a generator" @responses.activate def test_get_commands(self): "Testing that get_commands returns a generator" responses.add(responses.GET, self.path) commands = self.genie.get_commands() assert isinstance(commands, types.GeneratorType), "Did not return a generator"
def create_spark_version(genie_client: Genie, version: str, hadoop_app_id: str) -> None: human_version: str = ".".join(version) app_id: str = genie_client.create_application( load_yaml(f"applications/spark{version}.yml")) LOGGER.info( f"Created Spark {human_version} application with id = {app_id}") shell_command_id: str = genie_client.create_command( load_yaml(f"commands/sparkShell{version}.yml")) LOGGER.info( f"Created Spark Shell {human_version} command with id = {shell_command_id}" ) submit_command_id = genie_client.create_command( load_yaml(f"commands/sparkSubmit{version}.yml")) LOGGER.info( f"Created Spark Submit {human_version} command with id = {submit_command_id}" ) genie_client.set_application_for_command(shell_command_id, [hadoop_app_id, app_id]) LOGGER.info( f"Set applications for Spark Shell {human_version} command to = {','.join([hadoop_app_id, app_id])}" ) genie_client.set_application_for_command(submit_command_id, [hadoop_app_id, app_id]) LOGGER.info( f"Set applications for Spark Submit {human_version} command to = {','.join([hadoop_app_id, app_id])}" )
def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/clusters.*') self.cluster = { "id": "presto_test_20160318", "created": "2016-04-12T19:52:10.000Z", "updated": "2016-05-09T20:52:53.000Z", "tags": ["genie.name:test", "ver:1", "type:test"], "version": "1", "user": "******", "name": "testcluster", "description": "Test Cluster", "configs": [], "setupFile": "s3://test-bucket/cluster.sh", "status": "UP", "page": { "totalPages": 1, "number": 1, }, }
def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/applications.*') self.application = { "id": "test", "created": "2016-05-10T23:37:24.000Z", "updated": "2016-05-11T00:18:23.000Z", "tags": ["type:hadoop", "genie.id:test", "genie.name:test"], "version": "1", "user": "******", "name": "test", "description": "Test command", "configs": [], "setupFile": "s3://test-bucket/setup.sh", "dependencies": ["s3://test-bucket/test.tgz"], "status": "ACTIVE", "page": { "totalPages": 1, "number": 1, }, }
def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/jobs.*') self.job = { "id": "5afdcab5-16df-11e6-b78d-6003088f363e", "created": "2016-05-10T18:45:29.000Z", "updated": "2016-05-10T19:06:18.000Z", "tags": [], "version": "1.0", "user": "******", "name": "test_command_name", "description": "Test command", "commandArgs": "-f presto.q", "status": "SUCCEEDED", "statusMsg": "Job finished successfully.", "started": "2016-05-10T18:45:30.000Z", "finished": "2016-05-10T19:06:18.000Z", "archiveLocation": "s3://test-bucket/test.tar.gz", "clusterName": "test_cluster", "commandName": "test_command", "runtime": "TEST005", }
def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/commands.*') self.command = { "id": "test_command", "created": "2016-05-09T20:50:56.000Z", "updated": "2016-05-11T00:18:26.000Z", "tags": ["genie.id:test_command", "type:test", "ver:1", "genie.name:test"], "version": "1", "user": "******", "name": "presto", "description": None, "configs": [], "setupFile": None, "status": "ACTIVE", "executable": "sleep 1", "checkDelay": 5000, "page": { "totalPages": 1, "number": 1, }, }
class TestCommand(unittest.TestCase): def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/commands.*') self.command = { "id": "test_command", "created": "2016-05-09T20:50:56.000Z", "updated": "2016-05-11T00:18:26.000Z", "tags": ["genie.id:test_command", "type:test", "ver:1", "genie.name:test"], "version": "1", "user": "******", "name": "presto", "description": None, "configs": [], "dependencies": [], "setupFile": None, "status": "ACTIVE", "executable": "sleep 1", "checkDelay": 5000, "page": { "totalPages": 1, "number": 1, }, } @responses.activate def test_create_command(self): location = GENIE_URL + '/' + self.command['id'] responses.add(responses.POST, self.path, status=201, adding_headers={'Location': location}) new_command_name = self.genie.create_command(self.command) assert_equals(new_command_name, self.command['id']) @responses.activate def test_patch_command(self): patch1 = {'op': 'replace', 'path': '/version', 'value': '1.1'} patch2 = {'op': 'replace', 'path': '/user', 'value': 'test'} responses.add(responses.PATCH, self.path) self.genie.patch_command(self.command['id'], [patch1, patch2]) @responses.activate def test_get_command(self): "Testing that get_commands returns all required keys" responses.add(responses.GET, self.path, body=json.dumps(self.command)) command = self.genie.get_command(self.command['id']) assert_equals(command, self.command) @responses.activate def test_get_command_returns_none(self): responses.add(responses.GET, self.path, status=404) cluster = self.genie.get_command('does_not_exist') assert_equals(cluster, None) @responses.activate def test_get_command_with_params(self): resp = { "_embedded": { "commandList": [self.command] }, "page": { "totalPages": 0, "number": 0 } } resp = json.dumps(resp) responses.add(responses.GET, self.path, body=resp) commands = [ i for i in self.genie.get_commands( filters={'name': self.command['name']}) ] assert len( commands) == 1, "Didn't return the correct amount of commands" assert_equals(commands[0], self.command) @responses.activate def test_update_command(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_command(self.command) @responses.activate def test_delete_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.delete_command(self.command['id']) @responses.activate def test_create_and_delete_all_commands(self): responses.add(responses.DELETE, self.path, status=204) self.genie.delete_all_commands() @responses.activate def test_add_applications_for_command(self): responses.add(responses.POST, self.path, status=204) self.genie.add_applications_for_command(self.command['id'], ['test']) @responses.activate def test_add_configs_for_command(self): responses.add(responses.POST, self.path, status=204) self.genie.add_configs_for_command(self.command['id'], ['test']) @responses.activate def test_add_dependencies_for_command(self): responses.add(responses.POST, self.path, status=204) self.genie.add_dependencies_for_command(self.command['id'], ['test']) @responses.activate def test_add_tags_for_command(self): responses.add(responses.POST, self.path, status=204) self.genie.add_tags_for_command(self.command['id'], ['tag:test']) @responses.activate def test_get_applications_for_command(self): responses.add(responses.GET, self.path, body='["app1", "app2"]') apps = self.genie.get_applications_for_command('test') assert len(apps) == 2, "Did not return enough apps for command" @responses.activate def test_get_clusters_for_command(self): responses.add(responses.GET, self.path, body='["test1", "test2"]') clusters = self.genie.get_clusters_for_command(self.command['id']) assert len(clusters) == 2, "Did not return enough commands" assert "test1" in clusters, "Did not return the right clusters" @responses.activate def test_get_configs_for_command(self): responses.add(responses.GET, self.path, body='["conf1", "conf2"]') configs = self.genie.get_configs_for_command(self.command['id']) assert len(configs) == 2, "Did not return enough configs" assert "conf1" in configs, "did not return the right configs" @responses.activate def test_get_dependencies_for_command(self): responses.add(responses.GET, self.path, body='["dep1", "dep2"]') dependencies = self.genie.get_dependencies_for_command( self.command['id']) assert len(dependencies) == 2, "Did not return enough dependencies" assert "dep1" in dependencies, "Did not return the right dependencies" @responses.activate def test_get_tags_for_command(self): responses.add(responses.GET, self.path, body='["tag:1", "tag:2"]') tags = self.genie.get_tags_for_command(self.command['id']) assert len(tags) == 2, "Did not return enough tags" assert "tag:1" in tags, "Did not return the right tags" @responses.activate def test_remove_all_applications_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_applications_for_command(self.command['id']) @responses.activate def test_remove_all_configs_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_configs_for_command(self.command['id']) @responses.activate def test_remove_all_dependencies_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_dependencies_for_command(self.command['id']) @responses.activate def test_remove_all_tags_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_tags_for_command(self.command['id']) @responses.activate def test_remove_application_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_application_for_command(self.command['id'], 'test') @responses.activate def test_remove_tag_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_tag_for_command(self.command['id'], 'test') @responses.activate def test_set_application_for_command(self): responses.add(responses.PUT, self.path, status=204) self.genie.set_application_for_command('test', ['app1', 'app2']) @responses.activate def test_update_configs_for_command(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_configs_for_command('test', ['conf1', 'conf2']) @responses.activate def test_update_dependencies_for_command(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_dependencies_for_command('test', ['dep1', 'dep2']) @responses.activate def test_update_tags_for_command(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_tags_for_command('test', ['tag:test', 'tag2:test2'])
def setUp(self): self.genie = Genie(GENIE_CONF) self.required_apis = [ 'add_applications_for_command', 'add_commands_for_cluster', 'add_configs_for_application', 'add_configs_for_cluster', 'add_configs_for_command', 'add_dependencies_for_application', 'add_dependencies_for_cluster', 'add_dependencies_for_command', 'add_tags_for_application', 'add_tags_for_cluster', 'add_tags_for_command', 'create_application', 'create_cluster', 'create_command', 'delete_all_applications', 'delete_all_clusters', 'delete_all_commands', 'delete_application', 'delete_cluster', 'delete_command', 'get_application', 'get_applications', 'get_applications_for_command', 'get_cluster', 'get_clusters', 'get_clusters_for_command', 'get_command', 'get_commands', 'get_commands_for_cluster', 'get_configs_for_application', 'get_configs_for_cluster', 'get_configs_for_command', 'get_dependencies_for_application', 'get_dependencies_for_cluster', 'get_dependencies_for_command', 'get_job_applications', 'get_job_cluster', 'get_job_command', 'get_job_execution', 'get_job_output', 'get_job_request', 'get_job_status', 'get_tags_for_application', 'get_tags_for_cluster', 'get_tags_for_command', 'patch_application', 'patch_cluster', 'patch_command', 'remove_all_applications_for_command', 'remove_all_commands_for_cluster', 'remove_all_configs_for_application', 'remove_all_configs_for_cluster', 'remove_all_configs_for_command', 'remove_all_dependencies_for_application', 'remove_all_dependencies_for_cluster', 'remove_all_dependencies_for_command', 'remove_all_tags_for_application', 'remove_all_tags_for_cluster', 'remove_all_tags_for_command', 'remove_application_for_command', 'remove_commands_for_cluster', 'remove_tag_for_application', 'remove_tag_for_cluster', 'remove_tag_for_cluster', 'remove_tag_for_command', 'set_application_for_command', 'set_commands_for_cluster', 'submit_job', 'update_application', 'update_cluster', 'update_command', 'update_commands_for_cluster', 'update_configs_for_application', 'update_configs_for_cluster', 'update_configs_for_command', 'update_dependencies_for_application', 'update_tags_for_application', 'update_tags_for_cluster', 'update_tags_for_command', ]
class TestJob(unittest.TestCase): def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/jobs.*') self.job = { "id": "5afdcab5-16df-11e6-b78d-6003088f363e", "created": "2016-05-10T18:45:29.000Z", "updated": "2016-05-10T19:06:18.000Z", "tags": [], "version": "1.0", "user": "******", "name": "test_command_name", "description": "Test command", "commandArgs": "-f presto.q", "status": "SUCCEEDED", "statusMsg": "Job finished successfully.", "started": "2016-05-10T18:45:30.000Z", "finished": "2016-05-10T19:06:18.000Z", "archiveLocation": "s3://test-bucket/test.tar.gz", "clusterName": "test_cluster", "commandName": "test_command", "runtime": "TEST005", "dependencies": [], "configs": [], } @responses.activate def test_get_job(self): responses.add(responses.GET, self.path, body=json.dumps(self.job)) job = self.genie.get_job(self.job['id']) assert_equals(job, self.job) @responses.activate def test_get_job_returns_none(self): responses.add(responses.GET, self.path, status=404) cluster = self.genie.get_job('does_not_exist') assert_equals(cluster, None) @responses.activate def test_get_job_applications(self): apps = ['app1', 'app2'] responses.add(responses.GET, self.path, body=json.dumps(apps)) new_apps = self.genie.get_job_applications(self.job['id']) assert_equals(apps, new_apps) @responses.activate def test_get_job_cluster(self): cluster = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(cluster)) new_cluster = self.genie.get_job_cluster(self.job['id']) assert_equals(cluster, new_cluster) @responses.activate def test_get_job_command(self): command = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(command)) new_command = self.genie.get_job_command(self.job['id']) assert_equals(command, new_command) @responses.activate def test_get_job_execution(self): execution = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(execution)) new_execution = self.genie.get_job_execution(self.job['id']) assert_equals(execution, new_execution) @responses.activate def test_get_job_output(self): output = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(output)) new_output = self.genie.get_job_output(self.job['id']) assert_equals(output, new_output) @responses.activate def test_get_job_request(self): request = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(request)) new_request = self.genie.get_job_request(self.job['id']) assert_equals(request, new_request) @responses.activate def test_get_job_status(self): status = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(status)) new_status = self.genie.get_job_status(self.job['id']) assert_equals(status, new_status)
class TestCluster(unittest.TestCase): def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/clusters.*') self.cluster = { "id": "presto_test_20160318", "created": "2016-04-12T19:52:10.000Z", "updated": "2016-05-09T20:52:53.000Z", "tags": ["genie.name:test", "ver:1", "type:test"], "version": "1", "user": "******", "name": "testcluster", "description": "Test Cluster", "configs": [], "dependencies": [], "setupFile": "s3://test-bucket/cluster.sh", "status": "UP", "page": { "totalPages": 1, "number": 1, }, } @responses.activate def test_get_cluster(self): responses.add(responses.GET, self.path, body=json.dumps(self.cluster)) cluster = self.genie.get_cluster(self.cluster['id']) assert_equals(cluster, self.cluster) @responses.activate def test_get_cluster_returns_none(self): responses.add(responses.GET, self.path, status=404) cluster = self.genie.get_cluster(self.cluster['id']) assert_equals(cluster, None) @responses.activate def test_get_cluster_returns_none(self): responses.add(responses.GET, self.path, status=404) cluster = self.genie.get_cluster(self.cluster['id']) assert_equals(cluster, None) @responses.activate def test_get_commands_for_cluster(self): cmds = [{'id': 'test1'}, {'id': 'test2', 'name': 'test'}] responses.add(responses.GET, self.path, body=json.dumps(cmds)) commands = self.genie.get_commands_for_cluster(self.cluster['id']) assert_equals(cmds, commands) @responses.activate def test_get_tags_for_cluster(self): responses.add(responses.GET, self.path, body='["tag:true", "test:false"]') tags = self.genie.get_tags_for_cluster(self.cluster['id']) assert len(tags) == 2, "Did not return all of the tags for cluster" assert "tag:true" in tags, "Did not return the correct tags for cluster" @responses.activate def test_add_commands_for_cluster(self): responses.add(responses.POST, self.path, status=204) self.genie.add_commands_for_cluster(self.cluster['id'], ['cmd1', 'cmd2']) @responses.activate def test_add_configs_for_cluster(self): responses.add(responses.POST, self.path, status=204) self.genie.add_configs_for_cluster(self.cluster['id'], ['c1', 'c2']) @responses.activate def test_add_dependencies_for_cluster(self): responses.add(responses.POST, self.path, status=204) self.genie.add_dependencies_for_cluster(self.cluster['id'], ['d1', 'd2']) @responses.activate def test_add_tags_for_cluster(self): responses.add(responses.POST, self.path, status=204) self.genie.add_tags_for_cluster(self.cluster['id'], ['t:1', 't:2']) @responses.activate def test_create_cluster(self): location = GENIE_URL + '/' + self.cluster['id'] responses.add(responses.POST, self.path, status=201, adding_headers={'Location': location}) cluster_id = self.genie.create_cluster(self.cluster) assert_equals(cluster_id, self.cluster['id']) @responses.activate def test_delete_all_clusters(self): responses.add(responses.DELETE, self.path, status=204) self.genie.delete_all_clusters() @responses.activate def test_delete_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.delete_cluster(self.cluster['id']) @responses.activate def test_patch_cluster(self): patch1 = {'op': 'replace', 'path': '/version', 'value': '1.1'} patch2 = {'op': 'replace', 'path': '/user', 'value': 'test'} responses.add(responses.PATCH, self.path) self.genie.patch_cluster(self.cluster['id'], [patch1, patch2]) @responses.activate def test_remove_all_commands_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_commands_for_cluster(self.cluster['id']) @responses.activate def test_remove_all_configs_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_configs_for_cluster(self.cluster['id']) @responses.activate def test_remove_all_dependencies_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_dependencies_for_cluster(self.cluster['id']) @responses.activate def test_remove_all_tags_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_tags_for_cluster(self.cluster['id']) @responses.activate def test_remove_commands_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_commands_for_cluster(self.cluster['id']) @responses.activate def test_remove_tag_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_tag_for_cluster(self.cluster['id'], 'test:true') @responses.activate def test_set_commands_for_cluster(self): responses.add(responses.PUT, self.path, status=204) self.genie.set_commands_for_cluster(self.cluster['id'], ['cmd1', 'cmd2']) @responses.activate def test_update_cluster(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_cluster(self.cluster['id'], self.cluster) @responses.activate def test_update_commands_for_cluster(self): responses.add(responses.PUT, self.path, status=204) cmds = [{'id': 'test'}, {'id': 'test2'}] self.genie.update_commands_for_cluster(self.cluster['id'], cmds) @responses.activate def test_update_configs_for_cluster(self): responses.add(responses.PUT, self.path, status=204) confs = [{'id': 'test'}, {'id': 'test2'}] self.genie.update_configs_for_cluster(self.cluster['id'], confs) @responses.activate def test_update_dependencies_for_cluster(self): responses.add(responses.PUT, self.path, status=204) dependencies = [{'id': 'test'}, {'id': 'test2'}] self.genie.update_dependencies_for_cluster(self.cluster['id'], dependencies) @responses.activate def test_update_tags_for_cluster(self): responses.add(responses.PUT, self.path, status=204) tags = ['test:true', 'version:1.0', 'prod:false'] self.genie.update_tags_for_cluster(self.cluster['id'], tags)
class TestCommand(unittest.TestCase): def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/commands.*') self.command = { "id": "test_command", "created": "2016-05-09T20:50:56.000Z", "updated": "2016-05-11T00:18:26.000Z", "tags": ["genie.id:test_command", "type:test", "ver:1", "genie.name:test"], "version": "1", "user": "******", "name": "presto", "description": None, "configs": [], "setupFile": None, "status": "ACTIVE", "executable": "sleep 1", "checkDelay": 5000, "page": { "totalPages": 1, "number": 1, }, } @responses.activate def test_create_command(self): location = GENIE_URL + '/' + self.command['id'] responses.add(responses.POST, self.path, status=201, adding_headers={'Location': location}) new_command_name = self.genie.create_command(self.command) assert_equals(new_command_name, self.command['id']) @responses.activate def test_patch_command(self): patch1 = {'op': 'replace', 'path': '/version', 'value': '1.1'} patch2 = {'op': 'replace', 'path': '/user', 'value': 'test'} responses.add(responses.PATCH, self.path) self.genie.patch_command(self.command['id'], [patch1, patch2]) @responses.activate def test_get_command(self): "Testing that get_commands returns all required keys" responses.add(responses.GET, self.path, body=json.dumps(self.command)) command = self.genie.get_command(self.command['id']) assert_equals(command, self.command) @responses.activate def test_get_command_returns_none(self): responses.add(responses.GET, self.path, status=404) cluster = self.genie.get_command('does_not_exist') assert_equals(cluster, None) @responses.activate def test_get_command_with_params(self): resp = { "_embedded": {"commandList": [self.command]}, "page": {"totalPages": 0, "number": 0} } resp = json.dumps(resp) responses.add(responses.GET, self.path, body=resp) commands = [i for i in self.genie.get_commands( filters={'name': self.command['name']})] assert len(commands) == 1, "Didn't return the correct amount of commands" assert_equals(commands[0], self.command) @responses.activate def test_update_command(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_command(self.command) @responses.activate def test_delete_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.delete_command(self.command['id']) @responses.activate def test_create_and_delete_all_commands(self): responses.add(responses.DELETE, self.path, status=204) self.genie.delete_all_commands() @responses.activate def test_add_applications_for_command(self): responses.add(responses.POST, self.path, status=204) self.genie.add_applications_for_command(self.command['id'], ['test']) @responses.activate def test_add_configs_for_command(self): responses.add(responses.POST, self.path, status=204) self.genie.add_configs_for_command(self.command['id'], ['test']) @responses.activate def test_add_tags_for_command(self): responses.add(responses.POST, self.path, status=204) self.genie.add_tags_for_command(self.command['id'], ['tag:test']) @responses.activate def test_get_applications_for_command(self): responses.add(responses.GET, self.path, body='["app1", "app2"]') apps = self.genie.get_applications_for_command('test') assert len(apps) == 2, "Did not return enough apps for command" @responses.activate def test_get_clusters_for_command(self): responses.add(responses.GET, self.path, body='["test1", "test2"]') clusters = self.genie.get_clusters_for_command(self.command['id']) assert len(clusters) == 2, "Did not return enough commands" assert "test1" in clusters, "Did not return the right clusters" @responses.activate def test_get_configs_for_command(self): responses.add(responses.GET, self.path, body='["conf1", "conf2"]') configs = self.genie.get_configs_for_command(self.command['id']) assert len(configs) == 2, "Did not return enough configs" assert "conf1" in configs, "did not return the right configs" @responses.activate def test_get_tags_for_command(self): responses.add(responses.GET, self.path, body='["tag:1", "tag:2"]') tags = self.genie.get_tags_for_command(self.command['id']) assert len(tags) == 2, "Did not return enough tags" assert "tag:1" in tags, "Did not return the right tags" @responses.activate def test_remove_all_applications_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_applications_for_command(self.command['id']) @responses.activate def test_remove_all_configs_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_configs_for_command(self.command['id']) @responses.activate def test_remove_all_tags_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_tags_for_command(self.command['id']) @responses.activate def test_remove_application_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_application_for_command(self.command['id'], 'test') @responses.activate def test_remove_tags_for_command(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_tags_for_command(self.command['id'], 'test') @responses.activate def test_set_application_for_command(self): responses.add(responses.PUT, self.path, status=204) self.genie.set_application_for_command('test', ['app1', 'app2']) @responses.activate def test_update_configs_for_command(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_configs_for_command('test', ['conf1', 'conf2']) @responses.activate def test_update_tags_for_command(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_tags_for_command('test', ['tag:test', 'tag2:test2'])
import yaml from pygenie.client import Genie from pygenie.conf import GenieConf logging.basicConfig(level=logging.WARNING) LOGGER = logging.getLogger(__name__) def load_yaml(yaml_file): with open(yaml_file) as _file: return yaml.load(_file) genie_conf = GenieConf() genie_conf.genie.url = "http://genie:8080" genie = Genie(genie_conf) hadoop_application = load_yaml("applications/hadoop271.yml") hadoop_application_id = genie.create_application(hadoop_application) LOGGER.warn("Created Hadoop 2.7.1 application with id = [%s]" % hadoop_application_id) spark_163_application = load_yaml("applications/spark163.yml") spark_163_application_id = genie.create_application(spark_163_application) LOGGER.warn("Created Spark 1.6.3 application with id = [%s]" % spark_163_application_id) spark_201_application = load_yaml("applications/spark201.yml") spark_201_application_id = genie.create_application(spark_201_application) LOGGER.warn("Created Spark 2.0.1 application with id = [%s]" % spark_201_application_id) hadoop_command = load_yaml("commands/hadoop271.yml") hadoop_command_id = genie.create_command(hadoop_command)
# See the License for the specific language governing permissions and # limitations under the License. import logging from pygenie.client import Genie from pygenie.conf import GenieConf logging.basicConfig(level=logging.WARNING) LOGGER = logging.getLogger(__name__) genie_conf = GenieConf() genie_conf.genie.url = "http://genie:8080" genie = Genie(genie_conf) LOGGER.warn( "This script will simulate moving production tags to the test cluster in case of issue" ) LOGGER.warn("Beginning process of moving production load to the test cluster") # Add the sched:sla tag to the test cluster clusters = genie.get_clusters(filters={"name": "GenieDemoTest"}) for cluster in clusters: cluster_id = cluster["id"] LOGGER.warn( "Adding sched:sla tag to GenieDemoTest cluster with id = [%s]" % cluster_id) LOGGER.warn("Tags before = [%s]" % cluster["tags"]) genie.add_tags_for_cluster(cluster_id, ["sched:sla"])
# See the License for the specific language governing permissions and # limitations under the License. import logging from pygenie.client import Genie from pygenie.conf import GenieConf logging.basicConfig(level=logging.WARNING) LOGGER = logging.getLogger(__name__) genie_conf = GenieConf() genie_conf.genie.url = "http://genie:8080" genie = Genie(genie_conf) LOGGER.warn("This script will simulate moving production tags to the test cluster in case of issue") LOGGER.warn("Beginning process of moving production load to the test cluster") # Add the sched:sla tag to the test cluster clusters = genie.get_clusters(filters={"name": "GenieDemoTest"}) for cluster in clusters: cluster_id = cluster["id"] LOGGER.warn("Adding sched:sla tag to GenieDemoTest cluster with id = [%s]" % cluster_id) LOGGER.warn("Tags before = [%s]" % cluster["tags"]) genie.add_tags_for_cluster(cluster_id, ["sched:sla"]) LOGGER.warn("Tags after = [%s]" % genie.get_cluster(cluster_id)["tags"]) # Remove the sched:sla tag from the prod cluster clusters = genie.get_clusters(filters={"name": "GenieDemoProd"})
class TestApplication(unittest.TestCase): def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/applications.*') self.application = { "id": "test", "created": "2016-05-10T23:37:24.000Z", "updated": "2016-05-11T00:18:23.000Z", "tags": ["type:hadoop", "genie.id:test", "genie.name:test"], "version": "1", "user": "******", "name": "test", "description": "Test command", "configs": [], "setupFile": "s3://test-bucket/setup.sh", "dependencies": ["s3://test-bucket/test.tgz"], "status": "ACTIVE", "page": { "totalPages": 1, "number": 1, }, } @responses.activate def test_create_application(self): location = GENIE_URL + '/' + self.application['id'] responses.add(responses.POST, self.path, status=201, adding_headers={'Location': location}) app_id = self.genie.create_application(self.application) assert_equals(app_id, self.application['id']) @responses.activate def test_delete_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.delete_application(self.application['id']) @responses.activate def test_remove_all_application_configs(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_application_configs(self.application['id']) @responses.activate def test_update_application(self): responses.add(responses.PUT, self.path, status=204) self.genie.delete_application(self.application['id'], self.application) @responses.activate def test_patch_application(self): patch1 = {'op': 'replace', 'path': '/version', 'value': '1.1'} patch2 = {'op': 'replace', 'path': '/user', 'value': 'test'} responses.add(responses.PATCH, self.path) self.genie.patch_application(self.application['id'], [patch1, patch2]) @responses.activate def test_get_application(self): responses.add(responses.GET, self.path, body=json.dumps(self.application)) app = self.genie.get_application(self.application['id']) assert_equals(app, self.application) @responses.activate def test_get_application_returns_none(self): responses.add(responses.GET, self.path, status=404) cluster = self.genie.get_application('does_not_exist') assert_equals(cluster, None) @responses.activate def test_tags_for_application(self): tags = ['test:true', 'region:us-east-1', 'tag:very-long-tag-name'] responses.add(responses.GET, self.path, body=json.dumps(tags)) new_tags = self.genie.get_application(self.application['id']) assert_equals(tags, new_tags) @responses.activate def test_update_application(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_application(self.application) @responses.activate def test_update_configs_for_application(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_configs_for_application(self.application['id'], ['conf1', 'conf2']) @responses.activate def test_update_dependencies_for_application(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_dependencies_for_application(self.application['id'], ['dep1', 'dep2']) @responses.activate def test_update_tags_for_application(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_tags_for_application(self.application['id'], ['tag:test', 'genie:true']) @responses.activate def test_add_dependencies_for_application(self): responses.add(responses.POST, self.path, status=204) self.genie.add_dependencies_for_application(self.application['id'], ['conf1', 'conf2']) @responses.activate def test_add_tags_for_application(self): responses.add(responses.POST, self.path, status=204) self.genie.add_tags_for_application(self.application['id'], ['tag:test', 'genie:true']) @responses.activate def test_remove_all_dependencies_for_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_dependencies_for_application(self.application['id']) @responses.activate def test_remove_all_tags_for_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_tags_for_application(self.application['id']) @responses.activate def test_remove_tag_for_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_tag_for_application(self.application['id'], ['tag:true']) @responses.activate def test_get_commands_for_application(self): cmds = [{'id': 'test1'}, {'id':'test2', 'name':'test'}] responses.add(responses.GET, self.path, body=json.dumps(cmds)) commands = self.genie.get_commands_for_application(self.application['id']) assert_equals(cmds, commands) @responses.activate def test_get_configs_for_application(self): confs = [{'id': 'test1'}, {'id':'test2', 'name':'test'}] responses.add(responses.GET, self.path, body=json.dumps(confs)) configs = self.genie.get_configs_for_application(self.application['id']) assert_equals(confs, configs) @responses.activate def test_get_dependencies_for_application(self): deps = [{'id': 'test1'}, {'id':'test2', 'name':'test'}] responses.add(responses.GET, self.path, body=json.dumps(deps)) dependencies = self.genie.get_dependencies_for_application(self.application['id']) assert_equals(deps, dependencies)
class TestCluster(unittest.TestCase): def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/clusters.*') self.cluster = { "id": "presto_test_20160318", "created": "2016-04-12T19:52:10.000Z", "updated": "2016-05-09T20:52:53.000Z", "tags": ["genie.name:test", "ver:1", "type:test"], "version": "1", "user": "******", "name": "testcluster", "description": "Test Cluster", "configs": [], "setupFile": "s3://test-bucket/cluster.sh", "status": "UP", "page": { "totalPages": 1, "number": 1, }, } @responses.activate def test_get_cluster(self): responses.add(responses.GET, self.path, body=json.dumps(self.cluster)) cluster = self.genie.get_cluster(self.cluster['id']) assert_equals(cluster, self.cluster) @responses.activate def test_get_cluster_returns_none(self): responses.add(responses.GET, self.path, status=404) cluster = self.genie.get_cluster(self.cluster['id']) assert_equals(cluster, None) @responses.activate def test_get_cluster_returns_none(self): responses.add(responses.GET, self.path, status=404) cluster = self.genie.get_cluster(self.cluster['id']) assert_equals(cluster, None) @responses.activate def test_get_commands_for_cluster(self): cmds = [{'id': 'test1'}, {'id':'test2', 'name':'test'}] responses.add(responses.GET, self.path, body=json.dumps(cmds)) commands = self.genie.get_commands_for_cluster(self.cluster['id']) assert_equals(cmds, commands) @responses.activate def test_get_tags_for_cluster(self): responses.add(responses.GET, self.path, body='["tag:true", "test:false"]') tags = self.genie.get_tags_for_cluster(self.cluster['id']) assert len(tags) == 2, "Did not return all of the tags for cluster" assert "tag:true" in tags, "Did not return the correct tags for cluster" @responses.activate def test_add_commands_for_cluster(self): responses.add(responses.POST, self.path, status=204) self.genie.add_commands_for_cluster(self.cluster['id'], ['cmd1', 'cmd2']) @responses.activate def test_add_configs_for_cluster(self): responses.add(responses.POST, self.path, status=204) self.genie.add_configs_for_cluster(self.cluster['id'], ['c1', 'c2']) @responses.activate def test_add_tags_for_cluster(self): responses.add(responses.POST, self.path, status=204) self.genie.add_tags_for_cluster(self.cluster['id'], ['t:1', 't:2']) @responses.activate def test_create_cluster(self): location = GENIE_URL + '/' + self.cluster['id'] responses.add(responses.POST, self.path, status=201, adding_headers={'Location': location}) cluster_id = self.genie.create_cluster(self.cluster) assert_equals(cluster_id, self.cluster['id']) @responses.activate def test_delete_all_clusters(self): responses.add(responses.DELETE, self.path, status=204) self.genie.delete_all_clusters() @responses.activate def test_delete_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.delete_cluster(self.cluster['id']) @responses.activate def test_patch_cluster(self): patch1 = {'op': 'replace', 'path': '/version', 'value': '1.1'} patch2 = {'op': 'replace', 'path': '/user', 'value': 'test'} responses.add(responses.PATCH, self.path) self.genie.patch_cluster(self.cluster['id'], [patch1, patch2]) @responses.activate def test_remove_all_commands_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_commands_for_cluster(self.cluster['id']) @responses.activate def test_remove_all_configs_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_configs_for_cluster(self.cluster['id']) @responses.activate def test_remove_all_tags_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_tags_for_cluster(self.cluster['id']) @responses.activate def test_remove_commands_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_commands_for_cluster(self.cluster['id']) @responses.activate def test_remove_tag_for_cluster(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_tag_for_cluster(self.cluster['id'], 'test:true') @responses.activate def test_set_commands_for_cluster(self): responses.add(responses.PUT, self.path, status=204) self.genie.set_commands_for_cluster(self.cluster['id'], ['cmd1', 'cmd2']) @responses.activate def test_update_cluster(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_cluster(self.cluster['id'], self.cluster) @responses.activate def test_update_commands_for_cluster(self): responses.add(responses.PUT, self.path, status=204) cmds = [{'id': 'test'}, {'id': 'test2'}] self.genie.update_commands_for_cluster(self.cluster['id'], cmds) @responses.activate def test_update_configs_for_cluster(self): responses.add(responses.PUT, self.path, status=204) confs = [{'id': 'test'}, {'id': 'test2'}] self.genie.update_configs_for_cluster(self.cluster['id'], confs) @responses.activate def test_update_tags_for_cluster(self): responses.add(responses.PUT, self.path, status=204) tags = ['test:true', 'version:1.0', 'prod:false'] self.genie.update_tags_for_cluster(self.cluster['id'], tags)
class TestJob(unittest.TestCase): def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/jobs.*') self.job = { "id": "5afdcab5-16df-11e6-b78d-6003088f363e", "created": "2016-05-10T18:45:29.000Z", "updated": "2016-05-10T19:06:18.000Z", "tags": [], "version": "1.0", "user": "******", "name": "test_command_name", "description": "Test command", "commandArgs": "-f presto.q", "status": "SUCCEEDED", "statusMsg": "Job finished successfully.", "started": "2016-05-10T18:45:30.000Z", "finished": "2016-05-10T19:06:18.000Z", "archiveLocation": "s3://test-bucket/test.tar.gz", "clusterName": "test_cluster", "commandName": "test_command", "runtime": "TEST005", } @responses.activate def test_get_job(self): responses.add(responses.GET, self.path, body=json.dumps(self.job)) job = self.genie.get_job(self.job['id']) assert_equals(job, self.job) @responses.activate def test_get_job_returns_none(self): responses.add(responses.GET, self.path, status=404) cluster = self.genie.get_job('does_not_exist') assert_equals(cluster, None) @responses.activate def test_get_job_applications(self): apps = ['app1', 'app2'] responses.add(responses.GET, self.path, body=json.dumps(apps)) new_apps = self.genie.get_job_applications(self.job['id']) assert_equals(apps, new_apps) @responses.activate def test_get_job_cluster(self): cluster = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(cluster)) new_cluster = self.genie.get_job_cluster(self.job['id']) assert_equals(cluster, new_cluster) @responses.activate def test_get_job_command(self): command = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(command)) new_command = self.genie.get_job_command(self.job['id']) assert_equals(command, new_command) @responses.activate def test_get_job_execution(self): execution = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(execution)) new_execution = self.genie.get_job_execution(self.job['id']) assert_equals(execution, new_execution) @responses.activate def test_get_job_output(self): output = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(output)) new_output = self.genie.get_job_output(self.job['id']) assert_equals(output, new_output) @responses.activate def test_get_job_request(self): request = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(request)) new_request = self.genie.get_job_request(self.job['id']) assert_equals(request, new_request) @responses.activate def test_get_job_status(self): status = {'id': 'test'} responses.add(responses.GET, self.path, body=json.dumps(status)) new_status = self.genie.get_job_status(self.job['id']) assert_equals(status, new_status) @responses.activate def test_patch_job(self): patch1 = {'op': 'replace', 'path': '/version', 'value': '1.1'} patch2 = {'op': 'replace', 'path': '/user', 'value': 'test'} responses.add(responses.PATCH, self.path) self.genie.patch_job(self.job['id'], [patch1, patch2])
[hadoop_app_id, app_id]) LOGGER.info( f"Set applications for Spark Shell {human_version} command to = {','.join([hadoop_app_id, app_id])}" ) genie_client.set_application_for_command(submit_command_id, [hadoop_app_id, app_id]) LOGGER.info( f"Set applications for Spark Submit {human_version} command to = {','.join([hadoop_app_id, app_id])}" ) genie_conf: GenieConf = GenieConf() genie_conf.genie.url = "http://genie:8080" genie: Genie = Genie(genie_conf) hadoop_application_id: str = genie.create_application( load_yaml("applications/hadoop271.yml")) LOGGER.info( f"Created Hadoop 2.7.1 application with id = {hadoop_application_id}") hadoop_command_id: str = genie.create_command( load_yaml("commands/hadoop271.yml")) LOGGER.info(f"Created Hadoop command with id = {hadoop_command_id}") hdfs_command_id: str = genie.create_command(load_yaml("commands/hdfs271.yml")) LOGGER.info(f"Created HDFS command with id = {hdfs_command_id}") yarn_command_id: str = genie.create_command(load_yaml("commands/yarn271.yml")) LOGGER.info(f"Created Yarn command with id = {yarn_command_id}")
# See the License for the specific language governing permissions and # limitations under the License. import logging from pygenie.client import Genie from pygenie.conf import GenieConf logging.basicConfig(level=logging.WARNING) LOGGER = logging.getLogger(__name__) genie_conf = GenieConf() genie_conf.genie.url = "http://genie:8080" genie = Genie(genie_conf) LOGGER.warning( "This script will simulate moving production tags to the test cluster in case of issue" ) LOGGER.warning( "Beginning process of moving production load to the test cluster") # Add the sched:sla tag to the test cluster clusters = genie.get_clusters(filters={"name": "GenieDemoTest"}) for cluster in clusters: cluster_id = cluster["id"] LOGGER.warning( f"Adding sched:sla tag to GenieDemoTest cluster with id = {cluster_id}" ) LOGGER.warning(f"Tags before = {cluster['tags']}")
def setUp(self): self.genie = Genie(GENIE_CONF) self.cluster_name = 'test' self.path = re.compile(GENIE_URL + '/api/v3/.*')
from pygenie.conf import GenieConf logging.basicConfig(level=logging.WARNING) LOGGER = logging.getLogger(__name__) def load_yaml(yaml_file): with open(yaml_file) as _file: return yaml.load(_file) genie_conf = GenieConf() genie_conf.genie.url = "http://localhost:8080" genie = Genie(genie_conf) hadoop_application = load_yaml("applications/hadoop273.yml") hadoop_application_id = genie.create_application(hadoop_application) LOGGER.warn("Created Hadoop 2.7.3 application with id = [%s]" % hadoop_application_id) spark_application = load_yaml("applications/spark220.yml") spark_application_id = genie.create_application(spark_application) LOGGER.warn("Created spark 2.2.0 application with id = [%s]" % spark_application_id) hadoop_command = load_yaml("commands/hadoop273.yml") hadoop_command_id = genie.create_command(hadoop_command) LOGGER.warn("Created Hadoop command with id = [%s]" % hadoop_command_id)
class TestApplication(unittest.TestCase): def setUp(self): self.genie = Genie(GENIE_CONF) self.path = re.compile(GENIE_URL + '/api/v3/applications.*') self.application = { "id": "test", "created": "2016-05-10T23:37:24.000Z", "updated": "2016-05-11T00:18:23.000Z", "tags": ["type:hadoop", "genie.id:test", "genie.name:test"], "version": "1", "user": "******", "name": "test", "description": "Test command", "configs": [], "setupFile": "s3://test-bucket/setup.sh", "dependencies": ["s3://test-bucket/test.tgz"], "status": "ACTIVE", "page": { "totalPages": 1, "number": 1, }, } @responses.activate def test_create_application(self): location = GENIE_URL + '/' + self.application['id'] responses.add(responses.POST, self.path, status=201, adding_headers={'Location': location}) app_id = self.genie.create_application(self.application) assert_equals(app_id, self.application['id']) @responses.activate def test_delete_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.delete_application(self.application['id']) @responses.activate def test_remove_all_application_configs(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_configs_for_application(self.application['id']) @responses.activate def test_remove_all_configs_for_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_configs_for_application(self.application['id']) @responses.activate def test_remove_all_dependencies_for_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_dependencies_for_application( self.application['id']) @responses.activate def test_update_application(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_application(self.application) @responses.activate def test_patch_application(self): patch1 = {'op': 'replace', 'path': '/version', 'value': '1.1'} patch2 = {'op': 'replace', 'path': '/user', 'value': 'test'} responses.add(responses.PATCH, self.path) self.genie.patch_application(self.application['id'], [patch1, patch2]) @responses.activate def test_get_application(self): responses.add(responses.GET, self.path, body=json.dumps(self.application)) app = self.genie.get_application(self.application['id']) assert_equals(app, self.application) @responses.activate def test_get_application_returns_none(self): responses.add(responses.GET, self.path, status=404) cluster = self.genie.get_application('does_not_exist') assert_equals(cluster, None) @responses.activate def test_get_configs_for_application(self): configs = ['conf1', 'conf2', 'conf3'] responses.add(responses.GET, self.path, body=json.dumps(configs)) new_configs = self.genie.get_configs_for_application( self.application['id']) assert_equals(configs, new_configs) @responses.activate def test_get_dependencies_for_application(self): dependencies = ['dep1', 'dep2', 'dep3'] responses.add(responses.GET, self.path, body=json.dumps(dependencies)) new_dependencies = self.genie.get_dependencies_for_application( self.application['id']) assert_equals(dependencies, new_dependencies) @responses.activate def test_get_tags_for_application(self): tags = ['test:true', 'region:us-east-1', 'tag:very-long-tag-name'] responses.add(responses.GET, self.path, body=json.dumps(tags)) new_tags = self.genie.get_tags_for_application(self.application['id']) assert_equals(tags, new_tags) @responses.activate def test_update_application(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_application(self.application) @responses.activate def test_update_configs_for_application(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_configs_for_application(self.application['id'], ['conf1', 'conf2']) @responses.activate def test_update_dependencies_for_application(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_dependencies_for_application(self.application['id'], ['dep1', 'dep2']) @responses.activate def test_update_tags_for_application(self): responses.add(responses.PUT, self.path, status=204) self.genie.update_tags_for_application(self.application['id'], ['tag:test', 'genie:true']) @responses.activate def test_add_configs_for_application(self): responses.add(responses.POST, self.path, status=204) self.genie.add_configs_for_application(self.application['id'], ['conf1', 'conf2']) @responses.activate def test_add_dependencies_for_application(self): responses.add(responses.POST, self.path, status=204) self.genie.add_dependencies_for_application(self.application['id'], ['dep1', 'dep2']) @responses.activate def test_add_tags_for_application(self): responses.add(responses.POST, self.path, status=204) self.genie.add_tags_for_application(self.application['id'], ['tag:test', 'genie:true']) @responses.activate def test_remove_all_configs_for_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_configs_for_application(self.application['id']) @responses.activate def test_remove_all_dependencies_for_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_dependencies_for_application( self.application['id']) @responses.activate def test_remove_all_tags_for_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_all_tags_for_application(self.application['id']) @responses.activate def test_remove_tag_for_application(self): responses.add(responses.DELETE, self.path, status=204) self.genie.remove_tag_for_application(self.application['id'], 'tag:true') @responses.activate def test_get_commands_for_application(self): cmds = [{'id': 'test1'}, {'id': 'test2', 'name': 'test'}] responses.add(responses.GET, self.path, body=json.dumps(cmds)) commands = self.genie.get_commands_for_application( self.application['id']) assert_equals(cmds, commands)
from pygenie.conf import GenieConf logging.basicConfig(level=logging.WARNING) LOGGER = logging.getLogger(__name__) def load_yaml(yaml_file): with open(yaml_file) as _file: return yaml.load(_file) genie_conf = GenieConf() genie_conf.genie.url = "http://genie:8080" genie = Genie(genie_conf) hadoop_application = load_yaml("applications/hadoop271.yml") hadoop_application_id = genie.create_application(hadoop_application) LOGGER.warn("Created Hadoop 2.7.1 application with id = [%s]" % hadoop_application_id) spark_163_application = load_yaml("applications/spark163.yml") spark_163_application_id = genie.create_application(spark_163_application) LOGGER.warn("Created Spark 1.6.3 application with id = [%s]" % spark_163_application_id) spark_201_application = load_yaml("applications/spark201.yml") spark_201_application_id = genie.create_application(spark_201_application) LOGGER.warn("Created Spark 2.0.1 application with id = [%s]" % spark_201_application_id) hadoop_command = load_yaml("commands/hadoop271.yml") hadoop_command_id = genie.create_command(hadoop_command)