def setUp(self):
self.genie = Genie(GENIE_CONF)
self.path = re.compile(GENIE_URL + '/api/v3/commands.*')
self.command = {
"id":
"test_command",
"created":
"2016-05-09T20:50:56.000Z",
"updated":
"2016-05-11T00:18:26.000Z",
"tags":
["genie.id:test_command", "type:test", "ver:1", "genie.name:test"],
"version":
"1",
"user":
"******",
"name":
"presto",
"description":
None,
"configs": [],
"dependencies": [],
"setupFile":
None,
"status":
"ACTIVE",
"executable":
"sleep 1",
"checkDelay":
5000,
"page": {
"totalPages": 1,
"number": 1,
},
}
def setUp(self):
self.genie = Genie(GENIE_CONF)
self.path = re.compile(GENIE_URL + '/api/v3/clusters.*')
self.cluster = {
"id": "presto_test_20160318",
"created": "2016-04-12T19:52:10.000Z",
"updated": "2016-05-09T20:52:53.000Z",
"tags": ["genie.name:test", "ver:1", "type:test"],
"version": "1",
"user": "******",
"name": "testcluster",
"description": "Test Cluster",
"configs": [],
"setupFile": "s3://test-bucket/cluster.sh",
"status": "UP",
"page": {
"totalPages": 1,
"number": 1,
},
}
def setUp(self):
self.genie = Genie(GENIE_CONF)
self.path = re.compile(GENIE_URL + '/api/v3/applications.*')
self.application = {
"id": "test",
"created": "2016-05-10T23:37:24.000Z",
"updated": "2016-05-11T00:18:23.000Z",
"tags": ["type:hadoop", "genie.id:test", "genie.name:test"],
"version": "1",
"user": "******",
"name": "test",
"description": "Test command",
"configs": [],
"setupFile": "s3://test-bucket/setup.sh",
"dependencies": ["s3://test-bucket/test.tgz"],
"status": "ACTIVE",
"page": {
"totalPages": 1,
"number": 1,
},
}
def setUp(self):
self.genie = Genie(GENIE_CONF)
self.path = re.compile(GENIE_URL + '/api/v3/jobs.*')
self.job = {
"id": "5afdcab5-16df-11e6-b78d-6003088f363e",
"created": "2016-05-10T18:45:29.000Z",
"updated": "2016-05-10T19:06:18.000Z",
"tags": [],
"version": "1.0",
"user": "******",
"name": "test_command_name",
"description": "Test command",
"commandArgs": "-f presto.q",
"status": "SUCCEEDED",
"statusMsg": "Job finished successfully.",
"started": "2016-05-10T18:45:30.000Z",
"finished": "2016-05-10T19:06:18.000Z",
"archiveLocation": "s3://test-bucket/test.tar.gz",
"clusterName": "test_cluster",
"commandName": "test_command",
"runtime": "TEST005",
}
import yaml
from pygenie.client import Genie
from pygenie.conf import GenieConf
logging.basicConfig(level=logging.WARNING)
LOGGER = logging.getLogger(__name__)
def load_yaml(yaml_file):
with open(yaml_file) as _file:
return yaml.load(_file)
genie_conf = GenieConf()
genie_conf.genie.url = "http://genie:8080"
genie = Genie(genie_conf)
hadoop_application = load_yaml("applications/hadoop271.yml")
hadoop_application_id = genie.create_application(hadoop_application)
LOGGER.warn("Created Hadoop 2.7.1 application with id = [%s]" % hadoop_application_id)
spark_163_application = load_yaml("applications/spark163.yml")
spark_163_application_id = genie.create_application(spark_163_application)
LOGGER.warn("Created Spark 1.6.3 application with id = [%s]" % spark_163_application_id)
spark_201_application = load_yaml("applications/spark201.yml")
spark_201_application_id = genie.create_application(spark_201_application)
LOGGER.warn("Created Spark 2.0.1 application with id = [%s]" % spark_201_application_id)
hadoop_command = load_yaml("commands/hadoop271.yml")
hadoop_command_id = genie.create_command(hadoop_command)
def setUp(self):
self.genie = Genie(GENIE_CONF)
self.required_apis = [
'add_applications_for_command',
'add_commands_for_cluster',
'add_configs_for_application',
'add_configs_for_cluster',
'add_configs_for_command',
'add_dependencies_for_application',
'add_dependencies_for_cluster',
'add_dependencies_for_command',
'add_tags_for_application',
'add_tags_for_cluster',
'add_tags_for_command',
'create_application',
'create_cluster',
'create_command',
'delete_all_applications',
'delete_all_clusters',
'delete_all_commands',
'delete_application',
'delete_cluster',
'delete_command',
'get_application',
'get_applications',
'get_applications_for_command',
'get_cluster',
'get_clusters',
'get_clusters_for_command',
'get_command',
'get_commands',
'get_commands_for_cluster',
'get_configs_for_application',
'get_configs_for_cluster',
'get_configs_for_command',
'get_dependencies_for_application',
'get_dependencies_for_cluster',
'get_dependencies_for_command',
'get_job_applications',
'get_job_cluster',
'get_job_command',
'get_job_execution',
'get_job_output',
'get_job_request',
'get_job_status',
'get_tags_for_application',
'get_tags_for_cluster',
'get_tags_for_command',
'patch_application',
'patch_cluster',
'patch_command',
'remove_all_applications_for_command',
'remove_all_commands_for_cluster',
'remove_all_configs_for_application',
'remove_all_configs_for_cluster',
'remove_all_configs_for_command',
'remove_all_dependencies_for_application',
'remove_all_dependencies_for_cluster',
'remove_all_dependencies_for_command',
'remove_all_tags_for_application',
'remove_all_tags_for_cluster',
'remove_all_tags_for_command',
'remove_application_for_command',
'remove_commands_for_cluster',
'remove_tag_for_application',
'remove_tag_for_cluster',
'remove_tag_for_cluster',
'remove_tag_for_command',
'set_application_for_command',
'set_commands_for_cluster',
'submit_job',
'update_application',
'update_cluster',
'update_command',
'update_commands_for_cluster',
'update_configs_for_application',
'update_configs_for_cluster',
'update_configs_for_command',
'update_dependencies_for_application',
'update_tags_for_application',
'update_tags_for_cluster',
'update_tags_for_command',
]
def setUp(self):
self.genie = Genie(GENIE_CONF)
self.cluster_name = 'test'
self.path = re.compile(GENIE_URL + '/api/v3/.*')
