def init_emr_config(run_parallel=False, role=None):
cfg = ResourcesConfiguration()
emr_client = aws_common.connect_emr(role=role)
def init_emr_cluster_config(c):
if c['Status']['State'][0:10] != 'TERMINATED':
out1 = emr_client.describe_cluster(ClusterId=c['Id'])
cluster_details = out1['Cluster']
cluster = themis.model.emr_model.EmrCluster()
cluster.id = c['Id']
cluster.name = c['Name']
cluster.ip = 'N/A'
cluster.ip_public = cluster_details['MasterPublicDnsName']
has_ganglia = False
for app in out1['Cluster']['Applications']:
if app['Name'] == 'Hive' and not cluster.type:
cluster.type = 'Hive'
if app['Name'][0:6] == 'Presto':
cluster.type = 'Presto'
if app['Name'] == 'Ganglia':
has_ganglia = True
if has_ganglia:
LOG.info('Getting details for EMR cluster %s' % cluster.id)
# get private IP address of cluster
group_details = emr_client.list_instance_groups(
ClusterId=c['Id'])
for g in group_details['InstanceGroups']:
if g['InstanceGroupType'] == 'MASTER':
out2 = emr_client.list_instances(
ClusterId=c['Id'],
InstanceStates=[
'AWAITING_FULFILLMENT', 'PROVISIONING',
'BOOTSTRAPPING', 'RUNNING'
])
for inst in out2['Instances']:
if inst['InstanceGroupId'] == g['Id']:
cluster.ip = inst['PrivateDnsName']
cfg.emr.append(cluster)
else:
LOG.info('Ignoring cluster %s (Ganglia not installed)' %
cluster.id)
# load EMR resources
try:
clusters = list_all_clusters(role=role)
if run_parallel:
common.parallelize(clusters, init_emr_cluster_config)
else:
for c in clusters:
init_emr_cluster_config(c)
except Exception, e:
LOG.info('Unable to list EMR clusters using IAM role "%s"' % role)
def test_list_instances():
import constants
mock.aws_api.server.config['group_id_task_spot'] = 'group_task_spot'
mock.aws_api.server.config['group_id_task_od'] = 'group_task_od'
emr_client = aws_common.connect_emr()
out = emr_client.list_instances(ClusterId='testClusterID1')
assert len(out['Instances']) > 0
nodes = aws_common.get_cluster_nodes('testClusterID1')
assert len(nodes) > 1
def list_all_clusters(role=None):
emr_client = aws_common.connect_emr(role=role)
result = []
marker = None
for i in range(0, 10):
kwargs = {}
if marker:
kwargs['Marker'] = marker
out = emr_client.list_clusters(**kwargs)
result.extend(out['Clusters'])
marker = out.get('Marker')
if not marker:
break
return result
def list_all_clusters(role=None):
emr_client = aws_common.connect_emr(role=role)
result = []
marker = None
for i in range(0, 10):
kwargs = {}
if marker:
kwargs['Marker'] = marker
out = emr_client.list_clusters(**kwargs)
result.extend(out['Clusters'])
marker = out.get('Marker')
if not marker:
break
return result
def init_emr_config(run_parallel=False, role=None):
cfg = ResourcesConfiguration()
emr_client = aws_common.connect_emr(role=role)
def init_emr_cluster_config(c):
if c['Status']['State'][0:10] != 'TERMINATED':
out1 = emr_client.describe_cluster(ClusterId=c['Id'])
cluster_details = out1['Cluster']
cluster = themis.model.emr_model.EmrCluster()
cluster.id = c['Id']
cluster.name = c['Name']
cluster.ip = 'N/A'
cluster.ip_public = cluster_details['MasterPublicDnsName']
has_ganglia = False
for app in out1['Cluster']['Applications']:
if app['Name'] == 'Hive' and not cluster.type:
cluster.type = 'Hive'
if app['Name'][0:6] == 'Presto':
cluster.type = 'Presto'
if app['Name'] == 'Ganglia':
has_ganglia = True
if has_ganglia:
LOG.info('Getting details for EMR cluster %s' % cluster.id)
# get private IP address of cluster
group_details = emr_client.list_instance_groups(ClusterId=c['Id'])
for g in group_details['InstanceGroups']:
if g['InstanceGroupType'] == 'MASTER':
out2 = emr_client.list_instances(ClusterId=c['Id'],
InstanceStates=['AWAITING_FULFILLMENT', 'PROVISIONING', 'BOOTSTRAPPING', 'RUNNING'])
for inst in out2['Instances']:
if inst['InstanceGroupId'] == g['Id']:
cluster.ip = inst['PrivateDnsName']
cfg.emr.append(cluster)
else:
LOG.info('Ignoring cluster %s (Ganglia not installed)' % cluster.id)
# load EMR resources
try:
clusters = list_all_clusters(role=role)
if run_parallel:
common.parallelize(clusters, init_emr_cluster_config)
else:
for c in clusters:
init_emr_cluster_config(c)
except Exception, e:
LOG.info('Unable to list EMR clusters using IAM role "%s"' % role)
def test_aws_cli():
emr_client = aws_common.connect_emr()
out = emr_client.list_clusters()
assert out['Clusters'][0]['Id'] == 'testClusterID1'