def command_test02(args):
yaml = get_file_contents('dfms_cloud_init.yaml')
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
yaml = yaml.format(
'aws-chiles02',
boto_data[0],
boto_data[1],
)
user_data = MIMEMultipart()
user_data.attach(MIMEText(yaml))
user_data.attach(MIMEText('''#!/bin/bash -vx
df -h
'''))
encode = user_data.as_string().encode("ascii")
encoded_data = base64.b64encode(encode).decode('ascii')
ec2_controller = EC2Controller(AWS_AMI_ID, [{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': 0.03
}],
encoded_data,
'us-west-2',
tags=[{
'Key': 'Owner',
'Value': 'Kevin',
}, {
'Key': 'Name',
'Value': 'Test02',
}])
ec2_controller.start_instances()
def command_test01(args):
ec2_controller = EC2Controller(AWS_AMI_ID, [{
'number_instances': 2,
'instance_type': 'm4.large',
'spot_price': 0.03
}],
'',
'us-west-2',
tags=[{
'Key': 'Owner',
'Value': 'Kevin',
}, {
'Key': 'Name',
'Value': 'Test01',
}])
ec2_controller.start_instances()
def command_test03(args):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
cloud_init = get_file_contents('dfms_cloud_init.yaml')
cloud_init_dynamic = '''#cloud-config
# Write the boto file
write_files:
- path: "/root/.aws/credentials"
permissions: "0544"
owner: "root"
content: |
[{0}]
aws_access_key_id = {1}
aws_secret_access_key = {2}
- path: "/home/ec2-user/.aws/credentials"
permissions: "0544"
owner: "ec2-user:ec2-user"
content: |
[{0}]
aws_access_key_id = {1}
aws_secret_access_key = {2}
'''.format(
'aws-chiles02',
boto_data[0],
boto_data[1],
)
user_script = get_file_contents('node_manager_start_up.bash')
dynamic_script = '#!/bin/bash -vx\n' \
'runuser -l ec2-user -c \'cd /home/ec2-user/aws-chiles02/pipeline/aws_chiles02 && source /home/ec2-user/virtualenv/aws-chiles02/bin/activate && ' \
'python startup_complete.py startup_complete us-west-2 "Hello world"\''
user_data = get_user_data(
[cloud_init, cloud_init_dynamic, user_script, dynamic_script])
ec2_controller = EC2Controller(AWS_AMI_ID, [{
'number_instances': 1,
'instance_type': 'i2.xlarge',
'spot_price': 0.10
}],
user_data,
'us-west-2',
tags=[{
'Key': 'Owner',
'Value': 'Kevin',
}, {
'Key': 'Name',
'Value': 'Test03',
}])
ec2_controller.start_instances()
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price, volume, bottom_frequency, nodes, add_shutdown):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
[
{
'number_instances': nodes,
'instance_type': 'i2.2xlarge',
'spot_price': spot_price
}
],
get_node_manager_user_data(boto_data, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Daliuge NM - Find bad MS',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
nodes,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Daliuge DIM - Find bad MS',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphFindBadMeasurementSet(
bucket_name,
volume,
PARALLEL_STREAMS,
reported_running,
add_shutdown,
frequency_width,
bottom_frequency,
session_id,
host,
)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price,
volume, nodes, add_shutdown, password, log_level):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
database_connection, map_day_name, database_ip = setup_database(
password, bucket_name)
work_to_do = WorkToDo(frequency_width, bucket_name,
get_s3_uvsub_name(frequency_width),
database_connection)
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(nodes, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(ami_id,
nodes_required,
get_node_manager_user_data(
boto_data,
uuid,
max_request_size=50,
log_level=log_level),
AWS_REGION,
tags=[{
'Key': 'Owner',
'Value': getpass.getuser(),
}, {
'Key': 'Name',
'Value': 'Daliuge NM - Stats',
}, {
'Key': 'uuid',
'Value': uuid,
}])
ec2_data.start_instances()
reported_running = get_reported_running(uuid, node_count, wait=600)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id, [{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}],
get_data_island_manager_user_data(boto_data,
hosts,
uuid,
need_node_manager=True,
max_request_size=50),
AWS_REGION,
tags=[{
'Key': 'Owner',
'Value': getpass.getuser(),
}, {
'Key': 'Name',
'Value': 'Daliuge DIM - Stats',
}, {
'Key': 'uuid',
'Value': uuid,
}])
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600,
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][
0]
host = instance_details['ip_address']
graph = BuildGraphStats(work_to_do.work_to_do, bucket_name,
volume, PARALLEL_STREAMS,
reported_running, add_shutdown,
frequency_width, session_id,
map_day_name, password,
database_ip, host)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.error('Unable to find the AWS credentials')