def check_aws_provider(yaml):
provider_name = yaml.get("name")
print("Checking integrity of", provider_name, "with type:",
yaml.get("type"))
cred_environment = aws_cred_file_environment(yaml["aws_cred_file"])
runner = ansible_runner.run(playbook='aws_setup_checks.yml',
private_data_dir='ansible',
extravars={
"access_key_id":
cred_environment["AWS_ACCESS_KEY_ID"],
"access_key_secret":
cred_environment["AWS_SECRET_ACCESS_KEY"],
},
quiet=True)
events = [e for e in runner.events]
if runner.status == "failed":
printout_ansible_events(events)
print("Failed to mount the AWS S3 filesystem")
return False
print("Mount successful")
return True
def __init__(self, provider_info):
super().__init__(provider_info)
self.provider_type = "aws"
self.zone = provider_info["aws_zone"]
provider_info["zone"] = provider_info["aws_zone"]
self.provider_info = provider_info
for key, value in provider_info.items():
if value is not None:
self.raw_provider_info[key] = value
logger.info("AWS Cloud Handler Instantiating {}".format(self.name))
if "aws_cred_file" not in provider_info:
logger.error("Failed to provider aws_cred_file for account")
raise ValueError("Failed to provider aws_cred_file for account")
self.credential_file = provider_info["aws_cred_file"]
cred_environment = aws_cred_file_environment(self.credential_file)
for key, value in cred_environment.items():
os.environ[key] = value
if self.check_filesystem_mounted() == False:
print("Filesystem mount not found")
print("Remounting filesystem.... ")
if self.check_provider() == False or self.check_filesystem_mounted(
):
print("Failed to remount filesystem")
return None
else:
print("Filesystem remounted successfully!")
def check_provider(self):
cred_environment = aws_cred_file_environment(
self.provider_info["aws_cred_file"])
runner = ansible_runner.run(
playbook='aws_setup_checks.yml',
private_data_dir='ansible',
extravars={
"access_key_id": cred_environment["AWS_ACCESS_KEY_ID"],
"access_key_secret": cred_environment["AWS_SECRET_ACCESS_KEY"],
},
quiet=True)
events = [e for e in runner.events]
if runner.status == "failed":
print("Failed to mount the AWS S3 filesystem")
return False
print("Mount successful")
return True
def mount_aws_monkeyfs(yaml):
bucket_name = yaml["storage_name"]
local_mount_point = yaml["local_monkeyfs_path"]
print("Attempting to mount gcs bucket: {} to {}".format(
bucket_name, local_mount_point))
cred_environment = aws_cred_file_environment(yaml["aws_cred_file"])
runner = ansible_runner.run(playbook='aws_setup_checks.yml',
private_data_dir='ansible',
extravars={
"access_key_id":
cred_environment["AWS_ACCESS_KEY_ID"],
"access_key_secret":
cred_environment["AWS_SECRET_ACCESS_KEY"],
},
quiet=False)
if runner.status == "failed":
print("Failed to mount the AWS S3 filesystem")
return False
print("Mount successful")
return True
def create_aws_provider(provider_name, provider_yaml, args):
provider_type = "aws"
aws_key_file = args.identification_file
ssh_key_name = args.ssh_key_name
region_input = args.region
zone_input = args.zone
local_monkeyfs_path = os.path.join(os.getcwd(), "ansible/monkeyfs-aws")
monkeyfs_path = "/monkeyfs"
aws_storage_name = args.storage_name or generate_random_monkeyfs_name()
if not check_for_existing_local_command("s3fs"):
print("You must have s3fs installed.\n " +
"To install please follow the instructions here:\n" +
("https://github.com/s3fs-fuse/s3fs-fuse"))
exit(1)
cred_environment = None
if aws_key_file is not None:
aws_key_file = os.path.abspath(aws_key_file)
try:
cred_environment = aws_cred_file_environment(aws_key_file)
valid = True
except:
print("Failed to read file")
if aws_key_file is None or cred_environment is None:
if args.noinput:
raise ValueError(
"Please input the identity-file (aws credential key file)")
while True:
aws_key_file = input(
"AWS Account File (should have Access key ID and Secret Access Key in csv form)\n"
+ "Key: ").strip()
aws_key_file = os.path.abspath(aws_key_file)
try:
cred_environment = aws_cred_file_environment(aws_key_file)
break
except:
print("Failed to read and parse credentials")
def get_input_with_defaults(prompt_phrase, prompt_name, default_value,
noinput):
if noinput:
print(
f"{prompt_name} not provider. Defaulting to: {default_value}")
return default_value
return input(f"{prompt_phrase} ({default_value}): ") or default_value
if not args.region:
region_input = get_input_with_defaults(prompt_phrase="Set AWS region",
prompt_name="AWS Region",
default_value="us-east-1",
noinput=args.noinput)
if not args.zone:
zone_input = get_input_with_defaults(prompt_phrase="Set AWS Zone",
prompt_name="AWS Zone",
default_value=region_input + "a",
noinput=args.noinput)
if not args.ssh_key_name:
ssh_key_name = get_input_with_defaults(
prompt_phrase="Set AWS SSH Key Name",
prompt_name="AWS ssh key name",
default_value="monkey_aws",
noinput=args.noinput)
if not args.storage_name:
filesystem_ok = False
while not filesystem_ok:
if not args.noinput and not args.storage_name:
aws_storage_name = input(
"Set the monkey_fs aws s3 bucket name ({})".format(
aws_storage_name)) or aws_storage_name
filesystem_ok = create_aws_monkeyfs(
storage_name=aws_storage_name,
cred_environment=cred_environment,
region=region_input)
if not filesystem_ok:
if not args.storage_name:
print(f"Failed creating bucket: {aws_storage_name}\n" +
"Ensure storage_name is unique and AWS allowed")
sys.exit(1)
aws_storage_name = generate_random_monkeyfs_name()
print(f"Aws Credenitals File: {aws_key_file}")
print(f"Aws Region: {region_input}")
print(f"Aws Zone: {zone_input}")
print(f"Aws Key Name: {ssh_key_name}")
print(f"Aws Bucket Name: {aws_storage_name}")
aws_vars = round_trip_load(
str({
"name": provider_name,
"type": provider_type,
"aws_region": region_input,
"aws_zone": zone_input,
"aws_cred_file": aws_key_file,
"aws_key_name": ssh_key_name,
"storage_name": aws_storage_name,
"local_monkeyfs_path": local_monkeyfs_path,
"monkeyfs_path": monkeyfs_path,
"firewall_rule": "monkey-ansible-firewall",
}))
aws_vars.fa.set_block_style()
aws_vars.yaml_set_start_comment("\nAWS Provider: {}".format(provider_name))
aws_vars.yaml_add_eol_comment("Used for mounting filesystems",
"aws_cred_file")
aws_vars.yaml_set_comment_before_after_key(
"storage_name", before="\n\n###########\n# Optional\n###########")
aws_vars.yaml_add_eol_comment("Defaults to monkeyfs-XXXXXX",
"storage_name")
aws_vars.yaml_add_eol_comment("Defaults to /monkeyfs", "monkeyfs_path")
# Create filesystem bucket and pick a new id if failed
providers = provider_yaml.get("providers", [])
if providers is None:
providers = []
providers.append(aws_vars)
provider_yaml["providers"] = providers
print("\nWriting to providers.yml...")
with open('providers.yml', 'w') as file:
y = YAML()
provider_yaml.fa.set_block_style()
y.explicit_start = True
y.default_flow_style = False
y.dump(provider_yaml, file)
print("\nWriting aws vars file...")
write_vars_file(aws_vars)
# Creation of FS OK, now mounting FS to local mount point
if not mount_aws_monkeyfs(aws_vars):
print(
"Terminating, please ensure you have s3fs installed on the core machine"
)
exit(1)
print("\nWriting ansible inventory file...")
aws_inventory = round_trip_load(
str({
"aws_access_key": cred_environment["AWS_ACCESS_KEY_ID"],
"aws_secret_key": cred_environment["AWS_SECRET_ACCESS_KEY"],
"plugin": "aws_ec2",
"regions": [region_input],
"groups": {
"monkey": "'monkey' in tags.Name",
"monkey_aws": "'monkey' in tags.Name",
},
"hostnames": ["tag:Name"],
"filters": {
"tag:Monkey": "Monkey_AWS"
},
"compose": {
"ansible_host": "public_ip_address",
}
}))
aws_inventory.fa.set_block_style()
write_inventory_file(aws_inventory)
def setup_job(self, job, provider_info=dict()):
print("Setting up job: ", job)
job_uid = job["job_uid"]
credential_file = provider_info.get("aws_cred_file", None)
aws_storage_name = provider_info["storage_name"]
monkeyfs_path = provider_info.get("monkeyfs_path", "/monkeyfs")
if credential_file is None:
return False, "Service account credential file is not provided"
print("Mounting filesystem...")
cred_environment = aws_cred_file_environment(credential_file)
uuid = self.update_uuid()
setup_job_args = {
"aws_storage_name": aws_storage_name,
"monkeyfs_path": monkeyfs_path,
"access_key_id": cred_environment["AWS_ACCESS_KEY_ID"],
"access_key_secret": cred_environment["AWS_SECRET_ACCESS_KEY"]
}
runner = self.run_ansible_role(rolename="aws/mount_fs",
extravars=setup_job_args,
uuid=uuid)
if runner.status == "failed" or self.get_uuid() != uuid:
print("Failed to mount filesystem")
return False, "Failed to mount filesystem"
# TODO(alamp): Different home dirs if using non ubuntu distributions
home_dir_path = "/home/ubuntu"
for data_item in job.get("data", []):
print("Setting up data item", data_item)
success, msg = self.setup_data_item(data_item=data_item,
monkeyfs_path=monkeyfs_path,
home_dir_path=home_dir_path)
if not success:
return success, msg
success, msg = self.unpack_job_dir(job_uid=job_uid,
monkeyfs_path=monkeyfs_path,
home_dir_path=home_dir_path)
if not success:
return success, msg
for code_item in job.get("code", []):
success, msg = self.unpack_code_and_persist(
code_item=code_item,
monkeyfs_path=monkeyfs_path,
home_dir_path=home_dir_path)
if not success:
return success, msg
print("Success in unpacking all datasets")
print("Setting up logs folder")
success, msg = self.setup_logs_folder(job_uid=job_uid,
home_dir_path=home_dir_path)
if not success:
return success, msg
for persist_item in job.get("persist", []):
print("Setting up persist item", persist_item)
success, msg = self.setup_persist_folder(
job_uid=job_uid,
home_dir_path=home_dir_path,
persist=persist_item)
if not success:
return success, msg
print("Starting Persist")
success, msg = self.start_persist(job_uid=job_uid,
home_dir_path=home_dir_path,
persist=persist_item)
if not success:
return success, msg
print("Setting up dependency manager...")
success, msg = self.setup_dependency_manager(job["run"])
if not success:
return success, msg
return True, "Successfully setup the job"