def execute(self):
aws_creds_profile = self.runner_cfg.data['package_depot_aws_profile']
aws_creds_file = os.path.join(self.runner_cfg.data['runner_credentials_dir'], 'aws', 'config')
if os.path.exists(aws_creds_file):
config = ConfigParser.ConfigParser()
config.read([str(aws_creds_file)])
self.aws_access_key_id = config.get('profile ' + aws_creds_profile, 'aws_access_key_id')
self.aws_secret_access_key = config.get('profile ' + aws_creds_profile, 'aws_secret_access_key')
if self.args['artiball'].endswith('.tar.gz'):
artiball = self.args['artiball']
else:
artiball = self.args['artiball'] + '.tar.gz'
pack.set_incoming_s3_bucket()
self.logger.write('Downloading package from S3 bucket ' + pack.PACKAGE_INCOMING_S3_BUCKET, multi_line=False)
download_result = sky_boto_actions.download_from_s3(pack.PACKAGE_INCOMING_S3_BUCKET, artiball,
self.tmp_dir, self.logger,
access_key=self.aws_access_key_id,
secret_key=self.aws_secret_access_key,
dry_run=False)
self.result.output += download_result["result_string"]
if not download_result["valid"]:
self.result.status = sky_cfg.API_STATUS_FAIL
return self.result
artiball_file = tarfile.open(os.path.join(self.tmp_dir, artiball), 'r:gz')
artiball_dir = os.path.join(self.tmp_dir, artiball.split('.tar.gz')[0])
artiball_file.extractall(os.path.join(artiball_dir, 'skybase'))
self.logger.write('Validating package in ' + artiball_dir, multi_line=False)
validate_result = pack_actions.validate_with_schema(artiball_dir, 'artiball',
update_content_from_config=False)
if validate_result["valid"]:
self.result.output += "All content validated, ready for pack submit.\n"
else:
self.result.output += "Invalid content for submission, please verify artiball is valid.\n"
self.result.status = sky_cfg.API_STATUS_FAIL
return self.result
app_dir = os.path.join(artiball_dir, 'skybase', 'app')
yum_aws_creds_file = os.path.join(self.runner_cfg.data['runner_credentials_dir'], 'aws', 'config')
if os.path.exists(yum_aws_creds_file):
config = ConfigParser.ConfigParser()
config.read([str(yum_aws_creds_file)])
yum_replications = self.runner_cfg.data['yum_replication']
for yum_replication in yum_replications:
yum_aws_creds_profile = yum_replication['profile']
yum_aws_access_key_id = config.get('profile ' + yum_aws_creds_profile, 'aws_access_key_id')
yum_aws_secret_access_key = config.get('profile ' + yum_aws_creds_profile, 'aws_secret_access_key')
for f in os.listdir(app_dir):
if os.path.splitext(f)[1] == '.rpm':
# Hardcoding default to True for pack group command, revisit later
self.args['apply'] = True
if self.args['apply']:
upload_result = sky_boto_actions.upload_to_s3(yum_replication['name'],
os.path.join(app_dir, f),
self.logger, prefix='inbox/skybase',
access_key=yum_aws_access_key_id,
secret_key=yum_aws_secret_access_key,
dry_run=False)
else:
upload_result = sky_boto_actions.upload_to_s3(yum_replication['name'],
os.path.join(app_dir, f),
self.logger, prefix='inbox/skybase',
access_key=yum_aws_access_key_id,
secret_key=yum_aws_secret_access_key,
dry_run=True)
self.result.output += upload_result["result_string"]
if not upload_result["valid"]:
self.result.status = sky_cfg.API_STATUS_FAIL
return self.result
else:
self.result.output += "Cannot locate aws credentials, please confirm they are set in " \
+ yum_aws_creds_file + "\n"
self.result.status = sky_cfg.API_STATUS_FAIL
return self.result
else:
self.result.output += "Cannot locate aws credentials, please confirm they are set in " + aws_creds_file \
+ "\n"
self.result.status = sky_cfg.API_STATUS_FAIL
return self.result
file_path = os.path.join(self.tmp_dir, artiball)
depot_bucket_name = os.path.expanduser(self.runner_cfg.data['package_depot_S3_bucket'])
# Hardcoding default to True for pack group command, revisit later
self.args['apply'] = True
if self.args['apply']:
self.logger.write('Uploading package to S3 bucket ' + depot_bucket_name, multi_line=False)
upload_result = sky_boto_actions.upload_to_s3(depot_bucket_name, file_path, self.logger,
access_key=self.aws_access_key_id,
secret_key=self.aws_secret_access_key,
dry_run=False)
self.result.output += upload_result["result_string"]
if not upload_result["valid"]:
self.result.status = sky_cfg.API_STATUS_FAIL
return self.result
self.logger.write('Archiving package in S3 bucket ' + pack.PACKAGE_INCOMING_S3_BUCKET, multi_line=False)
move_result = sky_boto_actions.move_object_s3(pack.PACKAGE_INCOMING_S3_BUCKET, pack.PACKAGE_INCOMING_S3_BUCKET,
os.path.basename(file_path),
src_prefix=None, dst_prefix='archive',
access_key=self.aws_access_key_id,
secret_key=self.aws_secret_access_key,
dry_run=False)
else:
self.logger.write('DRY RUN :::: Dry running steps for package upload to S3 bucket ' + depot_bucket_name,
multi_line=False)
upload_result = sky_boto_actions.upload_to_s3(depot_bucket_name, file_path, self.logger,
access_key=self.aws_access_key_id,
secret_key=self.aws_secret_access_key,
dry_run=True)
self.result.output += upload_result["result_string"]
if not upload_result["valid"]:
self.result.status = sky_cfg.API_STATUS_FAIL
return self.result
self.logger.write('DRY RUN :::: Dry running steps for archiving package in S3 bucket '
+ pack.PACKAGE_INCOMING_S3_BUCKET, multi_line=False)
move_result = sky_boto_actions.move_object_s3(pack.PACKAGE_INCOMING_S3_BUCKET, pack.PACKAGE_INCOMING_S3_BUCKET,
os.path.basename(file_path),
src_prefix=None, dst_prefix='archive',
access_key=self.aws_access_key_id,
secret_key=self.aws_secret_access_key,
dry_run=True)
self.result.output += move_result["result_string"]
if not move_result["valid"]:
self.result.status = sky_cfg.API_STATUS_FAIL
return self.result
if hasattr(self,'tdir') and os.path.exists(self.tdir):
shutil.rmtree(self.tdir)
else:
shutil.rmtree(self.tmp_dir)
self.result.status = sky_cfg.API_STATUS_SUCCESS
return self.result
def execute(self):
if self.args['base_dir']:
base_dir = self.args['base_dir']
if base_dir.split('/')[-1] is not 'skybase':
base_dir = os.path.join(base_dir, 'skybase')
else:
base_dir = os.path.join(os.getcwd(), 'skybase')
pack.set_incoming_s3_bucket()
if self.args['artiball']:
if self.args['artiball'].endswith('.tar.gz'):
file_path = os.path.join(base_dir, 'package',
self.args['artiball'])
else:
file_path = os.path.join(base_dir, 'package',
self.args['artiball'] + '.tar.gz')
else:
packages = os.listdir(os.path.join(base_dir, 'package'))
for package in packages:
file_path = os.path.join(base_dir, 'package', package)
aws_creds_file = os.path.expanduser(os.path.join(
'~', '.aws', 'config'))
if os.path.exists(aws_creds_file):
config = ConfigParser.ConfigParser()
config.read([str(aws_creds_file)])
aws_access_key_id = config.get('default', 'aws_access_key_id')
aws_secret_access_key = config.get('default',
'aws_secret_access_key')
# Hardcoding default to True for pack group command, revisit later
self.args['apply'] = True
if self.args['apply']:
self.logger.write('Uploading package to S3 bucket ' +
pack.PACKAGE_INCOMING_S3_BUCKET,
multi_line=False)
upload_result = sky_boto_actions.upload_to_s3(
pack.PACKAGE_INCOMING_S3_BUCKET,
file_path,
self.logger,
access_key=aws_access_key_id,
secret_key=aws_secret_access_key,
dry_run=False)
else:
self.logger.write(
'DRY RUN :::: Dry running steps for package upload to S3 bucket '
+ pack.PACKAGE_INCOMING_S3_BUCKET,
multi_line=False)
upload_result = sky_boto_actions.upload_to_s3(
pack.PACKAGE_INCOMING_S3_BUCKET,
file_path,
self.logger,
access_key=aws_access_key_id,
secret_key=aws_secret_access_key,
dry_run=True)
if upload_result["valid"]:
self.result.status = sky_cfg.API_STATUS_SUCCESS
else:
self.result.status = sky_cfg.API_STATUS_FAIL
self.result.output += upload_result["result_string"]
else:
self.result.output += "Cannot locate aws credentials, please confirm they are set in " + aws_creds_file \
+ "\n"
self.result.status = sky_cfg.API_STATUS_FAIL
return self.result
return self.result
def execute(self):
if self.args["base_dir"]:
base_dir = self.args["base_dir"]
if base_dir.split("/")[-1] is not "skybase":
base_dir = os.path.join(base_dir, "skybase")
else:
base_dir = os.path.join(os.getcwd(), "skybase")
pack.set_incoming_s3_bucket()
if self.args["artiball"]:
if self.args["artiball"].endswith(".tar.gz"):
file_path = os.path.join(base_dir, "package", self.args["artiball"])
else:
file_path = os.path.join(base_dir, "package", self.args["artiball"] + ".tar.gz")
else:
packages = os.listdir(os.path.join(base_dir, "package"))
for package in packages:
file_path = os.path.join(base_dir, "package", package)
aws_creds_file = os.path.expanduser(os.path.join("~", ".aws", "config"))
if os.path.exists(aws_creds_file):
config = ConfigParser.ConfigParser()
config.read([str(aws_creds_file)])
aws_access_key_id = config.get("default", "aws_access_key_id")
aws_secret_access_key = config.get("default", "aws_secret_access_key")
# Hardcoding default to True for pack group command, revisit later
self.args["apply"] = True
if self.args["apply"]:
self.logger.write("Uploading package to S3 bucket " + pack.PACKAGE_INCOMING_S3_BUCKET, multi_line=False)
upload_result = sky_boto_actions.upload_to_s3(
pack.PACKAGE_INCOMING_S3_BUCKET,
file_path,
self.logger,
access_key=aws_access_key_id,
secret_key=aws_secret_access_key,
dry_run=False,
)
else:
self.logger.write(
"DRY RUN :::: Dry running steps for package upload to S3 bucket " + pack.PACKAGE_INCOMING_S3_BUCKET,
multi_line=False,
)
upload_result = sky_boto_actions.upload_to_s3(
pack.PACKAGE_INCOMING_S3_BUCKET,
file_path,
self.logger,
access_key=aws_access_key_id,
secret_key=aws_secret_access_key,
dry_run=True,
)
if upload_result["valid"]:
self.result.status = sky_cfg.API_STATUS_SUCCESS
else:
self.result.status = sky_cfg.API_STATUS_FAIL
self.result.output += upload_result["result_string"]
else:
self.result.output += (
"Cannot locate aws credentials, please confirm they are set in " + aws_creds_file + "\n"
)
self.result.status = sky_cfg.API_STATUS_FAIL
return self.result
return self.result