def post_startup(self):
# Copy Google key to instance and authenticate
if self.google_json is not None:
# Transfer key to instance
cmd = f'scp -i {self.ssh_private_key} -o CheckHostIP=no -o StrictHostKeyChecking=no {self.google_json} ' \
f'{self.ssh_connection_user}@{self.external_IP}:GCP.json'
Process.run_local_cmd(cmd, err_msg="Could not authenticate Google SDK on instance!")
# Activate service account
cmd = f'gcloud auth activate-service-account --key-file /home/{self.ssh_connection_user}/GCP.json'
self.run("authenticate_google", cmd)
self.wait_process("authenticate_google")
else:
logging.warning("(%s) Google JSON key not provided! "
"Instance will not be able to access GCP buckets!" % self.name)
# Authenticate AWS CLI
cmd = f'aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID \
&& aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY \
&& aws configure set default.region {self.region} \
&& aws configure set default.output json'
self.run("aws_configure", cmd)
self.wait_process("aws_configure")
def publish_report(self, report_path):
# Generate destination file path
dest_path = os.path.join(self.final_output_dir,
os.path.basename(report_path))
# Authenticate for gsutil use
cmd = "gcloud auth activate-service-account --key-file %s" % self.identity
Process.run_local_cmd(cmd,
err_msg="Authentication to Google Cloud failed!")
# Transfer report file to bucket
options_fast = '-m -o "GSUtil:sliced_object_download_max_components=200"'
cmd = "gsutil %s cp -r '%s' '%s' 1>/dev/null 2>&1 " % (
options_fast, report_path, dest_path)
logging.debug(f"Publish report cmd: {cmd}")
Process.run_local_cmd(
cmd,
err_msg=
"Could not transfer final report to the final output directory!")
# Check if the user has provided a Pub/Sub report topic
pubsub_topic = self.extra.get("report_topic", None)
pubsub_project = self.extra.get("pubsub_project", None)
# Send report to the Pub/Sub report topic if it's known to exist
if pubsub_topic and pubsub_project:
GooglePlatform.__send_pubsub_message(pubsub_topic, pubsub_project,
dest_path)
def push_log(self, log_path):
# Generate destination file path
dest_path = os.path.join(self.final_output_dir, os.path.basename(log_path))
# Authenticate for gsutil use
cmd = "gcloud auth activate-service-account --key-file %s" % self.identity
Process.run_local_cmd(cmd, err_msg="Authentication to Google Cloud failed!")
# Transfer report file to bucket
options_fast = '-m -o "GSUtil:sliced_object_download_max_components=200"'
cmd = "gsutil %s cp -r '%s' '%s' 1>/dev/null 2>&1 " % (options_fast, log_path, dest_path)
Process.run_local_cmd(cmd, err_msg="Could not transfer final log to the final output directory!")
def push_log(self, log_path):
# Generate destination file path
dest_path = os.path.join(self.final_output_dir,
os.path.basename(log_path))
# Transfer report file to bucket
# cmd = "aws s3 cp $( [ -d %s ] && echo --recursive ) %s %s" % \
# (log_path, log_path, dest_path)
options_fast = '-m -o "GSUtil:sliced_object_download_max_components=200"'
cmd = "gsutil %s cp -r '%s' '%s'" % (options_fast, log_path, dest_path)
err_msg = "Could not transfer final log to the final output directory!"
Process.run_local_cmd(cmd, err_msg=err_msg)
def push_log(self, log_path):
# Generate destination file path
dest_path = os.path.join(self.final_output_dir,
os.path.basename(log_path))
# Authenticate for gsutil use
cmd = "gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS"
Process.run_local_cmd(cmd,
err_msg="Authentication to Google Cloud failed!")
# Transfer report file to bucket
options_fast = '-m -o "GSUtil:sliced_object_download_max_components=200"'
cmd = "gsutil %s cp -r '%s' '%s' 1>/dev/null 2>&1 " % (
options_fast, log_path, dest_path)
Process.run_local_cmd(
cmd,
err_msg=
"Could not transfer final log to the final output directory!")
# Transfer failed module log file to bucket
failed_module_log_path = log_path.replace("cc_log.txt",
"failed_module_log.txt")
failed_module_dest_path = dest_path.replace("cc_log.txt",
"failed_module_log.txt")
options_fast = '-m -o "GSUtil:sliced_object_download_max_components=200"'
cmd = "gsutil %s cp -r '%s' '%s' 1>/dev/null 2>&1 " % (
options_fast, failed_module_log_path, failed_module_dest_path)
Process.run_local_cmd(
cmd,
err_msg=
"Could not transfer failed module log to the final output directory!"
)