def validate(self):
# Check that final output dir begins with gs://
if not self.final_output_dir.startswith("gs://"):
logging.error(
"Invalid final output directory: %s. Google bucket paths must begin with 'gs://'"
% self.final_output_dir)
raise IOError("Invalid final output directory!")
# Make gs bucket if it doesn't exists already
gs_bucket = GoogleCloudHelper.get_bucket_from_path(
self.final_output_dir)
if not GoogleCloudHelper.bucket_exists(gs_bucket):
logging.info("Bucket {0} does not exists. Creating it now!".format(
gs_bucket))
region = GoogleCloudHelper.get_region(self.zone)
GoogleCloudHelper.mb(gs_bucket,
project=self.google_project,
region=region)
# Set the minimum disk size based on size of disk image
disk_image = self.config["task_processor"]["disk_image"]
disk_image_info = GoogleCloudHelper.get_disk_image_info(disk_image)
self.MIN_DISK_SPACE = int(disk_image_info["diskSizeGb"])
# Check to see if the reporting Pub/Sub topic exists
if not GoogleCloudHelper.pubsub_topic_exists(self.report_topic):
logging.error("Reporting topic '%s' was not found!" %
self.report_topic)
raise IOError("Reporting topic '%s' not found!" %
self.report_topic)
# Indicate that report topic exists and has been validated
self.report_topic_validated = True
def __init__(self, name, nr_cpus, mem, disk_space, **kwargs):
# Call super constructor
super(Instance, self).__init__(name, nr_cpus, mem, disk_space, **kwargs)
# Get required arguments
self.zone = kwargs.pop("zone")
self.service_acct = kwargs.pop("service_acct")
self.disk_image = kwargs.pop("disk_image")
# Get optional arguments
self.is_boot_disk_ssd = kwargs.pop("is_boot_disk_ssd", False)
self.nr_local_ssd = kwargs.pop("nr_local_ssd", 0)
# Initialize the region of the instance
self.region = GoogleCloudHelper.get_region(self.zone)
# Initialize instance random id
self.rand_instance_id = self.name.rsplit("-",1)[-1]
# Indicates that instance is not resettable
self.is_preemptible = False
# Google instance type. Will be set at creation time based on google price scheme
self.instance_type = None
# Initialize the price of the run and the total cost of the run
self.price = 0
self.cost = 0
# Flag for whether startup script has completed running
self.__startup_script_complete = False
def __get_instance_config(self):
# Returns complete config for a task processor
params = {}
inst_params = self.config["task_processor"]
for param, value in inst_params.items():
params[param] = value
# Add platform-specific options
params["zone"] = self.zone
params["service_acct"] = self.service_acct
# Randomize the zone within the region if specified
if self.randomize_zone:
region = GoogleCloudHelper.get_region(self.zone)
params["zone"] = GoogleCloudHelper.select_random_zone(region)
# Get instance type
return params
def __init__(self, name, nr_cpus, mem, disk_space, **kwargs):
# Call super constructor
super(Instance, self).__init__(name, nr_cpus, mem, disk_space,
**kwargs)
# Get required arguments
self.zone = kwargs.pop("zone")
self.service_acct = kwargs.pop("service_acct")
self.disk_image = kwargs.pop("disk_image")
# Get optional arguments
self.is_boot_disk_ssd = kwargs.pop("is_boot_disk_ssd", False)
self.nr_local_ssd = kwargs.pop("nr_local_ssd", 0)
# Initialize the region of the instance
self.region = GoogleCloudHelper.get_region(self.zone)
# Initialize instance random id
self.rand_instance_id = self.name.rsplit("-", 1)[-1]
# Indicates that instance is not resettable
self.is_preemptible = False
# Google instance type. Will be set at creation time based on google price scheme
self.instance_type = None
# Initialize the price of the run and the total cost of the run
self.price = 0
self.cost = 0
# Initialize the SSH status
self.ssh_connections_increased = False
self.ssh_ready = False
# Number of times creation has been reset
self.creation_resets = 0
# API Rate limit errors count
self.api_rate_limit_retries = 0
# Initialize extenal IP
self.external_IP = None