def create(self):
# Begin running command to create the instance on Google Cloud
if not self.get_status() == Processor.OFF:
logging.error("(%s) Cannot create processor! One with that name already exits with current status: %s" % (
self.name, self.get_status()))
raise RuntimeError("Processor can only be created if it's 'OFF'!")
elif self.is_locked():
logging.error("(%s) Failed to create processor. Processor locked!" % self.name)
raise RuntimeError("Cannot create processor while locked!")
# Set status to indicate that commands can't be run on processor because it's busy
logging.info("(%s) Process 'create' started!" % self.name)
# Determine instance type and actual resource usage based on current Google prices in instance zone
self.nr_cpus, self.mem, self.instance_type = GoogleCloudHelper.get_optimal_instance_type(self.nr_cpus,
self.mem,
self.zone,
self.is_preemptible)
# Determine instance price at time of creation
self.price = GoogleCloudHelper.get_instance_price(self.nr_cpus,
self.mem,
self.disk_space,
self.instance_type,
self.zone,
self.is_preemptible,
self.is_boot_disk_ssd,
self.nr_local_ssd)
logging.debug("(%s) Instance type is %s. Price per hour: %s cents" % (self.name, self.instance_type, self.price))
# Generate gcloud create cmd
cmd = self.__get_gcloud_create_cmd()
# Try to create instance until either it's successful, we're out of retries, or the processor is locked
self.processes["create"] = Process(cmd,
cmd=cmd,
stdout=sp.PIPE,
stderr=sp.PIPE,
shell=True,
num_retries=self.default_num_cmd_retries)
self.wait_process("create")
# Wait for startup script to completely finish
logging.debug("(%s) Waiting for instance startup-script completion..." % self.name)
self.wait_until_ready()
logging.debug("(%s) Instance startup complete! %s Now live and ready to run commands!" % (self.name, self.name))
def create(self):
if self.is_locked():
logging.error(
"(%s) Failed to create processor. Processor locked!" %
self.name)
raise RuntimeError("Cannot create processor while locked!")
# Set status to indicate that commands can't be run on processor because it's busy
logging.info("(%s) Process 'create' started!" % self.name)
# Determine instance type and actual resource usage based on current Google prices in instance zone
self.nr_cpus, self.mem, self.instance_type = GoogleCloudHelper.get_optimal_instance_type(
self.nr_cpus, self.mem, self.zone, self.is_preemptible)
# Determine instance price at time of creation
self.price = GoogleCloudHelper.get_instance_price(
self.nr_cpus, self.mem, self.disk_space, self.instance_type,
self.zone, self.is_preemptible, self.is_boot_disk_ssd,
self.nr_local_ssd)
logging.debug("(%s) Instance type is %s. Price per hour: %s cents" %
(self.name, self.instance_type, self.price))
# Generate gcloud create cmd
cmd = self.__get_gcloud_create_cmd()
# Try to create instance until either it's successful, we're out of retries, or the processor is locked
self.processes["create"] = Process(
cmd,
cmd=cmd,
stdout=sp.PIPE,
stderr=sp.PIPE,
shell=True,
num_retries=self.default_num_cmd_retries)
self.wait_process("create")
# Wait for instance to be accessible through SSH
logging.debug("(%s) Waiting for instance to be accessible" % self.name)
self.wait_until_ready()
input_files = dstore.get_task_input_files("tool1_FASTQC")
print input_files
for input_file in input_files:
print "Before: %s" % input_file.get_path()
input_file.update_path(new_dir="/data/tool1_FASTQC/")
print "After: %s" % input_file.get_path()
print t1.get_argument("java")
######################### Test GoogleCloud helper
nr_cpus, mem, instance_type = GoogleCloudHelper.get_optimal_instance_type(
1, 1, "us-east1-b")
print nr_cpus, mem, instance_type
print GoogleCloudHelper.get_instance_price(nr_cpus, mem, 500, instance_type,
"us-east1-b")
print GoogleCloudHelper.get_instance_price(nr_cpus,
mem,
500,
instance_type,
"us-east1-b",
is_preemptible=True)
print GoogleCloudHelper.get_instance_price(nr_cpus,
mem,
500,
instance_type,
"us-east1-b",
is_preemptible=True,
nr_local_ssd=5)
print