def __init__(self, task_id, processor, workspace, docker_image=None):
self.task_id = task_id
self.processor = processor
self.workspace = workspace
self.storage_helper = StorageHelper(self.processor)
self.docker_helper = DockerHelper(self.processor)
self.docker_image = docker_image
def __init__(self, task_id, processor, workspace, docker_image=None):
self.task_id = task_id
self.processor = processor
self.workspace = workspace
self.storage_helper = StorageHelper(self.processor)
self.docker_helper = DockerHelper(self.processor)
self.docker_image = docker_image
# Create workspace directory structure
self.__create_workspace()
def validate(self):
# Assume all validations are working
has_errors = False
# Validate the sample set
sample_validator = SampleValidator(self.sample_data)
has_errors = sample_validator.validate() or has_errors
if not has_errors:
logging.debug("Sample sheet validated!")
# Validate the graph
graph_validator = GraphValidator(self.graph, self.resource_kit, self.sample_data)
has_errors = graph_validator.validate() or has_errors
if not has_errors:
logging.debug("Graph validated!")
# Validate the platform
self.platform.validate()
# Stop the pipeline before launching if there are any errors
if has_errors:
raise SystemError("One or more errors have been encountered during validation. "
"See the above logs for more information")
# Create helper processor and storage/docker helpers for checking input files
self.helper_processor = self.platform.get_helper_processor()
self.helper_processor.create()
self.storage_helper = StorageHelper(self.helper_processor)
self.docker_helper = DockerHelper(self.helper_processor)
# Validate all pipeline inputs can be found on platform
input_validator = InputValidator(self.resource_kit, self.sample_data, self.storage_helper, self.docker_helper)
has_errors = input_validator.validate() or has_errors
# Stop the pipeline if there are any errors
if has_errors:
raise SystemError("One or more errors have been encountered during validation. "
"See the above logs for more information")
# Validate that pipeline workspace can be created
workspace = self.datastore.get_task_workspace()
for dir_type, dir_path in workspace.get_workspace().items():
self.storage_helper.mkdir(dir_path=str(dir_path), job_name="mkdir_%s" % dir_type, wait=True)
logging.info("CloudCounductor run validated! Beginning pipeline execution.")
def __init__(self,
task_id,
processor,
final_output_dir,
final_tmp_dir,
docker_image=None):
self.task_id = task_id
self.processor = processor
self.storage_helper = StorageHelper(self.processor)
self.docker_helper = DockerHelper(self.processor)
self.docker_image = docker_image
self.final_output_dir = final_output_dir
self.final_tmp_dir = final_tmp_dir
# Create workspace directory structure
self.__create_workspace()
def validate(self):
# Assume all validations are working
has_errors = False
# Validate the sample set
sample_validator = SampleValidator(self.sample_data)
has_errors = sample_validator.validate() or has_errors
if not has_errors:
logging.debug("Sample sheet validated!")
# Validate the graph
graph_validator = GraphValidator(self.graph, self.resource_kit,
self.sample_data)
has_errors = graph_validator.validate() or has_errors
if not has_errors:
logging.debug("Graph validated!")
# Validate the platform
self.platform.validate()
# Stop the pipeline before launching if there are any errors
if has_errors:
raise SystemError(
"One or more errors have been encountered during validation. "
"See the above logs for more information")
# Create storage/docker helpers for checking input files
self.storage_helper = StorageHelper(None)
self.docker_helper = DockerHelper(None)
# Validate all pipeline inputs can be found on platform
input_validator = InputValidator(self.resource_kit, self.sample_data,
self.storage_helper,
self.docker_helper)
has_errors = input_validator.validate() or has_errors
# Stop the pipeline if there are any errors
if has_errors:
raise SystemError(
"One or more errors have been encountered during validation. "
"See the above logs for more information")
logging.info(
"CloudCounductor run validated! Beginning pipeline execution.")
rk = ResourceKit(rk_good)
ss_file = "/home/alex/Desktop/projects/gap/test/ss.json"
ss = SampleSet(ss_file)
proc = GoogleStandardProcessor(
"test-proc-4",
4,
12,
75,
zone="us-east1-c",
service_acct="*****@*****.**",
boot_disk_size=75,
disk_image="davelab-image-docker")
sh = StorageHelper(proc)
dh = DockerHelper(proc)
iv = InputValidator(rk, ss, sh, dh)
try:
proc.create()
proc.__configure_SSH()
print "We validatin'"
print iv.validate()
print "We done validatin'"
finally:
proc.destroy(wait=False)
exit(0)