def handler(event, context): ''' somewhere in the event data should be a jobid ''' # s3 bucket that stores the output bucket_name = event['config']['log_bucket'] s3 = utils.s3Utils(bucket_name, bucket_name, bucket_name) # info about the jobby job jobid = event['jobid'] job_started = "%s.job_started" % jobid job_success = "%s.success" % jobid job_error = "%s.error" % jobid job_log = "%s.log" % jobid job_log_location = "https://s3.amazonaws.com/%s/%s" % (bucket_name, job_log) # check to see ensure this job has started else fail if not s3.does_key_exist(job_started): raise EC2StartingException( "Failed to find jobid %s, ec2 is probably still booting" % jobid) # check to see if job has error, report if so if s3.does_key_exist(job_error): raise Exception("Job encountered an error check log at %s" % job_log_location) # check to see if job has completed if not throw retry error if s3.does_key_exist(job_success): print("completed successfully") return event else: raise StillRunningException("job %s still running" % jobid)
def calc_ebs_size(bucket, key): s3 = s3Utils(bucket, bucket, bucket) import pdb pdb.set_trace() size = s3.get_file_size(key, bucket, add_gb=3, size_in_gb=True) if size < 10: size = 10 return size
def mocked_s3(): ''' don't put this as a fixutre so it will be called within the s3 mock decorators on the associated tests ''' # We need to create the bucket since this is all in Moto's 'virtual' AWS account conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='dbmi-repo-registry') conn.create_bucket(Bucket='tc-systems') return s3Utils(outfile_bucket='tc-systems', sys_bucket='dbmi-repo-registry')
def s3(check_task_input): bucket_name = check_task_input['config']['log_bucket'] return utils.s3Utils(bucket_name, bucket_name, bucket_name)
def __init__(self, bucket, key, runner, accession=None): self.bucket = bucket self.key = key self.s3 = utils.s3Utils(self.bucket, self.bucket, self.bucket) self.runner = runner self.accession = accession
''' KISS scheduler, simply store list of registered urls on s3 file ''' import time from core.utils import s3Utils from botocore.exceptions import ClientError import logging from concurrent.futures import ThreadPoolExecutor log = logging.getLogger(__name__) s3 = s3Utils(outfile_bucket='tc-systems', sys_bucket='dbmi-repo-registry') def register(name, url, s3utils, update=False): ''' register just by writting a new file to s3 with key=name and data=url ''' try: exists = s3utils.read_s3(name) except ClientError as e: if e.response['Error']['Code'] == 'NoSuchKey': exists = False if exists and update is False: raise Exception("Already registered, use update option") return s3utils.s3_put(url, name)
def s3_utils(): return s3Utils(env=tibanna_env()['_tibanna']['env'])