Exemplo n.º 1
0
Arquivo: job.py Projeto: brianhou/GPIS
    def _setup(self):
        # initialize helper classes for interacting with GCE, GCS
        self.auth_http = instance.oauth_authorization(self.config)
        self.gce_helper = gce.Gce(self.auth_http, self.config, project_id=self.config['compute']['project'])
        self.gcs_helper = gcs.Gcs(self.auth_http, self.config, project_id=self.config['compute']['project'])
        self.instance_manager = instance.VMInstanceManager()

        self.instances = []
        self.launched_instances = {}
        self.user_terminated = False

        # get job id
        self.id = gen_job_id()
        self.job_name_root = 'job-%s' %(self.id)
        if self.config['update']:
            self.job_name_root = 'job-updater-%s' %(self.id)            

        # setup instance completion api call
        service_not_ready = True
        while service_not_ready:
            try:
                service = discovery.build('storage', self.config['compute']['api_version'], http=self.gce_helper.auth_http)
                self.bucket_req = service.objects().list(bucket=self.config['compute']['bucket'], prefix=self.job_name_root)
                service_not_ready = False
            except (ValueError, Exception) as e:
                logging.info('Connection failed. Retrying...')
Exemplo n.º 2
0
    def stage(self):
        """ Stages changes by creating update disks, if they don't already exist. """
        # setup vars
        compute_config = self.config_['compute']
        created_snapshots = False
        if not self.update_data_disks_:
            self.compute_update_data_disks()

        # authorize access to GCE api
        auth_http = instance.oauth_authorization(self.config_)
        gce_helper = gce.Gce(auth_http, self.config_, project_id=compute_config['project'])

        # for all zones, create a disk snapshot if they don't already exist
        for zone, disk, update_disk_name in zip(compute_config['zones'], compute_config['data_disks'], self.update_data_disks_):
            # check for existence of the update disk (taken as a flag for the existence of an update node)
            disk_valid = gce_helper.get_disk(update_disk_name, zone)
            if not disk_valid:
                # create a snapshot of the current disk
                logging.info('Snapshotting disk %s' %(disk))                    
                snapshot_response = gce_helper.snapshot_disk(disk, compute_config['project'], zone)

                # create a disk from the snapshot
                logging.info('Creating update disk %s from snapshot %s' %(update_disk_name, snapshot_response['snapshot_name']))
                gce_helper.create_disk(update_disk_name, zone=zone, size_gb=compute_config['disk_size_gb'],
                                       source_snapshot=snapshot_response['snapshot_name'])

                # delete the snapshot
                ss_del_response = gce_helper.delete_snapshot(snapshot_name=snapshot_response['snapshot_name'], project=compute_config['project'])
                created_snapshots = True
        return created_snapshots
Exemplo n.º 3
0
    def _setup(self):
        # initialize helper classes for interacting with GCE, GCS
        self.auth_http = instance.oauth_authorization(self.config)
        self.gce_helper = gce.Gce(self.auth_http,
                                  self.config,
                                  project_id=self.config['compute']['project'])
        self.gcs_helper = gcs.Gcs(self.auth_http,
                                  self.config,
                                  project_id=self.config['compute']['project'])
        self.instance_manager = instance.VMInstanceManager()

        self.instances = []
        self.launched_instances = {}
        self.user_terminated = False

        # get job id
        self.id = gen_job_id()
        self.job_name_root = 'job-%s' % (self.id)
        if self.config['update']:
            self.job_name_root = 'job-updater-%s' % (self.id)

        # setup instance completion api call
        service_not_ready = True
        while service_not_ready:
            try:
                service = discovery.build(
                    'storage',
                    self.config['compute']['api_version'],
                    http=self.gce_helper.auth_http)
                self.bucket_req = service.objects().list(
                    bucket=self.config['compute']['bucket'],
                    prefix=self.job_name_root)
                service_not_ready = False
            except (ValueError, Exception) as e:
                logging.info('Connection failed. Retrying...')
Exemplo n.º 4
0
    def push(self):
        """ Pushed changes by replacing original disks with update disks. Super critical section. """
        # setup vars
        compute_config = self.config_['compute']
        dt_now = dt.datetime.now()
        if not self.update_data_disks_:
            self.compute_update_data_disks()

        # authorize access to GCE api
        auth_http = instance.oauth_authorization(self.config_)
        gce_helper = gce.Gce(auth_http, self.config_, project_id=compute_config['project'])

        for zone, disk, update_disk in zip(compute_config['zones'], compute_config['data_disks'], self.update_data_disks_):
            # check for update disk existence
            disk_response = gce_helper.get_disk(update_disk, zone)
            if not disk_response:
                logging.error('Update disk %s does not exist' %(update_disk))
                continue

            # generate backup disk filename
            backup_disk = '%s-backup-%s-%s-%s-%sh-%sm-%ss' %(disk, dt_now.month, dt_now.day, dt_now.year, dt_now.hour, dt_now.minute, dt_now.second) 

            # snapshot the updated data disks
            snapshot_response = gce_helper.snapshot_disk(update_disk, compute_config['project'], zone)

            # delete previous disk and replace, if not in use
            disk_response = gce_helper.get_disk(disk, zone)
            if disk_response:
                if USERS_KEY not in disk_response.keys() or (USERS_KEY in disk_response.keys() and len(disk_response[USERS_KEY]) == 0):
                    # create new disk from snapshot
                    gce_helper.delete_disk(disk)
                    gce_helper.create_disk(disk, zone=zone, size_gb=compute_config['disk_size_gb'],
                                           source_snapshot=snapshot_response['snapshot_name'])

                    # delete update disk (don't delete if push can't be done now, otherwise changes won't be overwritten)
                    gce_helper.delete_disk(update_disk)

                elif USERS_KEY in disk_response.keys() and len(disk_response[USERS_KEY]) > 0:
                    # stage the push for a future time
                    logging.info('Master disk in use. Staging backup disk for a future push')
                    push_queue_filename = os.path.join(self.cache_dir_, PUSH_QUEUE_FILE)
                    f = open(push_queue_filename, 'a')
                    f.write(backup_disk + '\n')
            else:
                logging.warning('Master disk was not found') 

            # create backup disk from snapshot
            gce_helper.create_disk(backup_disk, zone=zone, size_gb=compute_config['disk_size_gb'],
                                   source_snapshot=snapshot_response['snapshot_name'])

            # delete the snapshot
            ss_del_response = gce_helper.delete_snapshot(snapshot_name=snapshot_response['snapshot_name'], project=compute_config['project'])
        return True
Exemplo n.º 5
0
    def stage(self):
        """ Stages changes by creating update disks, if they don't already exist. """
        # setup vars
        compute_config = self.config_['compute']
        created_snapshots = False
        if not self.update_data_disks_:
            self.compute_update_data_disks()

        # authorize access to GCE api
        auth_http = instance.oauth_authorization(self.config_)
        gce_helper = gce.Gce(auth_http,
                             self.config_,
                             project_id=compute_config['project'])

        # for all zones, create a disk snapshot if they don't already exist
        for zone, disk, update_disk_name in zip(compute_config['zones'],
                                                compute_config['data_disks'],
                                                self.update_data_disks_):
            # check for existence of the update disk (taken as a flag for the existence of an update node)
            disk_valid = gce_helper.get_disk(update_disk_name, zone)
            if not disk_valid:
                # create a snapshot of the current disk
                logging.info('Snapshotting disk %s' % (disk))
                snapshot_response = gce_helper.snapshot_disk(
                    disk, compute_config['project'], zone)

                # create a disk from the snapshot
                logging.info(
                    'Creating update disk %s from snapshot %s' %
                    (update_disk_name, snapshot_response['snapshot_name']))
                gce_helper.create_disk(
                    update_disk_name,
                    zone=zone,
                    size_gb=compute_config['disk_size_gb'],
                    source_snapshot=snapshot_response['snapshot_name'])

                # delete the snapshot
                ss_del_response = gce_helper.delete_snapshot(
                    snapshot_name=snapshot_response['snapshot_name'],
                    project=compute_config['project'])
                created_snapshots = True
        return created_snapshots
Exemplo n.º 6
0
    def push(self):
        """ Pushed changes by replacing original disks with update disks. Super critical section. """
        # setup vars
        compute_config = self.config_['compute']
        dt_now = dt.datetime.now()
        if not self.update_data_disks_:
            self.compute_update_data_disks()

        # authorize access to GCE api
        auth_http = instance.oauth_authorization(self.config_)
        gce_helper = gce.Gce(auth_http,
                             self.config_,
                             project_id=compute_config['project'])

        for zone, disk, update_disk in zip(compute_config['zones'],
                                           compute_config['data_disks'],
                                           self.update_data_disks_):
            # check for update disk existence
            disk_response = gce_helper.get_disk(update_disk, zone)
            if not disk_response:
                logging.error('Update disk %s does not exist' % (update_disk))
                continue

            # generate backup disk filename
            backup_disk = '%s-backup-%s-%s-%s-%sh-%sm-%ss' % (
                disk, dt_now.month, dt_now.day, dt_now.year, dt_now.hour,
                dt_now.minute, dt_now.second)

            # snapshot the updated data disks
            snapshot_response = gce_helper.snapshot_disk(
                update_disk, compute_config['project'], zone)

            # delete previous disk and replace, if not in use
            disk_response = gce_helper.get_disk(disk, zone)
            if disk_response:
                if USERS_KEY not in disk_response.keys() or (
                        USERS_KEY in disk_response.keys()
                        and len(disk_response[USERS_KEY]) == 0):
                    # create new disk from snapshot
                    gce_helper.delete_disk(disk)
                    gce_helper.create_disk(
                        disk,
                        zone=zone,
                        size_gb=compute_config['disk_size_gb'],
                        source_snapshot=snapshot_response['snapshot_name'])

                    # delete update disk (don't delete if push can't be done now, otherwise changes won't be overwritten)
                    gce_helper.delete_disk(update_disk)

                elif USERS_KEY in disk_response.keys() and len(
                        disk_response[USERS_KEY]) > 0:
                    # stage the push for a future time
                    logging.info(
                        'Master disk in use. Staging backup disk for a future push'
                    )
                    push_queue_filename = os.path.join(self.cache_dir_,
                                                       PUSH_QUEUE_FILE)
                    f = open(push_queue_filename, 'a')
                    f.write(backup_disk + '\n')
            else:
                logging.warning('Master disk was not found')

            # create backup disk from snapshot
            gce_helper.create_disk(
                backup_disk,
                zone=zone,
                size_gb=compute_config['disk_size_gb'],
                source_snapshot=snapshot_response['snapshot_name'])

            # delete the snapshot
            ss_del_response = gce_helper.delete_snapshot(
                snapshot_name=snapshot_response['snapshot_name'],
                project=compute_config['project'])
        return True