Ejemplo n.º 1
0
    def transform(self, Xb, yb):
        shared_array_name = str(uuid4())
        fnames, labels = Xb, yb
        args = []
        da_args = self.da_args()
        for i, fname in enumerate(fnames):
            args.append((i, shared_array_name, fname, da_args))

        if self.num_image_channels is None:
            test_img = data.load_augment(fnames[0], **da_args)
            self.num_image_channels = test_img.shape[-1]

        try:
            shared_array = SharedArray.create(
                shared_array_name,
                [len(Xb), self.w, self.h, self.num_image_channels],
                dtype=np.float32)

            self.pool.map(load_shared, args)
            Xb = np.array(shared_array, dtype=np.float32)

        finally:
            SharedArray.delete(shared_array_name)

        # if labels is not None:
        #     labels = labels[:, np.newaxis]

        return Xb, labels
    def transform(self, Xb, yb):

        shared_array_name = str(uuid4())
        try:
            shared_array = SharedArray.create(
                shared_array_name, [len(Xb), 3, self.config.get('w'), 
                                    self.config.get('h')], dtype=np.float32)
                                        
            fnames, labels = super(SharedIterator, self).transform(Xb, yb)
            args = []

            for i, fname in enumerate(fnames):
                kwargs = {k: self.config.get(k) for k in ['w', 'h']}
                if not self.deterministic:
                    kwargs.update({k: self.config.get(k) 
                                   for k in ['aug_params', 'sigma']})
                kwargs['transform'] = getattr(self, 'tf', None)
                kwargs['color_vec'] = getattr(self, 'color_vec', None)
                args.append((i, shared_array_name, fname, kwargs))

            self.pool.map(load_shared, args)
            Xb = np.array(shared_array, dtype=np.float32)

        finally:
            SharedArray.delete(shared_array_name)

        if labels is not None:
            labels = labels[:, np.newaxis]

        return Xb, labels
    def __init__(self, file_path, file_name):
        print('class', DBSCAN.eps, DBSCAN.minpts)
        self.core_points = []
        self.core_point_labels = []
        self.core_points_index = []
        self.border_points_index = []
        self.border_points = []
        self.border_point_labels = []
        self.noise_points = []
        # self.nearest_neighbours = {}      # use for small values, space complexity is O(n^2)
        self.n_threads = cpu_count()
        self.features = []
        self.labels = []
        self.features, self.labels = process_dataset(
            file_path, file_name)  # limit the size of the dataset
        size = 10000
        self.features, self.labels = self.features[:size, :], self.labels[:
                                                                          size]
        print('features: \n', self.features.shape)
        try:
            sa.delete("shm://features")
        except Exception as e:
            print('file does not exist')
        self.shared_memory = sa.create("shm://features", self.features.shape)

        # copy the array into the shared memory
        for row_index in range(self.features.shape[0]):
            for point_index in range(self.features.shape[1]):
                self.shared_memory[row_index,
                                   point_index] = self.features[row_index,
                                                                point_index]
        self.clusters = []
    def transform(self, Xb, yb):

        shared_array_name = str(uuid4())
        try:
            shared_array = SharedArray.create(
                shared_array_name,
                [len(Xb), 3,
                 self.config.get('w'),
                 self.config.get('h')],
                dtype=np.float32)

            fnames, labels = super(SharedIterator, self).transform(Xb, yb)
            args = []

            for i, fname in enumerate(fnames):
                kwargs = {k: self.config.get(k) for k in ['w', 'h']}
                if not self.deterministic:
                    kwargs.update({
                        k: self.config.get(k)
                        for k in ['aug_params', 'sigma']
                    })
                kwargs['transform'] = getattr(self, 'tf', None)
                kwargs['color_vec'] = getattr(self, 'color_vec', None)
                args.append((i, shared_array_name, fname, kwargs))

            self.pool.map(load_shared, args)
            Xb = np.array(shared_array, dtype=np.float32)

        finally:
            SharedArray.delete(shared_array_name)

        if labels is not None:
            labels = labels[:, np.newaxis]

        return Xb, labels
Ejemplo n.º 5
0
    def transform(self, fundus, grade):
        shared_array_fundus_rescale_name = str(uuid4())
        shared_array_fundus_rescale_mean_subtract_name = str(uuid4())

        try:
            shared_array_fundus_mean_subt = SharedArray.create(
                shared_array_fundus_rescale_name,
                [len(fundus), img_h, img_w, 3],
                dtype=np.float32)
            shared_array_fundus_z = SharedArray.create(
                shared_array_fundus_rescale_mean_subtract_name,
                [len(fundus), img_h, img_w, 3],
                dtype=np.float32)

            args = []
            for i, _ in enumerate(fundus):
                args.append((i, shared_array_fundus_rescale_name,
                             shared_array_fundus_rescale_mean_subtract_name,
                             fundus[i], self.is_train))

            self.pool.map(load_shared, args)
            fundus_rescale = np.array(shared_array_fundus_mean_subt,
                                      dtype=np.float32)
            fundus_rescale_mean_subtract = np.array(shared_array_fundus_z,
                                                    dtype=np.float32)
        finally:
            SharedArray.delete(shared_array_fundus_rescale_name)
            SharedArray.delete(shared_array_fundus_rescale_mean_subtract_name)

        return fundus, fundus_rescale, fundus_rescale_mean_subtract, grade
Ejemplo n.º 6
0
    def run(self):
        """
        # TODO: write description
        """
        try:
            self.t0 = time.time()
            self.t1 = self.t0
            q = self.channel.queue_declare(queue='detector')
            self.channel.queue_declare(queue='time_logs')
            if q.method.message_count >= 59:
                time.sleep(1)

            frame_num, timestamp, images_list = self.batch_generator.__next__()
            self.log_time("Took next batch:")

            sh_mem_adress = f"shm://{self.module_name}_{frame_num}"
            try:
                shared_mem = sa.create(sh_mem_adress, np.shape(images_list))
            except:
                sa.delete(sh_mem_adress)
                shared_mem = sa.create(sh_mem_adress, np.shape(images_list))
            self.log_time('Created shared memory')
            shared_mem[:] = np.array(images_list)
            self.log_time('Copied to shared memory:')
            self.channel.basic_publish(exchange='',
                                       routing_key='detector',
                                       body=sh_mem_adress)

            self.log_time('Published message:')
            ########################################################################
            del frame_num, timestamp, images_list

            self.log_time('Full time:', from_start=True)
        except StopIteration:  # no more frames left in videos_provider
            print('stop iter')
Ejemplo n.º 7
0
    def shard_array_to_s3_mp(self, array, indices, s3_bucket, s3_keys):
        """Shard array to S3 in parallel.

        :param ndarray array: array to be put into S3
        :param list indices: indices corrsponding to the s3 keys
        :param str s3_bucket: S3 bucket to use
        :param list s3_keys: List of S3 keys corresponding to the indices.
        """
        def work_shard_array_to_s3(s3_key, index, array_name, s3_bucket):
            array = sa.attach(array_name)
            if sys.version_info >= (3, 5):
                data = bytes(array[index].data)
            else:
                data = bytes(np.ascontiguousarray(array[index]).data)

            if self.enable_compression:
                cctx = zstd.ZstdCompressor(level=9, write_content_size=True)
                data = cctx.compress(data)

            self.s3aio.s3io.put_bytes(s3_bucket, s3_key, data)

        array_name = '_'.join(['SA3IO', str(uuid.uuid4()), str(os.getpid())])
        sa.create(array_name, shape=array.shape, dtype=array.dtype)
        shared_array = sa.attach(array_name)
        shared_array[:] = array
        results = self.pool.map(work_shard_array_to_s3, s3_keys, indices,
                                repeat(array_name), repeat(s3_bucket))

        sa.delete(array_name)
Ejemplo n.º 8
0
    def delete_created_arrays(self):
        """Delete all created shared memory arrays.

          Arrays are prefixed by 'S3' or 'DCCORE'.
        """
        for a in self.list_created_arrays():
            sa.delete(a)
Ejemplo n.º 9
0
    def __exit__(self, *args):

        for array in self._shared:
            try:
                sa.delete(array)
            except FileNotFoundError:
                pass
Ejemplo n.º 10
0
def main():
    """Main function"""
    filepath, name, prefix, dtype = parse_arguments()

    if name is None:
        name = os.path.splitext(os.path.basename(filepath))[0]
        if prefix is not None:
            name = prefix + '_' + name

    print("Loading data from '{}'.".format(filepath))
    if filepath.endswith('.npy'):
        data = np.load(filepath)
        data = data.astype(dtype)
        print("Saving data to shared memory.")
        sa.delete(name)
        sa_array = sa.create(name, data.shape, data.dtype)
        np.copyto(sa_array, data)
    else:
        with np.load(filepath) as loaded:
            print("Saving data to shared memory.")
            sa_array = sa.create(name, loaded['shape'], dtype)
            sa_array[[x for x in loaded['nonzero']]] = True

    print("Successfully saved: (name='{}', shape={}, dtype={})".format(
        name, sa_array.shape, sa_array.dtype))
Ejemplo n.º 11
0
def to_shared_memory(object, name):
    logging.info("Writing to shared memory %s" % name)
    meta_information = {}
    for property_name in object.properties:
        data = object.__getattribute__(property_name)

        if data is None:
            data = np.zeros(0)

        # Wrap single ints in arrays
        if data.shape == ():
            data = np.array([data], dtype=data.dtype)

        data_type = data.dtype
        data_shape = data.shape
        meta_information[property_name] = (data_type, data_shape)

        # Make shared memory and copy data to buffer
        #logging.info("Field %s has shape %s and type %s" % (property_name, data_shape, data_type))
        try:
            sa.delete(name + "_" + property_name)
            logging.info("Deleted already shared memory")
        except FileNotFoundError:
            logging.info("No existing shared memory, can create new one")

        shared_array = sa.create(name + "_" + property_name, data_shape, data_type)
        shared_array[:] = data

    f = open(name + "_meta.shm", "wb")
    pickle.dump(meta_information, f)
    logging.info("Done writing to shared memory")
Ejemplo n.º 12
0
def create_new_sa_array(name, shape, dtype):
    try:
        sa.delete(name)
    except FileNotFoundError:
        pass
    finally:
        sa_array = sa.create(name, shape, dtype=dtype)
    return sa_array
Ejemplo n.º 13
0
def create_new_sa_array(name, shape, dtype):
    try:
        sa.delete(name)
    except FileNotFoundError:
        pass
    finally:
        sa_array = sa.create(name, shape, dtype=dtype)
    return sa_array
Ejemplo n.º 14
0
def get_publisher(channel: str, shape: tuple, dtype) -> np.ndarray:
    # Create an array in shared memory.
    short_name = channel.split("://")[-1]
    mapping = {e.name.decode(): e for e in sa.list()}
    if short_name in mapping:
        array = mapping[short_name]
        if array.dtype == dtype and array.dims == shape:
            return sa.attach(channel)
        sa.delete(short_name)

    return sa.create(channel, shape, dtype)
Ejemplo n.º 15
0
    def __del__(self):
        if self.use_shared_memory:
            self.logger.info('Deleting GT database from shared memory')
            cur_rank, num_gpus = common_utils.get_dist_info()
            sa_key = self.sampler_cfg.DB_DATA_PATH[0]
            if cur_rank % num_gpus == 0 and os.path.exists(
                    f"/dev/shm/{sa_key}"):
                SharedArray.delete(f"shm://{sa_key}")

            if num_gpus > 1:
                dist.barrier()
            self.logger.info('GT database has been removed from shared memory')
Ejemplo n.º 16
0
    def get_byte_range_mp(self,
                          s3_bucket,
                          s3_key,
                          s3_start,
                          s3_end,
                          block_size,
                          new_session=False):
        """Gets bytes from a S3 object within a range in parallel.

        :param str s3_bucket: name of the s3 bucket.
        :param str s3_key: name of the s3 key.
        :param int s3_start: begin of range.
        :param int s3_end: begin of range.
        :param int block_size: block size for download.
        :param bool new_session: Flag to create a new session or reuse existing session.
            True: create new session
            False: reuse existing session
        :return: Requested bytes
        """
        def work_get(block_number, array_name, s3_bucket, s3_key, s3_max_size,
                     block_size):
            start = block_number * block_size
            end = (block_number + 1) * block_size
            if end > s3_max_size:
                end = s3_max_size
            d = self.get_byte_range(s3_bucket, s3_key, start, end, True)
            # d = np.frombuffer(d, dtype=np.uint8, count=-1, offset=0)
            shared_array = sa.attach(array_name)
            shared_array[start:end] = d

        if not self.enable_s3:
            return self.get_byte_range(s3_bucket, s3_key, s3_start, s3_end,
                                       new_session)

        s3 = self.s3_resource(new_session)

        s3o = s3.Bucket(s3_bucket).Object(s3_key).get()
        s3_max_size = s3o['ContentLength']
        s3_obj_size = s3_end - s3_start
        num_streams = int(np.ceil(s3_obj_size / block_size))
        blocks = range(num_streams)
        array_name = '_'.join(
            ['S3IO', s3_bucket, s3_key,
             str(uuid.uuid4()),
             str(os.getpid())])
        sa.create(array_name, shape=s3_obj_size, dtype=np.uint8)
        shared_array = sa.attach(array_name)

        self.pool.map(work_get, blocks, repeat(array_name), repeat(s3_bucket),
                      repeat(s3_key), repeat(s3_max_size), repeat(block_size))

        sa.delete(array_name)
        return shared_array
Ejemplo n.º 17
0
def create_shared_array(name, shape, dtype):
    """Create shared array. Prompt if a file with the same name existed."""
    try:
        return sa.create(name, shape, dtype)
    except FileExistsError:
        response = ""
        while response.lower() not in ["y", "n", "yes", "no"]:
            response = input("Existing array (also named " + name +
                             ") was found. Replace it? (y/n) ")
        if response.lower() in ("n", "no"):
            sys.exit(0)
        sa.delete(name)
        return sa.create(name, shape, dtype)
Ejemplo n.º 18
0
def get_silhouette(profile, cluster, stepwise, pool):
    logging.info('Calculating pairwise distance ...')
    dist = getDistance(profile, 'p_dist', pool)
    with NamedTemporaryFile(dir='.', prefix='HCCeval_') as file:
        dist_buf = 'file://{0}.dist'.format(file.name)
        dist2 = sa.create(dist_buf, dist.shape[:2], dist.dtype)
        dist2[:] = dist[:, :, 0] + dist[:, :, 0].T
        del dist
        logging.info('Calculating Silhouette score ...')
        silhouette = np.array(
            pool.map(get_silhouette2, [[dist_buf, tag] for tag in cluster.T]))
        sa.delete(dist_buf)
    return silhouette
Ejemplo n.º 19
0
 def clear(
     self,
     name=None
 ):  # I previously wrote a __del__ function but it will automatically clear memory after script call
     if name is None:
         for key in self.keys:
             sa.delete(key)
         self.keys = []
     else:
         delkeys = [x for x in self.keys if x.startswith(name)]
         for key in delkeys:
             sa.delete(key)
         self.keys = [x for x in self.keys if not x.startswith(name)]
Ejemplo n.º 20
0
def getDistance(data, func_name, pool, start=0, allowed_missing=0.0):
    with NamedTemporaryFile(dir='.', prefix='HCC_') as file :
        prefix = 'file://{0}'.format(file.name)
        func = eval(func_name)
        mat_buf = '{0}.mat.sa'.format(prefix)
        mat = sa.create(mat_buf, shape = data.shape, dtype = data.dtype)
        mat[:] = data[:]
        dist_buf = '{0}.dist.sa'.format(prefix)
        dist = sa.create(dist_buf, shape = [mat.shape[0] - start, mat.shape[0], 2], dtype = np.int32)
        dist[:] = 0
        __parallel_dist(mat_buf, func, dist_buf, mat.shape, pool, start, allowed_missing)
        sa.delete(mat_buf)
        os.unlink(dist_buf[7:])
    return dist
Ejemplo n.º 21
0
def delete_shared_memory(file_names, wlabel=True):
    for fname in file_names:
        fn = fname.split('/')[-1][:12]
        if os.path.exists("/dev/shm/{}_xyz".format(fn)):
            SA.delete("shm://{}_xyz".format(fn))
            SA.delete("shm://{}_rgb".format(fn))
            if wlabel:
                SA.delete("shm://{}_label".format(fn))
                SA.delete("shm://{}_instance_label".format(fn))
Ejemplo n.º 22
0
    def load_delete(label):

        sa_name = args.dataset + split + '_' + label
        if args.cmd == 'load':
            data = dataset[label]

            sa_data = sa.create("shm://" + sa_name,
                                np.shape(data),
                                dtype=data.dtype)

            print('Transferring %s to shared memory ' % (sa_name))
            sa_data[:] = data
        elif args.cmd == 'delete':
            sa.delete(sa_name)

            print('Deleted %s from the shared memory' % sa_name)
        else:
            raise NotImplementedError
Ejemplo n.º 23
0
    def generate_stability_for_medoid(self, masked_parts_medoid,
                                      spatial_states_for_sw, l_labels_sorted):
        """
        Generate Stability maps for dynamic parcels for one seed

        :param masked_parts_medoid: Dask array
        :param spatial_states_for_sw:
        :param l_labels_sorted: list
        :param chunksize_voxels: int
            Size of the chunk in an array
        :return:
            darr_stab_maps
            len(l_labels_sorted)
        """
        try:
            SharedArray.delete('stab_maps')
        except:
            pass

        stab_maps = SharedArray.create(
            'stab_maps', (len(l_labels_sorted), masked_parts_medoid.shape[1]))

        def compute_stability_map(masked_parts_medoid, spatial_states_for_sw,
                                  state, idx):

            stab_maps[idx, :] = masked_parts_medoid[spatial_states_for_sw ==
                                                    state, :].mean(axis=0)

        processes = []
        idx = 0
        for state in l_labels_sorted:
            process = Process(target=compute_stability_map,
                              args=(masked_parts_medoid, spatial_states_for_sw,
                                    state, idx))
            processes.append(process)
            process.start()
            idx += 1

        for process in processes:
            process.join()

        SharedArray.delete('stab_maps')

        return stab_maps
Ejemplo n.º 24
0
  def transform(self, Xb, yb):
    shared_array_name = str(uuid4())
    try:
      shared_array = SharedArray.create(
          shared_array_name, [len(Xb), self.w, self.h, 3], dtype=np.float32)

      fnames, labels = Xb, yb
      args = []
      da_args = self.da_args()
      for i, fname in enumerate(fnames):
        args.append((i, shared_array_name, fname, da_args))

      self.pool.map(load_shared, args)
      Xb = np.array(shared_array, dtype=np.float32)

    finally:
      SharedArray.delete(shared_array_name)

    return Xb, labels
Ejemplo n.º 25
0
            def data_func(measurement):
                if not use_threads:
                    data = numpy.full(sources.shape + geobox.shape,
                                      measurement['nodata'],
                                      dtype=measurement['dtype'])
                    for index, datasets in numpy.ndenumerate(sources.values):
                        _fuse_measurement(
                            data[index],
                            datasets,
                            geobox,
                            measurement,
                            fuse_func=fuse_func,
                            skip_broken_datasets=skip_broken_datasets,
                            driver_manager=driver_manager)
                else:

                    def work_load_data(array_name, index, datasets):
                        data = sa.attach(array_name)
                        _fuse_measurement(
                            data[index],
                            datasets,
                            geobox,
                            measurement,
                            fuse_func=fuse_func,
                            skip_broken_datasets=skip_broken_datasets,
                            driver_manager=driver_manager)

                    array_name = '_'.join(
                        ['DCCORE',
                         str(uuid.uuid4()),
                         str(os.getpid())])
                    sa.create(array_name,
                              shape=sources.shape + geobox.shape,
                              dtype=measurement['dtype'])
                    data = sa.attach(array_name)
                    data[:] = measurement['nodata']

                    pool = ThreadPool(32)
                    pool.map(work_load_data, repeat(array_name),
                             *zip(*numpy.ndenumerate(sources.values)))
                    sa.delete(array_name)
                return data
Ejemplo n.º 26
0
    def transform(self, fundus, vessel, grade):
        shared_array_fundus_mean_subt_name = str(uuid4())
        shared_array_fundus_z_name = str(uuid4())
        shared_array_vessel_name = str(uuid4())
        
        try:
            shared_array_fundus_mean_subt = SharedArray.create(
                shared_array_fundus_mean_subt_name, [len(fundus), img_h, img_w, 3], dtype=np.float32)
            shared_array_fundus_z = SharedArray.create(
                shared_array_fundus_z_name, [len(fundus), img_h, img_w, 3], dtype=np.float32)
            shared_array_vessel = SharedArray.create(
                shared_array_vessel_name, [len(fundus), img_h, img_w, 1], dtype=np.float32)
            
            n_grades = len(grade)
            if self.grade_type == "DR":
                grade_onehot = np.zeros((n_grades, n_grade_dr))
            elif self.grade_type == "DME":
                grade_onehot = np.zeros((n_grades, n_grade_dme))
            for i in range(n_grades):
                grade_onehot[i, grade[i]] = 1
            
            args = []
            for i, _ in enumerate(fundus):
                args.append((i, shared_array_fundus_mean_subt_name, shared_array_fundus_z_name, shared_array_vessel_name, fundus[i], vessel[i], self.is_train, self.normalize))

            self.pool.map(load_shared, args)
            fundus_mean_subt_img = np.array(shared_array_fundus_mean_subt, dtype=np.float32)
            fundus_z_img = np.array(shared_array_fundus_z, dtype=np.float32)
            vessel_img = np.array(shared_array_vessel, dtype=np.float32)
        finally:
            SharedArray.delete(shared_array_fundus_mean_subt_name)
            SharedArray.delete(shared_array_fundus_z_name)
            SharedArray.delete(shared_array_vessel_name)

        return fundus, fundus_mean_subt_img, fundus_z_img, vessel_img, grade_onehot
Ejemplo n.º 27
0
    def transform(self, fundus, vessel, coords):
        shared_array_fundus_name = str(uuid4())
        shared_array_vessel_name = str(uuid4())
        shared_array_lm_name = str(uuid4())
        try:
            shared_array_fundus = SharedArray.create(
                shared_array_fundus_name, [len(fundus), img_h, img_w, 3],
                dtype=np.float32)
            shared_array_vessel = SharedArray.create(
                shared_array_vessel_name, [len(fundus), img_h, img_w, 1],
                dtype=np.float32)
            shared_array_lm = SharedArray.create(shared_array_lm_name,
                                                 [len(fundus), 4],
                                                 dtype=np.float32)

            args = []

            for i, fname in enumerate(fundus):
                args.append((i, shared_array_fundus_name,
                             shared_array_vessel_name, shared_array_lm_name,
                             fundus[i], vessel[i], coords[i], self.is_train))

            self.pool.map(load_shared, args)
            fundus_img = np.array(shared_array_fundus, dtype=np.float32)
            vessel_img = np.array(shared_array_vessel, dtype=np.float32)
            coords_arr = np.array(shared_array_lm, dtype=np.float32)
        finally:
            SharedArray.delete(shared_array_fundus_name)
            SharedArray.delete(shared_array_vessel_name)
            SharedArray.delete(shared_array_lm_name)

        return fundus_img, vessel_img, coords_arr, fundus
    def transform(self, fundus, grade):
        shared_array_ex_name = str(uuid4())
        shared_array_he_name = str(uuid4())
        shared_array_ma_name = str(uuid4())
        shared_array_se_name = str(uuid4())
        shared_array_fundus_rescale_mean_subtract_name = str(uuid4())
        
        try:
            shared_array_ex = SharedArray.create(
                shared_array_ex_name, (len(fundus),) + feature_shape_ex_he, dtype=np.float32)
            shared_array_he = SharedArray.create(
                shared_array_he_name, (len(fundus),) + feature_shape_ex_he, dtype=np.float32)
            shared_array_ma = SharedArray.create(
                shared_array_ma_name, (len(fundus),) + feature_shape_ma, dtype=np.float32)
            shared_array_se = SharedArray.create(
                shared_array_se_name, (len(fundus),) + feature_shape_se, dtype=np.float32)
            shared_array_fundus_rescale_mean_subtract = SharedArray.create(
                shared_array_fundus_rescale_mean_subtract_name, (len(fundus),) + img_shape, dtype=np.float32)
            
            args = []
            for i, _ in enumerate(fundus):
                args.append((i, shared_array_ex_name, shared_array_he_name, shared_array_ma_name, shared_array_se_name,
                              shared_array_fundus_rescale_mean_subtract_name, fundus[i], self.features_home, self.is_train))
    
            self.pool.map(load_shared, args)
            ex = np.array(shared_array_ex, dtype=np.float32)
            he = np.array(shared_array_he, dtype=np.float32)
            ma = np.array(shared_array_ma, dtype=np.float32)
            se = np.array(shared_array_se, dtype=np.float32)
            fundus_rescale_mean_subtract = np.array(shared_array_fundus_rescale_mean_subtract, dtype=np.float32)
            
        finally:
            SharedArray.delete(shared_array_fundus_rescale_mean_subtract_name)
            SharedArray.delete(shared_array_ex_name)
            SharedArray.delete(shared_array_he_name)
            SharedArray.delete(shared_array_ma_name)
            SharedArray.delete(shared_array_se_name)

        return fundus, ex, he, ma, se, fundus_rescale_mean_subtract, grade
Ejemplo n.º 29
0
    def clean_shared_memory(self):
        self.logger.info(
            f'Clean training data from shared memory (file limit={self.shared_memory_file_limit})'
        )

        cur_rank, num_gpus = common_utils.get_dist_info()
        all_infos = self.infos[:self.shared_memory_file_limit] \
            if self.shared_memory_file_limit < len(self.infos) else self.infos
        cur_infos = all_infos[cur_rank::num_gpus]
        for info in cur_infos:
            pc_info = info['point_cloud']
            sequence_name = pc_info['lidar_sequence']
            sample_idx = pc_info['sample_idx']

            sa_key = f'{sequence_name}___{sample_idx}'
            if not os.path.exists(f"/dev/shm/{sa_key}"):
                continue

            SharedArray.delete(f"shm://{sa_key}")

        if num_gpus > 1:
            dist.barrier()
        self.logger.info('Training data has been deleted from shared memory')
Ejemplo n.º 30
0
    def callback(self, method, body):
        with self.cycle_time.labels(module=self.module_name,
                                    name=socket.gethostname()).time():
            self.t0 = time.time()
            self.t1 = self.t0
            torch.cuda.set_device(np.random.randint(10 % 3))

            message = body.decode()
            if message == 'END':
                self.channel.basic_publish(exchange='',
                                           routing_key='reid',
                                           body=body)
                return

            frame_num = map(
                int,
                message.split('_')[-1])  # takes frame_num from adress

            images_list = sa.attach(message)
            self.log_time("Read image from shm:")

            bboxes = self.detector.predict_with_scores(images_list)
            self.log_time("Detector predicted:")

            bboxes = np.array([tensor[0].numpy() for tensor in bboxes[0]])
            self.log_time("Detector output into array converted:")

            if bboxes.shape[0] != 0:
                sh_mem_adress = f"shm://{self.module_name}_{frame_num}"
                try:
                    shared_mem = sa.create(sh_mem_adress, bboxes.shape)
                except:
                    sa.delete(sh_mem_adress)
                    shared_mem = sa.create(sh_mem_adress, bboxes.shape)
                self.log_time("Shared memory created:")

                # copy image to shared memory
                shared_mem[:] = np.array(bboxes)
                self.log_time("Detector copied to shared memory:")

                sa.delete(message)
                sa.delete(sh_mem_adress)

                del images_list, bboxes
                torch.cuda.empty_cache()

            self.channel.basic_ack(delivery_tag=method.delivery_tag)
            self.log_time('Full time:', from_start=True)
            self.last_success.labels(
                module=self.module_name,
                name=socket.gethostname()).set_to_current_time()
            #with self.cycle_time.labels(module=self.module_name, name=socket.gethostname()).time():
            push_to_gateway('pushgateway:9091',
                            job='Test ' + str(self.start_time),
                            registry=self.registry)
    def transform(self, fundus_fnames, vessel_fnames, seg_fnames):
        assert len(fundus_fnames) == len(vessel_fnames) and len(
            fundus_fnames) == len(seg_fnames)
        n_imgs = len(fundus_fnames)
        fundus_shared_array_name = str(uuid4())
        vessel_shared_array_name = str(uuid4())
        seg_shared_array_name = str(uuid4())
        try:
            fundus_shared_array = SharedArray.create(fundus_shared_array_name,
                                                     [n_imgs, img_h, img_w, 3],
                                                     dtype=np.float32)
            vessel_shared_array = SharedArray.create(vessel_shared_array_name,
                                                     [n_imgs, img_h, img_w, 1],
                                                     dtype=np.float32)
            seg_shared_array = SharedArray.create(
                seg_shared_array_name, [len(seg_fnames), img_h, img_w, 1],
                dtype=np.float32)

            args = []

            for i in range(n_imgs):
                args.append(
                    (i, fundus_shared_array_name, vessel_shared_array_name,
                     seg_shared_array_name, self.augment, fundus_fnames[i],
                     vessel_fnames[i], seg_fnames[i]))

            self.pool.map(load_shared, args)
            funduses = np.array(fundus_shared_array, dtype=np.float32)
            vessels = np.array(vessel_shared_array, dtype=np.float32)
            segs = np.array(seg_shared_array, dtype=np.float32)

        finally:
            SharedArray.delete(fundus_shared_array_name)
            SharedArray.delete(vessel_shared_array_name)
            SharedArray.delete(seg_shared_array_name)

        return fundus_fnames, funduses, vessels, segs
Ejemplo n.º 32
0
 def shutdown(self):
     if self.multiprocessing:
         self.executor.close()
         sa.delete(self.sharedprefix + 'W')
         sa.delete(self.sharedprefix + 'V')
         sa.delete(self.sharedprefix + 'Tau2')
         sa.delete(self.sharedprefix + 'sigma2')
         sa.delete(self.sharedprefix + 'lam2')
         sa.delete(self.sharedprefix + 'Constraints_A')
         sa.delete(self.sharedprefix + 'Constraints_C')
         sa.delete(self.sharedprefix + 'Delta_data')
         sa.delete(self.sharedprefix + 'Delta_row')
         sa.delete(self.sharedprefix + 'Delta_col')
         if self.Row_constraints is not None:
             sa.delete(self.sharedprefix + 'Row_constraints')
         if self.Mu_ep is not None:
             sa.delete(self.sharedprefix + 'Mu_ep')
             sa.delete(self.sharedprefix + 'Sigma_ep')
     else:
         self.executor.shutdown()
Ejemplo n.º 33
0
 def _cleanup(self):
     if self.wfunc is None:
         sa.delete(self.id)
Ejemplo n.º 34
0
 def cleanup():
     print('Cleaning up')
     sa.delete('creature_gfx')