def _fetch_results_saved(self): """ Read YATSM results and return """ self.yatsm_model = MockResult() row, col = self.series[0].py, self.series[0].px data_cfg = { 'output': os.path.join(self.location, self.config['results_folder'].value), 'output_prefix': (self.config['results_pattern'].value .replace('*', '')) } result_filename = get_output_name(data_cfg, row) logger.info('Attempting to open: {f}'.format(f=result_filename)) if not os.path.isfile(result_filename): qgis_log('Could not find result for row {r} ({fn})'.format( r=row, fn=result_filename)) return z = np.load(result_filename) if 'record' not in z.files: raise KeyError('Cannot find "record" within saved result ({})' .format(result_filename)) if 'metadata' not in z.files: raise KeyError('Cannot find "metadata" within saved result ({})' .format(result_filename)) metadata = z['metadata'].item() if 'design' not in metadata['YATSM']: raise KeyError('Cannot find "design" within saved result metadata ' '({})'.format(result_filename)) self._design_info = metadata['YATSM']['design'] rec = z['record'] idx = np.where((rec['px'] == col) & (rec['py'] == row))[0] self.yatsm_model.record = rec[idx]
def classify(ctx, config, algo, job_number, total_jobs, resume): cfg = parse_config_file(config) df = csvfile_to_dataframe(cfg['dataset']['input_file'], cfg['dataset']['date_format']) nrow = get_image_attribute(df['filename'][0])[0] classifier = joblib.load(algo) # Split into lines and classify job_lines = distribute_jobs(job_number, total_jobs, nrow) logger.debug('Responsible for lines: {l}'.format(l=job_lines)) start_time = time.time() logger.info('Starting to run lines') for job_line in job_lines: filename = get_output_name(cfg['dataset'], job_line) if not os.path.exists(filename): logger.warning('No model result found for line {l} ' '(file {f})'.format(l=job_line, f=filename)) pass if resume and try_resume(filename): logger.debug('Already processed line {l}'.format(l=job_line)) continue logger.debug('Classifying line {l}'.format(l=job_line)) classify_line(filename, classifier) logger.debug('Completed {n} lines in {m} minutes'.format( n=len(job_lines), m=round((time.time() - start_time) / 60.0, 2)) )
def main(args): """ Classify dataset """ # Parse config and file data dataset_config, yatsm_config = parse_config_file(args['config_file']) # Get some attributes about the dataset dates, sensors, images = csvfile_to_dataset( dataset_config['input_file'], date_format=dataset_config['date_format'] ) nrow, _, _, _ = get_image_attribute(images[0]) # Read in the saved classification result try: _ = open(args['algo']) except: logger.error('Could not open pickled classifier') sys.exit(1) classifier = joblib.load(args['algo']) # Split into lines and classify job_lines = calculate_lines(args['job_number'] - 1, args['total_jobs'], nrow) logger.debug('Responsible for lines: {l}'.format(l=job_lines)) start_time = time.time() logger.info('Starting to run lines') for job_line in job_lines: filename = get_output_name(dataset_config, job_line) if not os.path.exists(filename): logger.warning('No model result found for line {l} ' '(file {f})'.format(l=job_line, f=filename)) pass if args['resume'] and try_resume(filename): logger.debug('Already processed line {l}'.format(l=job_line)) continue logger.debug('Classifying line {l}'.format(l=job_line)) classify_line(filename, classifier) logger.debug('Completed {n} lines in {m} minutes'.format( n=len(job_lines), m=round((time.time() - start_time) / 60.0, 2)) )
def _fetch_results_saved(self): """ Read YATSM results and return """ self.yatsm_model = MockResult() row, col = self.series[0].py, self.series[0].px data_cfg = { 'output': os.path.join(self.location, self.config['results_folder'].value), 'output_prefix': (self.config['results_pattern'].value.replace('*', '')) } result_filename = get_output_name(data_cfg, row) logger.info('Attempting to open: {f}'.format(f=result_filename)) if not os.path.isfile(result_filename): qgis_log('Could not find result for row {r} ({fn})'.format( r=row, fn=result_filename)) return z = np.load(result_filename) if 'record' not in z.files: raise KeyError( 'Cannot find "record" within saved result ({})'.format( result_filename)) if 'metadata' not in z.files: raise KeyError( 'Cannot find "metadata" within saved result ({})'.format( result_filename)) metadata = z['metadata'].item() if 'design' not in metadata['YATSM']: raise KeyError('Cannot find "design" within saved result metadata ' '({})'.format(result_filename)) self._design = metadata['YATSM']['design_matrix'] self._design_info = metadata['YATSM']['design'] rec = z['record'] idx = np.where((rec['px'] == col) & (rec['py'] == row))[0] self.yatsm_model.record = rec[idx]
def get_training_inputs(cfg, exit_on_missing=False): """ Returns X features and y labels specified in config file Args: cfg (dict): YATSM configuration dictionary exit_on_missing (bool, optional): exit if input feature cannot be found Returns: X (np.ndarray): matrix of feature inputs for each training data sample y (np.ndarray): array of labeled training data samples row (np.ndarray): row pixel locations of `y` col (np.ndarray): column pixel locations of `y` labels (np.ndarraY): label of `y` if found, else None """ # Find and parse training data roi = reader.read_image(cfg["classification"]["training_image"]) logger.debug("Read in training data") if len(roi) == 2: logger.info("Found labels for ROIs -- including in output") labels = roi[1] else: roi = roi[0] labels = None # Determine start and end dates of training sample relevance try: training_start = dt.strptime( cfg["classification"]["training_start"], cfg["classification"]["training_date_format"] ).toordinal() training_end = dt.strptime( cfg["classification"]["training_end"], cfg["classification"]["training_date_format"] ).toordinal() except: logger.error("Failed to parse training data start or end dates") raise # Loop through samples in ROI extracting features mask_values = cfg["classification"]["roi_mask_values"] mask = ~np.in1d(roi, mask_values).reshape(roi.shape) row, col = np.where(mask) y = roi[row, col] X = [] out_y = [] out_row = [] out_col = [] _row_previous = None for _row, _col, _y in izip(row, col, y): # Load result if _row != _row_previous: output_name = utils.get_output_name(cfg["dataset"], _row) try: rec = np.load(output_name)["record"] _row_previous = _row except: logger.error("Could not open saved result file %s" % output_name) if exit_on_missing: raise else: continue # Find intersecting time segment i = np.where((rec["start"] < training_start) & (rec["end"] > training_end) & (rec["px"] == _col))[0] if i.size == 0: logger.debug("Could not find model for label %i at x/y %i/%i" % (_y, _col, _row)) continue elif i.size > 1: raise TrainingDataException("Found more than one valid model for label %i at x/y %i/%i" % (_y, _col, _row)) # Extract coefficients with intercept term rescaled coef = rec[i]["coef"][0, :] coef[0, :] = coef[0, :] + coef[1, :] * (rec[i]["start"] + rec[i]["end"]) / 2.0 X.append(np.concatenate((coef.reshape(coef.size), rec[i]["rmse"][0]))) out_y.append(_y) out_row.append(_row) out_col.append(_col) out_row = np.array(out_row) out_col = np.array(out_col) if labels is not None: labels = labels[out_row, out_col] return np.array(X), np.array(out_y), out_row, out_col, labels
def line(ctx, config, job_number, total_jobs, resume, check_cache, do_not_run, verbose_yatsm): if verbose_yatsm: logger_algo.setLevel(logging.DEBUG) # Parse config cfg = parse_config_file(config) if ('phenology' in cfg and cfg['phenology'].get('enable')) and not pheno: click.secho('Could not import yatsm.phenology but phenology metrics ' 'are requested', fg='red') click.secho('Error: %s' % pheno_exception, fg='red') raise click.Abort() # Make sure output directory exists and is writable output_dir = cfg['dataset']['output'] try: os.makedirs(output_dir) except OSError as e: # File exists if e.errno == 17: pass elif e.errno == 13: click.secho('Cannot create output directory %s' % output_dir, fg='red') raise click.Abort() if not os.access(output_dir, os.W_OK): click.secho('Cannot write to output directory %s' % output_dir, fg='red') raise click.Abort() # Test existence of cache directory read_cache, write_cache = test_cache(cfg['dataset']) logger.info('Job {i} of {n} - using config file {f}'.format(i=job_number, n=total_jobs, f=config)) df = csvfile_to_dataframe(cfg['dataset']['input_file'], cfg['dataset']['date_format']) df['image_ID'] = get_image_IDs(df['filename']) # Get attributes of one of the images nrow, ncol, nband, dtype = get_image_attribute(df['filename'][0]) # Calculate the lines this job ID works on job_lines = distribute_jobs(job_number, total_jobs, nrow) logger.debug('Responsible for lines: {l}'.format(l=job_lines)) # Calculate X feature input dates = np.asarray(df['date']) kws = {'x': dates} kws.update(df.to_dict()) X = patsy.dmatrix(cfg['YATSM']['design_matrix'], kws) cfg['YATSM']['design'] = X.design_info.column_name_indexes # Form YATSM class arguments fit_indices = np.arange(cfg['dataset']['n_bands']) if cfg['dataset']['mask_band'] is not None: fit_indices = fit_indices[:-1] if cfg['YATSM']['reverse']: X = np.flipud(X) # Create output metadata to save md = { 'YATSM': cfg['YATSM'], cfg['YATSM']['algorithm']: cfg[cfg['YATSM']['algorithm']] } if cfg['phenology']['enable']: md.update({'phenology': cfg['phenology']}) # Begin process start_time_all = time.time() for line in job_lines: out = get_output_name(cfg['dataset'], line) if resume: try: np.load(out) except: pass else: logger.debug('Already processed line %s' % line) continue logger.debug('Running line %s' % line) start_time = time.time() Y = read_line(line, df['filename'], df['image_ID'], cfg['dataset'], ncol, nband, dtype, read_cache=read_cache, write_cache=write_cache, validate_cache=False) if do_not_run: continue if cfg['YATSM']['reverse']: Y = np.fliplr(Y) output = [] for col in np.arange(Y.shape[-1]): _Y = Y.take(col, axis=2) # Mask idx_mask = cfg['dataset']['mask_band'] - 1 valid = cyprep.get_valid_mask( _Y, cfg['dataset']['min_values'], cfg['dataset']['max_values']).astype(bool) valid *= np.in1d(_Y.take(idx_mask, axis=0), cfg['dataset']['mask_values'], invert=True).astype(np.bool) _Y = np.delete(_Y, idx_mask, axis=0)[:, valid] _X = X[valid, :] _dates = dates[valid] # Run model cls = cfg['YATSM']['algorithm_cls'] algo_cfg = cfg[cfg['YATSM']['algorithm']] yatsm = cls(lm=cfg['YATSM']['prediction_object'], **algo_cfg.get('init', {})) yatsm.px = col yatsm.py = line try: yatsm.fit(_X, _Y, _dates, **algo_cfg.get('fit', {})) except TSLengthException: continue if yatsm.record is None or len(yatsm.record) == 0: continue # Postprocess if cfg['YATSM'].get('commission_alpha'): yatsm.record = postprocess.commission_test( yatsm, cfg['YATSM']['commission_alpha']) for prefix, lm in zip(cfg['YATSM']['refit']['prefix'], cfg['YATSM']['refit']['prediction_object']): yatsm.record = postprocess.refit_record(yatsm, prefix, lm, keep_regularized=True) if cfg['phenology']['enable']: pcfg = cfg['phenology'] ltm = pheno.LongTermMeanPhenology(**pcfg.get('init', {})) yatsm.record = ltm.fit(yatsm, **pcfg.get('fit', {})) output.extend(yatsm.record) logger.debug(' Saving YATSM output to %s' % out) np.savez(out, record=np.array(output), version=__version__, metadata=md) run_time = time.time() - start_time logger.debug('Line %s took %ss to run' % (line, run_time)) logger.info('Completed {n} lines in {m} minutes'.format( n=len(job_lines), m=round((time.time() - start_time_all) / 60.0, 2)))
def main(dataset_config, yatsm_config, check=False, resume=False, do_not_run=False, read_cache=False, write_cache=False, validate_cache=False): """ Read in dataset and YATSM for a complete line Args: dataset_config (dict): dict of dataset configuration options yatsm_config (dict): dict of YATSM algorithm options check (bool, optional): check to make sure images are readible resume (bool, optional): do not overwrite existing results, instead continue from first non-existing result file do_not_run (bool, optional): Don't run YATSM read_cache (bool, optional): try to read from cache directory (default: False) write_cache (bool, optional): try to to write to cache directory (default: False) validate_cache (bool, optional): ensure data from cache file come from images specified in configuration (default: False) """ # Read in dataset dates, sensors, images = csvfile_to_dataset( dataset_config['input_file'], date_format=dataset_config['date_format'] ) image_IDs = get_image_IDs(images) # Check for existence of files and remove missing if check: to_delete = [] for i, img in enumerate(images): if not os.path.isfile(img): logger.warning('Could not find file {f} -- removing'. format(f=img)) to_delete.append(i) if len(to_delete) == 0: logger.debug('Checked and found all input images') else: logger.warning('Removing {n} images'.format(n=len(to_delete))) dates = np.delete(dates, np.array(to_delete)) images = np.delete(images, np.array(to_delete)) # Get attributes of one of the images nrow, ncol, nband, dtype = get_image_attribute(images[0]) # Calculate the lines this job ID works on job_lines = calculate_lines(job_number, total_jobs, nrow) logger.debug('Responsible for lines: {l}'.format(l=job_lines)) # Calculate X feature input X = patsy.dmatrix(yatsm_config['design_matrix'], {'x': dates, 'sensor': sensors}) # Start running YATSM start_time_all = time.time() logger.info('Starting to run lines') for job_line in job_lines: if resume: try: z = np.load(get_output_name(dataset_config, job_line)) except: pass else: del z logger.debug('Already processed line {l}'.format(l=job_line)) continue logger.debug('Running line {l}'.format(l=job_line)) start_time = time.time() try: run_line(job_line, X, images, image_IDs, dataset_config, yatsm_config, nrow, ncol, nband, dtype, do_not_run=do_not_run, read_cache=read_cache, write_cache=write_cache, validate_cache=validate_cache) except Exception as e: logger.error('Could not process line {l}'.format(l=job_line)) logger.error(type(e)) logger.error(str(e)) logger.debug('Took {s}s to run'.format( s=round(time.time() - start_time, 2))) logger.info('Completed {n} lines in {m} minutes'.format( n=len(job_lines), m=round((time.time() - start_time_all) / 60.0, 2) ))
def run_line(line, X, images, image_IDs, dataset_config, yatsm_config, nrow, ncol, nband, dtype, do_not_run=False, read_cache=False, write_cache=False, validate_cache=False): """ Runs YATSM for a line Args: line (int): line to be run from image dates (ndarray): np.array of X feature from ordinal dates images (ndarray): np.array of image filenames image_IDs (iterable): list image identifying strings dataset_config (dict): dict of dataset configuration options yatsm_config (dict): dict of YATSM algorithm options nrow (int): number of rows ncol (int): number of columns nband (int): number of bands dtype (type): NumPy datatype do_not_run (bool, optional): don't run YATSM read_cache (bool, optional): try to read from cache directory (default: False) write_cache (bool, optional): try to to write to cache directory (default: False) validate_cache (bool, optional): ensure data from cache file come from images specified in configuration (default: False) """ # Setup output output = [] Y = read_line(line, images, image_IDs, dataset_config, ncol, nband, dtype, read_cache=read_cache, write_cache=write_cache, validate_cache=validate_cache) if do_not_run: return # About to run YATSM logger.debug(' running YATSM') # Raise or lower logging level for YATSM _level = logger.level logger.setLevel(loglevel_YATSM) for c in xrange(Y.shape[-1]): try: result = run_pixel(X, Y[..., c], dataset_config, yatsm_config, px=c, py=line) except TSLengthException: continue output.extend(result) # Return logging level logger.setLevel(_level) # Save output outfile = get_output_name(dataset_config, line) logger.debug(' saving YATSM output to {f}'.format(f=outfile)) np.savez(outfile, version=__version__, consecutive=yatsm_config['consecutive'], threshold=yatsm_config['threshold'], min_obs=yatsm_config['min_obs'], min_rmse=yatsm_config['min_rmse'], test_indices=yatsm_config['test_indices'], design=yatsm_config['design_matrix'], design_matrix=X.design_info.column_name_indexes, retrain_time=yatsm_config['retrain_time'], screening=yatsm_config['screening'], screening_crit=yatsm_config['screening_crit'], remove_noise=yatsm_config['remove_noise'], dynamic_rmse=yatsm_config['dynamic_rmse'], commission_alpha=yatsm_config['commission_alpha'], reverse=yatsm_config['reverse'], robust=yatsm_config['robust'], lassocv=yatsm_config['lassocv'], record=np.array(output))
def main(dataset_config, yatsm_config, check=False, resume=False, do_not_run=False, read_cache=False, write_cache=False, validate_cache=False): """ Read in dataset and YATSM for a complete line Args: dataset_config (dict): dict of dataset configuration options yatsm_config (dict): dict of YATSM algorithm options check (bool, optional): check to make sure images are readible resume (bool, optional): do not overwrite existing results, instead continue from first non-existing result file do_not_run (bool, optional): Don't run YATSM read_cache (bool, optional): try to read from cache directory (default: False) write_cache (bool, optional): try to to write to cache directory (default: False) validate_cache (bool, optional): ensure data from cache file come from images specified in configuration (default: False) """ # Read in dataset dates, sensors, images = csvfile_to_dataset( dataset_config['input_file'], date_format=dataset_config['date_format']) image_IDs = get_image_IDs(images) # Check for existence of files and remove missing if check: to_delete = [] for i, img in enumerate(images): if not os.path.isfile(img): logger.warning( 'Could not find file {f} -- removing'.format(f=img)) to_delete.append(i) if len(to_delete) == 0: logger.debug('Checked and found all input images') else: logger.warning('Removing {n} images'.format(n=len(to_delete))) dates = np.delete(dates, np.array(to_delete)) images = np.delete(images, np.array(to_delete)) # Get attributes of one of the images nrow, ncol, nband, dtype = get_image_attribute(images[0]) # Calculate the lines this job ID works on job_lines = calculate_lines(job_number, total_jobs, nrow) logger.debug('Responsible for lines: {l}'.format(l=job_lines)) # Calculate X feature input X = patsy.dmatrix(yatsm_config['design_matrix'], { 'x': dates, 'sensor': sensors }) # Start running YATSM start_time_all = time.time() logger.info('Starting to run lines') for job_line in job_lines: if resume: try: z = np.load(get_output_name(dataset_config, job_line)) except: pass else: del z logger.debug('Already processed line {l}'.format(l=job_line)) continue logger.debug('Running line {l}'.format(l=job_line)) start_time = time.time() try: run_line(job_line, X, images, image_IDs, dataset_config, yatsm_config, nrow, ncol, nband, dtype, do_not_run=do_not_run, read_cache=read_cache, write_cache=write_cache, validate_cache=validate_cache) except Exception as e: logger.error('Could not process line {l}'.format(l=job_line)) logger.error(type(e)) logger.error(str(e)) logger.debug( 'Took {s}s to run'.format(s=round(time.time() - start_time, 2))) logger.info('Completed {n} lines in {m} minutes'.format( n=len(job_lines), m=round((time.time() - start_time_all) / 60.0, 2)))
def get_training_inputs(dataset_config, exit_on_missing=False): """ Returns X features and y labels specified in config file Args: dataset_config (dict): dataset configuration exit_on_missing (bool, optional): exit if input feature cannot be found Returns: X (np.ndarray): matrix of feature inputs for each training data sample y (np.ndarray): array of labeled training data samples row (np.ndarray): row pixel locations of `y` col (np.ndarray): column pixel locations of `y` labels (np.ndarraY): label of `y` if found, else None """ # Find and parse training data roi = reader.read_image(dataset_config['training_image']) logger.debug('Read in training data') if len(roi) == 2: logger.info('Found labels for ROIs -- including in output') labels = roi[1] else: roi = roi[0] labels = None # Determine start and end dates of training sample relevance try: training_start = dt.strptime( dataset_config['training_start'], dataset_config['training_date_format']).toordinal() training_end = dt.strptime( dataset_config['training_end'], dataset_config['training_date_format']).toordinal() except: logger.error('Failed to parse training data start or end dates') raise # Loop through samples in ROI extracting features mask = ~np.in1d(roi, dataset_config['roi_mask_values']).reshape(roi.shape) row, col = np.where(mask) y = roi[row, col] X = [] out_y = [] out_row = [] out_col = [] rec = None _row_previous = None for _row, _col, _y in izip(row, col, y): # Load result if _row != _row_previous: try: rec = np.load(utils.get_output_name( dataset_config, _row))['record'] _row_previous = _row except: logger.error('Could not open saved result file {f}'.format( f=utils.get_output_name(dataset_config, _row))) if exit_on_missing: raise else: continue # Find intersecting time segment i = np.where((rec['start'] < training_start) & (rec['end'] > training_end) & (rec['px'] == _col))[0] if i.size == 0: logger.debug( 'Could not find model for label {l} at x/y {c}/{r}'.format( l=_y, c=_col, r=_row)) continue elif i.size > 1: raise TrainingDataException('Found more than one valid model for \ label {l} at x/y {x}/{y}'.format(l=_y, x=_col, y=_row)) # Extract coefficients with intercept term rescaled coef = rec[i]['coef'][0, :] coef[0, :] = (coef[0, :] + coef[1, :] * (rec[i]['start'] + rec[i]['end']) / 2.0) X.append(np.concatenate( (coef.reshape(coef.size), rec[i]['rmse'][0]))) out_y.append(_y) out_row.append(_row) out_col.append(_col) if not out_y: logger.error('Could not find any matching timeseries segments') raise click.Abort() logger.info('Found matching time segments for {m} out of {n} labels'. format(m=len(out_y), n=y.size)) out_row = np.array(out_row) out_col = np.array(out_col) if labels is not None: labels = labels[out_row, out_col] return (np.array(X), np.array(out_y), out_row, out_col, labels)
def get_training_inputs(dataset_config, exit_on_missing=False): """ Returns X features and y labels specified in config file Args: dataset_config (dict): dataset configuration exit_on_missing (bool, optional): exit if input feature cannot be found Returns: X (np.ndarray): matrix of feature inputs for each training data sample y (np.ndarray): array of labeled training data samples row (np.ndarray): row pixel locations of `y` col (np.ndarray): column pixel locations of `y` labels (np.ndarraY): label of `y` if found, else None """ # Find and parse training data try: roi_ds = gdal.Open(dataset_config['training_image'], gdal.GA_ReadOnly) except: logger.error('Could not read in training image') raise logger.info('Reading in training data') roi = roi_ds.GetRasterBand(1).ReadAsArray() if roi_ds.RasterCount == 2: logger.info('Found labels for ROIs -- including in output') labels = roi_ds.GetRasterBand(2).ReadAsArray() else: labels = None # Determine start and end dates of training sample relevance try: training_start = dt.strptime( dataset_config['training_start'], dataset_config['training_date_format']).toordinal() training_end = dt.strptime( dataset_config['training_end'], dataset_config['training_date_format']).toordinal() except: logger.error('Failed to parse training data start or end dates') raise # Loop through samples in ROI extracting features mask = ~np.in1d(roi, dataset_config['roi_mask_values']).reshape(roi.shape) row, col = np.where(mask) y = roi[row, col] X = [] out_y = [] out_row = [] out_col = [] rec = None _row_previous = None for _row, _col, _y in izip(row, col, y): # Load result if _row != _row_previous: try: rec = np.load(utils.get_output_name(dataset_config, _row))['record'] _row_previous = _row except: logger.error('Could not open saved result file {f}'.format( f=utils.get_output_name(dataset_config, _row))) if exit_on_missing: raise else: continue # Find intersecting time segment i = np.where((rec['start'] < training_start) & (rec['end'] > training_end) & (rec['px'] == _col))[0] if i.size == 0: logger.debug( 'Could not find model for label {l} at x/y {c}/{r}'.format( l=_y, c=_col, r=_row)) continue elif i.size > 1: raise TrainingDataException('Found more than one valid model for \ label {l} at x/y {x}/{y}'.format(l=_y, x=_col, y=_row)) # Extract coefficients with intercept term rescaled coef = rec[i]['coef'][0, :] coef[0, :] = (coef[0, :] + coef[1, :] * (rec[i]['start'] + rec[i]['end']) / 2.0) X.append(np.concatenate((coef.reshape(coef.size), rec[i]['rmse'][0]))) out_y.append(_y) out_row.append(_row) out_col.append(_col) if not out_y: logger.error('Could not find any matching timeseries segments') sys.exit(1) logger.info( 'Found matching time segments for {m} out of {n} labels'.format( m=len(out_y), n=y.size)) out_row = np.array(out_row) out_col = np.array(out_col) if labels is not None: labels = labels[out_row, out_col] return (np.array(X), np.array(out_y), out_row, out_col, labels)