def test_refit_none():
""" Test refit if model is None/[]
"""
refit = refit_record(None, 'ols', None)
assert refit is None
refit = refit_record([], 'ols', None)
assert refit is None
def test_refit_none():
""" Test refit if model is None/[]
"""
refit = refit_record(None, 'ols', None)
assert refit is None
refit = refit_record([], 'ols', None)
assert refit is None
def test_refit_nochange_rlm(sim_nochange):
""" Test record refitting of one record using robust linear models
"""
from yatsm.regression import RLM
estimator = RLM(maxiter=10)
refit = refit_record(sim_nochange, 'rlm', estimator, keep_regularized=True)
assert 'rlm_coef' in refit.dtype.names
assert 'rlm_rmse' in refit.dtype.names
coef = np.array([[-3.84164779e+03, -3.84164779e+03],
[5.26200993e-03, 5.26200993e-03]])
rmse = np.array([0.96866816, 0.96866816])
np.testing.assert_allclose(refit[0]['rlm_coef'], coef)
np.testing.assert_allclose(refit[0]['rlm_rmse'], rmse)
def test_refit_nochange_rlm(sim_nochange):
""" Test record refitting of one record using robust linear models
"""
from yatsm.regression import RLM
estimator = RLM(maxiter=10)
refit = refit_record(sim_nochange, 'rlm', estimator,
keep_regularized=True)
assert 'rlm_coef' in refit.dtype.names
assert 'rlm_rmse' in refit.dtype.names
coef = np.array([[-3.84164779e+03, -3.84164779e+03],
[5.26200993e-03, 5.26200993e-03]])
rmse = np.array([0.96866816, 0.96866816])
np.testing.assert_allclose(refit[0]['rlm_coef'], coef)
np.testing.assert_allclose(refit[0]['rlm_rmse'], rmse)
def test_refit_nochange_reg(sim_nochange):
""" Test refit ``keep_regularized=False`` (i.e., not ignoring coef == 0)
"""
from sklearn.linear_model import LinearRegression as OLS
estimator = OLS()
refit = refit_record(sim_nochange, 'ols', estimator,
keep_regularized=False)
assert 'ols_coef' in refit.dtype.names
assert 'ols_rmse' in refit.dtype.names
coef = np.array([[-3.83016528e+03, -3.83016528e+03],
[5.24635240e-03, 5.24635240e-03]])
rmse = np.array([0.96794599, 0.96794599])
np.testing.assert_allclose(refit[0]['ols_coef'], coef)
np.testing.assert_allclose(refit[0]['ols_rmse'], rmse)
def test_refit_nochange_reg(sim_nochange):
""" Test refit ``keep_regularized=False`` (i.e., not ignoring coef == 0)
"""
from sklearn.linear_model import LinearRegression as OLS
estimator = OLS()
refit = refit_record(sim_nochange,
'ols',
estimator,
keep_regularized=False)
assert 'ols_coef' in refit.dtype.names
assert 'ols_rmse' in refit.dtype.names
coef = np.array([[-3.83016528e+03, -3.83016528e+03],
[5.24635240e-03, 5.24635240e-03]])
rmse = np.array([0.96794599, 0.96794599])
np.testing.assert_allclose(refit[0]['ols_coef'], coef)
np.testing.assert_allclose(refit[0]['ols_rmse'], rmse)
def test_refit_issue_79(sim_nochange):
""" Issue 79: missing coverage for case when record['coef'] are all zero
Fix is to use ``refit_[(coef|rmse)]`` prefix variable to index the record
name
"""
from yatsm.regression import RLM
estimator = RLM(maxiter=10)
# Set record.coef to 0.
sim_nochange.record['coef'] = np.zeros_like(sim_nochange.record['coef'])
refit = refit_record(sim_nochange, 'rlm', estimator, keep_regularized=True)
assert 'rlm_coef' in refit.dtype.names
assert 'rlm_rmse' in refit.dtype.names
coef = np.zeros_like(sim_nochange.record[0]['coef'])
rmse = np.array([0.97117668, 0.97117668])
np.testing.assert_allclose(refit[0]['rlm_coef'], coef)
np.testing.assert_allclose(refit[0]['rlm_rmse'], rmse)
def test_refit_issue_79(sim_nochange):
""" Issue 79: missing coverage for case when record['coef'] are all zero
Fix is to use ``refit_[(coef|rmse)]`` prefix variable to index the record
name
"""
from yatsm.regression import RLM
estimator = RLM(maxiter=10)
# Set record.coef to 0.
sim_nochange.record['coef'] = np.zeros_like(sim_nochange.record['coef'])
refit = refit_record(sim_nochange, 'rlm', estimator,
keep_regularized=True)
assert 'rlm_coef' in refit.dtype.names
assert 'rlm_rmse' in refit.dtype.names
coef = np.zeros_like(sim_nochange.record[0]['coef'])
rmse = np.array([0.97117668, 0.97117668])
np.testing.assert_allclose(refit[0]['rlm_coef'], coef)
np.testing.assert_allclose(refit[0]['rlm_rmse'], rmse)
def line(ctx, config, job_number, total_jobs,
resume, check_cache, do_not_run, verbose_yatsm):
if verbose_yatsm:
logger_algo.setLevel(logging.DEBUG)
# Parse config
cfg = parse_config_file(config)
if ('phenology' in cfg and cfg['phenology'].get('enable')) and not pheno:
click.secho('Could not import yatsm.phenology but phenology metrics '
'are requested', fg='red')
click.secho('Error: %s' % pheno_exception, fg='red')
raise click.Abort()
# Make sure output directory exists and is writable
output_dir = cfg['dataset']['output']
try:
os.makedirs(output_dir)
except OSError as e:
# File exists
if e.errno == 17:
pass
elif e.errno == 13:
click.secho('Cannot create output directory %s' % output_dir,
fg='red')
raise click.Abort()
if not os.access(output_dir, os.W_OK):
click.secho('Cannot write to output directory %s' % output_dir,
fg='red')
raise click.Abort()
# Test existence of cache directory
read_cache, write_cache = test_cache(cfg['dataset'])
logger.info('Job {i} of {n} - using config file {f}'.format(i=job_number,
n=total_jobs,
f=config))
df = csvfile_to_dataframe(cfg['dataset']['input_file'],
cfg['dataset']['date_format'])
df['image_ID'] = get_image_IDs(df['filename'])
# Get attributes of one of the images
nrow, ncol, nband, dtype = get_image_attribute(df['filename'][0])
# Calculate the lines this job ID works on
job_lines = distribute_jobs(job_number, total_jobs, nrow)
logger.debug('Responsible for lines: {l}'.format(l=job_lines))
# Calculate X feature input
dates = np.asarray(df['date'])
kws = {'x': dates}
kws.update(df.to_dict())
X = patsy.dmatrix(cfg['YATSM']['design_matrix'], kws)
cfg['YATSM']['design'] = X.design_info.column_name_indexes
# Form YATSM class arguments
fit_indices = np.arange(cfg['dataset']['n_bands'])
if cfg['dataset']['mask_band'] is not None:
fit_indices = fit_indices[:-1]
if cfg['YATSM']['reverse']:
X = np.flipud(X)
# Create output metadata to save
md = {
'YATSM': cfg['YATSM'],
cfg['YATSM']['algorithm']: cfg[cfg['YATSM']['algorithm']]
}
if cfg['phenology']['enable']:
md.update({'phenology': cfg['phenology']})
# Begin process
start_time_all = time.time()
for line in job_lines:
out = get_output_name(cfg['dataset'], line)
if resume:
try:
np.load(out)
except:
pass
else:
logger.debug('Already processed line %s' % line)
continue
logger.debug('Running line %s' % line)
start_time = time.time()
Y = read_line(line, df['filename'], df['image_ID'], cfg['dataset'],
ncol, nband, dtype,
read_cache=read_cache, write_cache=write_cache,
validate_cache=False)
if do_not_run:
continue
if cfg['YATSM']['reverse']:
Y = np.fliplr(Y)
output = []
for col in np.arange(Y.shape[-1]):
_Y = Y.take(col, axis=2)
# Mask
idx_mask = cfg['dataset']['mask_band'] - 1
valid = cyprep.get_valid_mask(
_Y,
cfg['dataset']['min_values'],
cfg['dataset']['max_values']).astype(bool)
valid *= np.in1d(_Y.take(idx_mask, axis=0),
cfg['dataset']['mask_values'],
invert=True).astype(np.bool)
_Y = np.delete(_Y, idx_mask, axis=0)[:, valid]
_X = X[valid, :]
_dates = dates[valid]
# Run model
cls = cfg['YATSM']['algorithm_cls']
algo_cfg = cfg[cfg['YATSM']['algorithm']]
yatsm = cls(lm=cfg['YATSM']['prediction_object'],
**algo_cfg.get('init', {}))
yatsm.px = col
yatsm.py = line
try:
yatsm.fit(_X, _Y, _dates, **algo_cfg.get('fit', {}))
except TSLengthException:
continue
if yatsm.record is None or len(yatsm.record) == 0:
continue
# Postprocess
if cfg['YATSM'].get('commission_alpha'):
yatsm.record = postprocess.commission_test(
yatsm, cfg['YATSM']['commission_alpha'])
for prefix, lm in zip(cfg['YATSM']['refit']['prefix'],
cfg['YATSM']['refit']['prediction_object']):
yatsm.record = postprocess.refit_record(yatsm, prefix, lm,
keep_regularized=True)
if cfg['phenology']['enable']:
pcfg = cfg['phenology']
ltm = pheno.LongTermMeanPhenology(**pcfg.get('init', {}))
yatsm.record = ltm.fit(yatsm, **pcfg.get('fit', {}))
output.extend(yatsm.record)
logger.debug(' Saving YATSM output to %s' % out)
np.savez(out,
record=np.array(output),
version=__version__,
metadata=md)
run_time = time.time() - start_time
logger.debug('Line %s took %ss to run' % (line, run_time))
logger.info('Completed {n} lines in {m} minutes'.format(
n=len(job_lines),
m=round((time.time() - start_time_all) / 60.0, 2)))
