def plot3d():
"""Create a GIF from 3D image in the training set."""
for channel in ["ch1", "ch2", "ch3", "ch4", "ch6"]:
config = default_config()
config.features = [Feature.image]
config.channels = [channel]
config.crop_size = (300, 300)
config.time_interval_min = 30
config.num_images = 10
train_dataset, _, _ = load_data(config=config)
images = _first_image(train_dataset)
image_names = []
for i, image in enumerate(images[0]):
plt.cla()
plt.clf()
plt.axis("off")
plt.tight_layout()
image2d = image[:, :, 0]
plt.imshow(image2d, cmap="gray")
name = f"assets/{i}.png"
plt.savefig(name)
image_names.append(name)
make_gif(f"assets/image-3d-{channel}.gif", image_names)
def setUp(self):
config = default_config()
config.error_strategy = dataloader.ErrorStrategy.ignore
config.features = [dataloader.Feature.target_ghi]
metadata_loader = metadata.MetadataLoader(file_name=CATALOG_PATH)
self.dataset = dataloader.create_dataset(
lambda: metadata_loader.load(A_STATION, A_STATION_COORDINATE),
config=config)
def config(self) -> dataloader.DataloaderConfig:
"""Configuration."""
config = default_config()
config.features = [
dataloader.Feature.metadata,
dataloader.Feature.target_ghi,
]
return config
def config(self) -> dataloader.DataloaderConfig:
"""Configuration."""
config = default_config()
config.num_images = 1
config.features = [
dataloader.Feature.image, dataloader.Feature.target_ghi
]
return config
def config(self) -> dataloader.DataloaderConfig:
"""Configuration."""
config = default_config()
config.num_images = self.num_images
config.time_interval_min = self.time_interval_min
config.skip_missing_past_images = True
config.features = [dataloader.Feature.image]
return config
def __init__(self):
loader = MetadataLoader(CATALOG_PATH)
config = default_config()
config.error_strategy = dataloader.ErrorStrategy.ignore
config.features = [dataloader.Feature.target_ghi]
self.dataset = dataloader.create_dataset(
lambda: loader.load(STATION, COORDINATES, skip_missing=False),
config=config)
def config(self) -> dataloader.DataloaderConfig:
"""Configuration."""
config = default_config()
config.num_images = self.num_images
config.ratio = 0.1
config.time_interval_min = 30
config.features = [
dataloader.Feature.image, dataloader.Feature.target_ghi
]
return config
def config(self) -> dataloader.DataloaderConfig:
"""Configuration."""
config = default_config()
config.num_images = self.language_model.num_images
config.time_interval_min = self.language_model.time_interval_min
config.features = [
dataloader.Feature.image,
dataloader.Feature.metadata,
dataloader.Feature.target_ghi,
]
return config
def __init__(self, num_images):
self.num_images = num_images
config = default_config()
config.features = [dataloader.Feature.image]
def gen():
for _ in range(num_images):
yield Metadata(IMAGE_PATH, "8bits", 10, datetime.now(),
COORDINATES)
self.dataset = dataloader.create_dataset(lambda: gen(), config=config)
def config(self, dry_run=False) -> dataloader.DataloaderConfig:
"""Configuration."""
config = default_config()
config.num_images = 1
config.ratio = 1
config.features = [
dataloader.Feature.target_ghi,
dataloader.Feature.metadata,
dataloader.Feature.image,
dataloader.Feature.target_ghi,
]
if dry_run:
config.error_strategy = dataloader.ErrorStrategy.stop
return config
def __init__(self, dropout=0.3, crop_size=32):
"""Initialize the autoencoder."""
super().__init__(NAME_AUTOENCODER)
self.scaling_image = preprocessing.MinMaxScaling(
preprocessing.IMAGE_MIN, preprocessing.IMAGE_MAX)
self.crop_size = crop_size
self.default_config = default_config()
self.default_config.num_images = 1
self.default_config.features = [dataloader.Feature.image]
num_channels = len(self.default_config.channels)
self.encoder = Encoder(dropout=dropout)
self.decoder = Decoder(num_channels, dropout=dropout)
def find_target_ghi_minmax_value(dataset=None):
"""Find the minimum value of target ghi.
The values are found based on the training dataset.
Return:
Tuple with (max_value, min_value)
"""
if dataset is None:
config = default_config()
config.features = [dataloader.Feature.target_ghi]
dataset, _, _ = load_data(config=config)
max_value = dataset.reduce(0.0, _reduce_max)
min_value = dataset.reduce(max_value, _reduce_min)
return max_value, min_value
def test_iterate_multiple_times(self):
config = train.default_config()
config.error_strategy = dataloader.ErrorStrategy.ignore
config.features = [dataloader.Feature.target_ghi]
metadata_loader = metadata.MetadataLoader(file_name=CATALOG_PATH)
dataset = dataloader.create_dataset(
lambda: metadata_loader.load(A_STATION, A_STATION_COORDINATE),
config=config)
first_run_data = 0
for data in dataset:
first_run_data += 1
second_run_data = 0
for data in dataset:
second_run_data += 1
self.assertEqual(first_run_data, second_run_data)
def cache(size, cache_dir):
config = default_config()
config.features = [Feature.image]
config.crop_size = size
# Create image cache dir
config.image_cache_dir = cache_dir + f"/image_cache_{size}"
config.image_cache_dir = config.image_cache_dir.replace("(", "")
config.image_cache_dir = config.image_cache_dir.replace(",", "")
config.image_cache_dir = config.image_cache_dir.replace(")", "")
config.image_cache_dir = config.image_cache_dir.replace(" ", "-")
logger.info(
f"Caching images with size {size} in dir {config.image_cache_dir}")
dataset_train, dataset_valid, dataset_test = load_data(
enable_tf_caching=False, config=config)
_create_cache("train", dataset_train)
_create_cache("valid", dataset_valid)
_create_cache("test", dataset_test)
os.system(f"tar -cf {config.image_cache_dir}.tar {config.image_cache_dir}")