def calculate_confidence_interval(array: np.ndarray,
confidence_percent: float = 0.95):
logger.debug(
f'Calculating confidence interval with confidence percent {confidence_percent}'
)
array_mean = np.nanmean(array)
sd = np.sqrt(np.nansum(np.power(array - array_mean, 2)) / array.size - 1)
alpha = 1 - confidence_percent
interval = stats.t.ppf(1.0 - (alpha / 2.0),
array.size - 1) * (sd / np.sqrt(array.size))
return array_mean - interval, array_mean + interval
def _label_hist_rectangles(rectangles, text_offset: float = 0.3):
logger.debug(
f'Labeling histogram rectangles with text offset = {text_offset}')
for rect in rectangles:
height = rect.get_height()
if height > 0:
plot.text(rect.get_x() + rect.get_width() / 2.,
height + text_offset,
f'{height}',
ha='center',
va='bottom')
def parse_image_file_name(image_file_path: str):
logger.debug(f'Parsing file meta info @ "{image_file_path}"')
basename = os.path.splitext(os.path.basename(image_file_path))[0]
match = re.fullmatch(IMAGE_FILE_NAME_PATTERN, basename)
if match:
capture_date = datetime.datetime.strptime(match.group(1), "%d%m%Y").date()
field_id = int(match.group(2))
revision = int(match.group(3))
mysterious_date = datetime.datetime.strptime(match.group(4), "%Y%m%d").date()
satellite = match.group(5)
return field_id, revision, capture_date, mysterious_date, satellite
else:
return None
def _determine_date_locator(dates_range: Sequence[datetime]):
min_date, max_date = min(dates_range), max(dates_range)
diff = max_date - min_date
logger.debug(f'Selecting date locator for timeline diff = {diff}')
locator = dates.YearLocator()
locator_label = 'year'
if diff.days < 31 * 10:
locator_label = 'days'
locator = dates.DayLocator()
if diff.days < 365 * 10:
locator_label = 'month'
locator = dates.MonthLocator()
logger.debug(f'Selected {locator_label} date locator')
return locator
def plot_clouds_impact_for_a_period(time_stamps: Sequence[datetime],
non_clouded_counts: Sequence[int],
partially_clouded_counts: Sequence[int],
fully_clouded_counts: Sequence[int]):
"""Accepts date stamps array with cloud statistics for each date
and plots bars histogram with this data"""
axes, figure = plot.gca(), plot.gcf()
logger.debug('Configuring histogram axes')
_setup_axes(axes,
title='Clouded images statistics',
x_label='Time stamps',
y_label='Count of images')
logger.debug('Configuring date formatting')
dates_formatter = dates.DateFormatter('%d/%m/%Y')
dates_locator = _determine_date_locator(time_stamps)
axes.xaxis.set_major_locator(dates_locator)
axes.xaxis.set_major_formatter(dates_formatter)
colors = ['mediumslateblue', 'plum', 'thistle']
bars_height = [
non_clouded_counts, partially_clouded_counts, fully_clouded_counts
]
time_stamps = dates.date2num(time_stamps)
bar_width = 10
def create_bar(y_axis, color, n=0):
# Asserting that there would be only 3 bars
return _construct_bar(axes,
y_values=y_axis,
x_values=time_stamps + bar_width * n - bar_width,
bar_width=bar_width,
color=color,
should_label_bars=False)
logger.debug('Creating 3 bars')
bars = tuple(
create_bar(data, color, index)
for index, (data, color) in enumerate(zip(bars_height, colors)))
logger.debug('Update legend')
labels = ['Non clouded', 'Partially clouded', 'Fully clouded']
axes.legend(bars, labels)
logger.debug('Formatting figure')
axes.xaxis_date()
axes.autoscale(tight=True)
figure.autofmt_xdate()
def import_locally_stored_image(image_file_path: str):
supported_extensions = SupportedDrivers.formats_list()
try:
file_extension = get_file_extension(image_file_path).lower()
if file_extension in supported_extensions:
file_meta_info = parse_image_file_name(image_file_path)
if file_meta_info:
field_id, revision, capture_date, mysterious_date, satellite = file_meta_info
logger.debug(f'Importing image bit map for image at "{image_file_path}"')
bitmap = read_image_bitmap(image_file_path)
data_tuple = field_id, revision, bitmap, capture_date, satellite, mysterious_date
return image_file_path, data_tuple
else:
logger.info('Unknown extension "%s" file at "%s"', file_extension, image_file_path)
except Exception as error:
raise IPLError(f'Unable to import image at "{image_file_path}", reason : {error}')
return None
def write_image_bitmap(image_file_path: str,
array: np.ndarray,
selected_driver: str = 'GTiff'):
logger.debug(f'Writing image data to "{image_file_path}"')
width, height = array.shape
sharing_mode_on = selected_driver == 'GTiff'
if array.dtype is not np.uint8:
array = array.astype(np.uint8)
try:
if os.path.isfile(image_file_path):
os.remove(image_file_path)
with silence_logs():
with rast.open(image_file_path, mode='w', driver=selected_driver,
width=width, height=height, count=1, dtype=array.dtype,
sharing=sharing_mode_on) as image_file:
image_file.write(array, 1)
except rast.RasterioIOError as error:
raise IPLError(f'Unable to export image, reason : {error}')
def _construct_bar(ax,
y_values: Sequence[int],
x_values: Optional[Sequence[Any]] = None,
bar_width: float = 0.9,
should_label_bars: bool = True,
**kwargs):
if x_values is None:
x_values = range(len(y_values))
logger.debug(f'Plotting bar with width {bar_width}')
rectangles = ax.bar(x_values,
y_values,
align='center',
width=bar_width,
**kwargs)
if should_label_bars:
_label_hist_rectangles(rectangles)
return rectangles
def __init__(self, db_location: str = DATABASE_LOCATION):
logger.debug('Registering adapters')
sqlite3.register_adapter(np.ndarray, _adapt_array)
sqlite3.register_converter("np_array", _convert_array)
logger.debug(f'Connecting to database at "{db_location}"')
self.connection = sqlite3.connect(db_location,
detect_types=sqlite3.PARSE_DECLTYPES)
logger.debug('Connected successfully !')
self.cursor = self.connection.cursor()
def _setup_axes(ax, title: str, x_label: str, y_label: str):
logger.debug('Setting labels')
ax.set_title(title)
ax.set_xlabel(x_label)
ax.set_ylabel(y_label)
logger.debug('Configuring grid')
ax.grid(True, axis='y')
logger.debug('Making plot looking more prettier')
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['bottom'].set_visible(False)
ax.spines['left'].set_visible(False)
ax.xaxis.set_ticks_position('none')
ax.yaxis.set_ticks_position('none')
def plot_values_frequencies(unique_values_occurrences: Dict[IMAGE_DATA_TYPE,
int], **kwargs):
axes, figure = plot.gca(), plot.gcf()
occurrences = np.fromiter(unique_values_occurrences.values(),
dtype=np.int64)
logger.debug('Configuring axes')
_setup_axes(axes,
title='Frequencies histogram',
x_label='Values',
y_label='Occurrences')
logger.debug('Constructing occurrences bar')
_construct_bar(axes, occurrences, should_label_bars=False, **kwargs)
values_ = np.fromiter(unique_values_occurrences.keys(),
dtype=IMAGE_DATA_TYPE)
logger.debug('Configuring X-labels')
axes.set_xticks(range(len(values_)))
axes.set_xticklabels(values_, rotation=45)
def construct_values_occurrences_map(array: np.ndarray):
unique_values, counts = np.unique(array, return_counts=True)
logger.debug('Constructing value occurrences map in matrix')
return {value: count for value, count in zip(unique_values, counts)}
def calculate_clouds_percentile(array: np.ndarray):
logger.debug('Calculating clouds percentile')
boolean_mask = np.logical_or(array == 254, array == 255)
return np.count_nonzero(boolean_mask) / array.size
def show_plots():
logger.debug('Showing plots')
plot.show()
def plot_statistics_for_a_period(time_stamps: Sequence[datetime],
mean: Sequence[float],
lower_ci: Sequence[float],
upper_ci: Sequence[float],
legend_name: Optional[str] = None):
axes, figure = plot.gca(), plot.gcf()
logger.debug('Configuring axes')
_setup_axes(axes,
title='Time period CI statistics',
x_label='Time stamps',
y_label='Average')
logger.debug('Configuring date formatting')
dates_formatter = dates.DateFormatter('%d/%m/%Y')
dates_locator = _determine_date_locator(time_stamps)
axes.xaxis.set_major_locator(dates_locator)
axes.xaxis.set_major_formatter(dates_formatter)
logger.debug('Selecting color')
random_line_color = np.random.rand(3)
logger.debug('Plotting CI')
plot.fill_between(time_stamps,
lower_ci,
upper_ci,
color=random_line_color,
alpha=0.5)
logger.debug('Plotting mean line')
if legend_name:
plot.plot(time_stamps,
mean,
color=random_line_color,
label=legend_name)
axes.legend()
else:
plot.plot(time_stamps, mean, color=random_line_color)
logger.debug('Formatting figure')
axes.xaxis_date()
axes.autoscale(tight=True)
figure.autofmt_xdate()
def read_image_bitmap(image_file_path: str) -> np.ndarray:
logger.debug(f'Reading image at {image_file_path}, band = 1')
with rast.open(image_file_path) as raster:
return raster.read(1).astype(IMAGE_DATA_TYPE)