def plot(self, channels=None, figure=None, axes=None,
subplot=None, title=None, bottom=-0.2,
# Our own options
start_time=None, end_time=None, time_units=None, plot_func=plt.bar,
**kwargs):
if plot_func == plt.bar:
data2 = copy.copy(self.data)
# Make zero values visible
data2 -= bottom
kwargs['bottom'] = bottom
self = copy.copy(self)
self.data = data2
if channels is None:
self.data = np.array([np.max(self.data, axis=0)])
self.channels = np.array([','.join(self.channels)])
channels = self.channels
else:
ci = self.get_channel_index(channels)
self.data = np.array([np.max(self.data[ci], axis=0)])
self.channels = ','.join(self.channels[ci])
if time_units is None:
time_units = dt64.get_units(self.sample_start_time)
if 'width' not in kwargs:
kwargs['width'] = dt64.dt64_to(self.nominal_cadence,
time_units) * 1.0
if 'align' not in kwargs:
kwargs['align'] = 'center'
if 'color' not in kwargs:
kwargs['color'] = self.get_color()
if 'edgecolor' not in kwargs:
kwargs['edgecolor'] = np.tile(0.6, (1,3))
if 'linewidth' not in kwargs:
kwargs['linewidth'] = 1
else:
bottom = 0
r = Data.plot(self, channels=channels, figure=figure, axes=axes,
subplot=subplot, units_prefix='',
title=title,
start_time=start_time, end_time=end_time,
time_units=time_units, plot_func=plot_func, **kwargs)
ax = plt.gca()
if plot_func == plt.bar:
ax.set_ylabel('Local K index (' + ','.join(channels) + ')')
ax.set_ylim(bottom, 9)
ax.yaxis.set_major_locator(mpl.ticker.MultipleLocator(1))
return r
def load_temperature_data(file_name, archive_data,
project, site, data_type, channels, start_time,
end_time, **kwargs):
awn_channels = np.array(['Sensor temperature', # Mag sensor
'System temperature'])
extra_channels = np.array(['Detector temperature',
'Sky temperature',
'Ambient temperature'])
chan_array = np.array(channels)
cidx_a = []
cidx_b = []
for n in range(len(channels)):
if channels[n] in awn_channels:
cidx_a.append(n)
elif channels[n] in extra_channels:
cidx_b.append(n)
assert len(cidx_a) + len(cidx_b) == chan_array.size
if len(cidx_a):
awn_file_name = file_name.replace('_cloud.txt', '.txt')
a = load_cloud_data(awn_file_name,
archive_data,
project, site, data_type,
chan_array[cidx_a],
start_time, end_time,
data_cols=[5], **kwargs)
else:
a = None
if len(cidx_b):
b = load_cloud_data(file_name, archive_data,
project, site, data_type,
chan_array[cidx_b],
start_time, end_time,
data_cols=[1,2,3], **kwargs)
if a is None:
return b
else:
# Join a and b
# assert 1 or a.start_time == b.start_time and \
# a.end_time == b.end_time and \
# np.all(a.sample_start_time == b.sample_end_time) and \
# np.all(a.sample_end_time == b.sample_end_time) and \
# a.nominal_cadence == b.nominal_cadence and \
# a.units == b.units
r = copy.deepcopy(a)
cidx_a.extend(cidx_b)
r.channels = chan_array[cidx_a]
# Bug in data recording process. Work around for now
assert a.sample_start_time.size == b.sample_end_time.size, \
'Different length data sets'
# assert np.all(np.abs(a.sample_start_time - b.sample_start_time) \
# <= np.timedelta64(1, 's'))
# assert np.all(np.abs(a.sample_end_time - b.sample_end_time) \
# <= np.timedelta64(1, 's'))
# b.sample_start_time = a.sample_start_time
# b.sample_end_time = a.sample_end_time
# # -- end of hack --
# # Find common sample times
# common_sample_start_time = np.intersect1d(a.sample_start_time,
# b.sample_start_time)
# # Find where they are located in a and b
# s_a_idx = np.nonzero(np.in1d(a.sample_start_time,
# common_sample_start_time))[0]
# s_b_idx = np.nonzero(np.in1d(a.sample_start_time,
# common_sample_start_time))[0]
# assert np.all(a.sample_end_time[s_a_idx] \
# == b.sample_end_time[s_b_idx]), \
# 'Sample end times do not match'
# ns = common_sample_start_time.size
ns = a.sample_start_time.size
r.data = np.zeros([a.channels.size + b.channels.size, ns])
r_a_cidx = range(a.channels.size)
r_b_cidx = range(a.channels.size, r.channels.size)
integ_units = dt64.smallest_unit([dt64.get_units(a.integration_interval),
dt64.get_units(b.integration_interval)])
r.data[r_a_cidx] = a.data
r.data[r_b_cidx] = b.data
r.integration_interval = \
np.zeros([r.channels.size, ns]).astype('m8[' + integ_units +
']')
r.integration_interval[r_a_cidx] = \
dt64.dt64_to(a.integration_interval, integ_units)
r.integration_interval[r_b_cidx] = \
dt64.dt64_to(b.integration_interval, integ_units)
return r
else:
return a
def load_data(project,
site,
data_type,
start_time,
end_time,
archive=None,
channels=None,
path=None,
load_function=None,
raise_all=False,
cadence=None,
aggregate=None,
filter_function=None,
use_cache=None,
now=None):
'''Load data.
project: name of the project (upper case)
site: site abbreviation (upper case)
data_type: class name of the data type to be loaded
start_time: start time (inclusive) of the data set
end_time: end time (exclusive) of the data set
The following optional parameters are recognised:
archive: name of the archive. Required if more than one archive is
present and there is not an archive called "default".
channels: data channel(s) to load. All are loaded if not specified
path: URL or file path, specified as a strftime format specifier.
Alternatively can be a function reference which is passed the
time and returns the filename. If given this overrides the
standard load path.
load_function: Pass responsibility for loading the data to the given
function reference, after validating the input parameters.
'''
archive, ad = get_archive_info(project, site, data_type,
archive=archive)
cad_units = dt64.get_units(ad['nominal_cadence'])
start_time = start_time.astype('datetime64[%s]' % cad_units)
end_time = end_time.astype('datetime64[%s]' % cad_units)
if channels is None:
channels = ad['channels']
else:
# Could be as single channel name or a list of channels
if isinstance(channels, six.string_types):
if channels not in ad['channels']:
raise Exception('Unknown channel')
else:
for c in channels:
if c not in ad['channels']:
raise Exception('Unknown channel')
if path is None:
path = ad['path']
if load_function is None:
load_function = ad.get('load_function')
if filter_function is None:
filter_function = ad.get('filter_function')
if load_function:
# Pass responsibility for loading to some other
# function. Parameters have already been checked.
return load_function(project,
site,
data_type,
start_time,
end_time,
archive=archive,
channels=channels,
path=path,
raise_all=raise_all,
cadence=cadence,
aggregate=aggregate,
filter_function=filter_function)
data = []
for t in dt64.dt64_range(dt64.floor(start_time, ad['duration']),
end_time,
ad['duration']):
# A local copy of the file to be loaded, possibly an
# uncompressed version.
temp_file_name = None
t2 = t + ad['duration']
if hasattr(path, '__call__'):
# Function: call it with relevant information to get the path
file_name = path(t, project=project, site=site,
data_type=data_type, archive=archive,
channels=channels)
else:
file_name = dt64.strftime(t, path)
url_parts = urlparse(file_name)
if url_parts.scheme in ('ftp', 'http', 'https'):
if ad.get('cache_dir'):
if now is None:
now = np.datetime64('now', 's')
dtd = ad.get('data_transfer_delay', np.timedelta64(0, 's'))
if use_cache is None:
if t2 + dtd < now:
uc = True # OK to try fetching from the cache
else:
uc = False
logger.debug('data too new to cache')
else:
uc = use_cache
cache_filename = os.path.normpath(os.path.join(ad['cache_dir'],
file_name.replace(':', '/')))
logger.debug('cache file: ' + cache_filename)
if uc:
if os.path.exists(cache_filename):
file_name = cache_filename
logger.debug('cache hit')
else:
file_name = download_url(file_name, dest=cache_filename)
else:
# Download but discard after use
file_name = download_url(file_name)
temp_file_name = file_name
else:
# No cache so discard after use
file_name = download_url(file_name)
temp_file_name = file_name
if file_name is None:
continue
elif url_parts.scheme == 'file':
file_name = url_parts.path
if not os.path.exists(file_name):
logger.info('missing file %s', file_name)
continue
# Now only need to access local files
if os.path.splitext(url_parts.path)[1] in ('.gz', '.dgz'):
# Transparently uncompress
gunzipped_file = None
try:
logger.debug('unzipping %s', file_name)
gunzipped_file = NamedTemporaryFile(prefix=__name__,
delete=False)
with gzip.open(file_name, 'rb') as gzip_file:
shutil.copyfileobj(gzip_file, gunzipped_file)
gunzipped_file.close()
except KeyboardInterrupt:
raise
except Exception as e:
if gunzipped_file:
gunzipped_file.close()
os.unlink(gunzipped_file.name)
gunzipped_file = None
continue
finally:
if temp_file_name:
logger.debug('deleting temporary file ' + temp_file_name)
os.unlink(temp_file_name)
temp_file_name = gunzipped_file.name
file_name = temp_file_name
logger.info('loading ' + file_name)
try:
tmp = ad['load_converter'](file_name,
ad,
project=project,
site=site,
data_type=data_type,
start_time=t,
end_time=t2,
channels=channels,
archive=archive,
path=path,
raise_all=raise_all)
if tmp is not None:
if cadence is not None and cadence <= ad['duration']:
tmp.set_cadence(cadence,
aggregate=aggregate,
inplace=True)
data.append(tmp)
except KeyboardInterrupt:
raise
except Exception as e:
if raise_all:
raise
logger.info('Could not load ' + file_name)
logger.debug(str(e))
logger.debug(traceback.format_exc())
finally:
if temp_file_name:
logger.debug('deleting temporary file ' + temp_file_name)
os.unlink(temp_file_name)
if len(data) == 0:
return None
r = concatenate(data, sort=False)
r.extract(inplace=True,
start_time=start_time,
end_time=end_time,
channels=channels)
if cadence is not None and cadence > ad['duration']:
# cadence too large to apply on results of loading each file,
# apply to combined object
r.set_cadence(cadence,
aggregate=aggregate,
inplace=True)
if filter_function:
logger.debug('filtering with function %s', filter_function.__name__)
r = filter_function(r)
return r
def __init__(self,
project=None,
site=None,
channels=None,
start_time=None,
end_time=None,
sample_start_time=np.array([]),
sample_end_time=np.array([]),
integration_interval=np.array([]),
nominal_cadence=None,
data=np.array([]),
units=None,
sort=False,
magdata=None,
magqdc=None,
thresholds=None,
colors=None,
fit=None,
fit_params={},
input_channels=None,
range_=False):
Data.__init__(self,
project=project,
site=site,
channels=channels,
start_time=start_time,
end_time=end_time,
sample_start_time=sample_start_time,
sample_end_time=sample_end_time,
integration_interval=integration_interval,
nominal_cadence=nominal_cadence,
data=data,
units=units,
sort=sort)
if magdata is not None and magqdc is not None:
assert magdata.units == magqdc.units, 'Units must match'
cadence = np.timedelta64(60, 'm')
cad_units = dt64.get_units(cadence)
if isinstance(magqdc, ap.magdata.MagQDC):
aligned = magqdc.align(magdata, fit=fit, **fit_params)
else:
aligned = magqdc
if input_channels is None:
input_channels = []
if 'H' in magdata.channels:
input_channels.append('H')
elif 'X' in magdata.channels:
input_channels.append('X')
if 'D' in magdata.channels:
input_channels.append('D')
elif 'E' in magdata.channels:
input_channels.append('E') # IAGA notation for D when in nT
elif 'Y' in magdata.channels:
input_channels.append('Y')
input_channels = np.array(input_channels).flatten()
cidx = magdata.get_channel_index(input_channels)
if magdata.nominal_cadence <= np.timedelta64(5, 's'):
# Throw away ~30 seconds
n = int(np.timedelta64(30, 's') / magdata.nominal_cadence)
else:
# Throw away up to 2.5 minutes
n = int(np.timedelta64(150, 's') / magdata.nominal_cadence)
nth_largest = ap.tools.NthLargest(n)
disturbance = magdata.extract(channels=input_channels)
disturbance.data = np.abs(magdata.data[cidx] -
aligned.data[cidx])
disturbance.set_cadence(cadence,
inplace= True,
aggregate=nth_largest)
data = disturbance.data.copy()
if range_:
range_high = magdata.extract(channels=input_channels)
range_high.set_cadence(cadence,
inplace= True,
aggregate=nth_largest)
nth_smallest = ap.tools.NthLargest(n, smallest=True)
range_low = magdata.extract(channels=input_channels)
range_low.set_cadence(cadence,
inplace= True,
aggregate=nth_smallest)
hourly_range = copy.deepcopy(range_high)
hourly_range.data = range_high.data - range_low.data
assert np.all(hourly_range.sample_start_time ==
disturbance.sample_start_time), \
'sample start times differ'
data = np.vstack([data, hourly_range.data])
self.project = magdata.project
self.site = magdata.site
if range_:
self.channels = np.array(['Activity'])
else:
self.channels = np.array(['Disturbance'])
self.start_time = disturbance.start_time.astype('datetime64[%s]' % cad_units)
self.end_time = disturbance.end_time.astype(self.start_time.dtype)
self.sample_start_time = disturbance.sample_start_time
self.sample_end_time = disturbance.sample_end_time
self.integration_interval = None
self.nominal_cadence = cadence
# Take the maximum value for all
self.data = np.max(data, axis=0).reshape([1, data.shape[1]])
self.units = magdata.units
if sort:
self.sort(inplace=True)
if self.nominal_cadence <= np.timedelta64(5, 's'):
# Throw away ~30 seconds
n = int(np.timedelta64(30, 's') / self.nominal_cadence)
# elif self.nominal_cadence < np.timedelta64(2, 'm'):
else:
# Throw away up to 2.5 minutes
n = int(np.timedelta64(150, 's') / self.nominal_cadence)
nth_largest = ap.tools.NthLargest(n)
self.set_cadence(np.timedelta64(60, 'm'),
inplace= True, aggregate=nth_largest)
if thresholds is None:
self.thresholds = self.get_site_info('activity_thresholds')
else:
self.thresholds = thresholds
if colors is None:
self.colors = self.get_site_info('activity_colors')
else:
self.colors = colors
def plot(self, channels=None, figure=None, axes=None,
subplot=None, units_prefix=None, title=None,
# Our own options
start_time=None, end_time=None, time_units=None,
plot_func=plt.bar, plot_thresholds=True,
**kwargs):
if units_prefix is None and self.units == 'T':
# Use nT for plotting
units_prefix = 'n'
if plot_func == plt.bar:
if time_units is None:
time_units = dt64.get_units(self.sample_start_time)
if 'width' not in kwargs:
kwargs['width'] = dt64.dt64_to(self.nominal_cadence, time_units)
if 'align' not in kwargs:
kwargs['align'] = 'center'
if 'color' not in kwargs:
kwargs['color'] = self.get_color()
if 'edgecolor' not in kwargs:
kwargs['edgecolor'] = np.tile(0.6, (1,3))
if 'linewidth' not in kwargs:
kwargs['linewidth'] = 0.5
if 'zorder' not in kwargs:
kwargs['zorder'] = 2
r = Data.plot(self, channels=channels, figure=figure, axes=axes,
subplot=subplot, units_prefix=units_prefix,
title=title,
start_time=start_time, end_time=end_time,
time_units=time_units, plot_func=plot_func, **kwargs)
if plot_thresholds:
if axes is None:
axes = plt.gca()
mul = ap.str_units(0, 'T', units_prefix, wantstr=False)['mul']
tmp = self.extract(start_time, end_time, channels=channels)
ymax = np.nan
ylim = axes.get_ylim()
if np.size(tmp.data):
ymax = np.nanmax(tmp.data) / mul
if np.isnan(ymax):
ymax = axes.get_ylim()[1]
new_ylim_top = ylim[1]
for n in range(len(self.thresholds)-1, 0, -1):
y = self.thresholds[n] / mul
if y >= ymax:
new_ylim_top = y*1.05
axes.plot(axes.xaxis.get_view_interval(),
self.thresholds[[n,n]] / mul,
color=self.colors[n],
linestyle='-',
linewidth=2,
zorder=1)
axes.set_ylim(top=new_ylim_top)
return r
def load_data(project,
site,
data_type,
start_time,
end_time,
archive=None,
channels=None,
path=None,
load_function=None,
raise_all=False,
cadence=None,
aggregate=None,
filter_function=None,
use_cache=None,
now=None):
'''Load data.
project: name of the project (upper case)
site: site abbreviation (upper case)
data_type: class name of the data type to be loaded
start_time: start time (inclusive) of the data set
end_time: end time (exclusive) of the data set
The following optional parameters are recognised:
archive: name of the archive. Required if more than one archive is
present and there is not an archive called "default".
channels: data channel(s) to load. All are loaded if not specified
path: URL or file path, specified as a strftime format specifier.
Alternatively can be a function reference which is passed the
time and returns the filename. If given this overrides the
standard load path.
load_function: Pass responsibility for loading the data to the given
function reference, after validating the input parameters.
'''
archive, ad = get_archive_info(project, site, data_type, archive=archive)
cad_units = dt64.get_units(ad['nominal_cadence'])
start_time = start_time.astype('datetime64[%s]' % cad_units)
end_time = end_time.astype('datetime64[%s]' % cad_units)
if channels is None:
channels = ad['channels']
else:
# Could be as single channel name or a list of channels
if isinstance(channels, six.string_types):
if channels not in ad['channels']:
raise Exception('Unknown channel')
else:
for c in channels:
if c not in ad['channels']:
raise Exception('Unknown channel')
if path is None:
path = ad['path']
if load_function is None:
load_function = ad.get('load_function')
if filter_function is None:
filter_function = ad.get('filter_function')
if load_function:
# Pass responsibility for loading to some other
# function. Parameters have already been checked.
return load_function(project,
site,
data_type,
start_time,
end_time,
archive=archive,
channels=channels,
path=path,
raise_all=raise_all,
cadence=cadence,
aggregate=aggregate,
filter_function=filter_function)
data = []
for t in dt64.dt64_range(dt64.floor(start_time, ad['duration']), end_time,
ad['duration']):
# A local copy of the file to be loaded, possibly an
# uncompressed version.
temp_file_name = None
t2 = t + ad['duration']
if hasattr(path, '__call__'):
# Function: call it with relevant information to get the path
file_name = path(t,
project=project,
site=site,
data_type=data_type,
archive=archive,
channels=channels)
else:
file_name = dt64.strftime(t, path)
url_parts = urlparse(file_name)
if url_parts.scheme in ('ftp', 'http', 'https'):
if ad.get('cache_dir'):
if now is None:
now = np.datetime64('now', 's')
dtd = ad.get('data_transfer_delay', np.timedelta64(0, 's'))
if use_cache is None:
if t2 + dtd < now:
uc = True # OK to try fetching from the cache
else:
uc = False
logger.debug('data too new to cache')
else:
uc = use_cache
cache_filename = os.path.normpath(
os.path.join(ad['cache_dir'], file_name.replace(':', '/')))
logger.debug('cache file: ' + cache_filename)
if uc:
if os.path.exists(cache_filename):
file_name = cache_filename
logger.debug('cache hit')
else:
file_name = download_url(file_name,
dest=cache_filename)
else:
# Download but discard after use
file_name = download_url(file_name)
temp_file_name = file_name
else:
# No cache so discard after use
file_name = download_url(file_name)
temp_file_name = file_name
if file_name is None:
continue
elif url_parts.scheme == 'file':
file_name = url_parts.path
if not os.path.exists(file_name):
logger.info('missing file %s', file_name)
continue
# Now only need to access local files
if os.path.splitext(url_parts.path)[1] in ('.gz', '.dgz'):
# Transparently uncompress
gunzipped_file = None
try:
logger.debug('unzipping %s', file_name)
gunzipped_file = NamedTemporaryFile(prefix=__name__,
delete=False)
with gzip.open(file_name, 'rb') as gzip_file:
shutil.copyfileobj(gzip_file, gunzipped_file)
gunzipped_file.close()
except KeyboardInterrupt:
raise
except Exception as e:
if gunzipped_file:
gunzipped_file.close()
os.unlink(gunzipped_file.name)
gunzipped_file = None
continue
finally:
if temp_file_name:
logger.debug('deleting temporary file ' + temp_file_name)
os.unlink(temp_file_name)
temp_file_name = gunzipped_file.name
file_name = temp_file_name
logger.info('loading ' + file_name)
try:
tmp = ad['load_converter'](file_name,
ad,
project=project,
site=site,
data_type=data_type,
start_time=t,
end_time=t2,
channels=channels,
archive=archive,
path=path,
raise_all=raise_all)
if tmp is not None:
if cadence is not None and cadence <= ad['duration']:
tmp.set_cadence(cadence, aggregate=aggregate, inplace=True)
data.append(tmp)
except KeyboardInterrupt:
raise
except Exception as e:
if raise_all:
raise
logger.info('Could not load ' + file_name)
logger.debug(str(e))
logger.debug(traceback.format_exc())
finally:
if temp_file_name:
logger.debug('deleting temporary file ' + temp_file_name)
os.unlink(temp_file_name)
if len(data) == 0:
return None
r = concatenate(data, sort=False)
r.extract(inplace=True,
start_time=start_time,
end_time=end_time,
channels=channels)
if cadence is not None and cadence > ad['duration']:
# cadence too large to apply on results of loading each file,
# apply to combined object
r.set_cadence(cadence, aggregate=aggregate, inplace=True)
if filter_function:
logger.debug('filtering with function %s', filter_function.__name__)
r = filter_function(r)
return r
def plot(
self,
channels=None,
figure=None,
axes=None,
subplot=None,
title=None,
bottom=-0.2,
# Our own options
start_time=None,
end_time=None,
time_units=None,
plot_func=plt.bar,
**kwargs):
if plot_func == plt.bar:
data2 = copy.copy(self.data)
# Make zero values visible
data2 -= bottom
kwargs['bottom'] = bottom
self = copy.copy(self)
self.data = data2
if channels is None:
self.data = np.array([np.max(self.data, axis=0)])
self.channels = np.array([','.join(self.channels)])
channels = self.channels
else:
ci = self.get_channel_index(channels)
self.data = np.array([np.max(self.data[ci], axis=0)])
self.channels = ','.join(self.channels[ci])
if time_units is None:
time_units = dt64.get_units(self.sample_start_time)
if 'width' not in kwargs:
kwargs['width'] = dt64.dt64_to(self.nominal_cadence,
time_units) * 1.0
if 'align' not in kwargs:
kwargs['align'] = 'center'
if 'color' not in kwargs:
kwargs['color'] = self.get_color()
if 'edgecolor' not in kwargs:
kwargs['edgecolor'] = np.tile(0.6, (1, 3))
if 'linewidth' not in kwargs:
kwargs['linewidth'] = 1
else:
bottom = 0
r = Data.plot(self,
channels=channels,
figure=figure,
axes=axes,
subplot=subplot,
units_prefix='',
title=title,
start_time=start_time,
end_time=end_time,
time_units=time_units,
plot_func=plot_func,
**kwargs)
ax = plt.gca()
if plot_func == plt.bar:
ax.set_ylabel('Local K index (' + ','.join(channels) + ')')
ax.set_ylim(bottom, 9)
ax.yaxis.set_major_locator(mpl.ticker.MultipleLocator(1))
return r
def __init__(self,
project=None,
site=None,
channels=None,
start_time=None,
end_time=None,
sample_start_time=np.array([]),
sample_end_time=np.array([]),
integration_interval=np.array([]),
nominal_cadence=None,
data=np.array([]),
units=None,
sort=False,
magdata=None,
magqdc=None,
thresholds=None,
colors=None,
fit=None,
fit_params={},
input_channels=None,
range_=False):
Data.__init__(self,
project=project,
site=site,
channels=channels,
start_time=start_time,
end_time=end_time,
sample_start_time=sample_start_time,
sample_end_time=sample_end_time,
integration_interval=integration_interval,
nominal_cadence=nominal_cadence,
data=data,
units=units,
sort=sort)
if magdata is not None and magqdc is not None:
assert magdata.units == magqdc.units, 'Units must match'
cadence = np.timedelta64(60, 'm')
cad_units = dt64.get_units(cadence)
if isinstance(magqdc, ap.magdata.MagQDC):
aligned = magqdc.align(magdata, fit=fit, **fit_params)
else:
aligned = magqdc
if input_channels is None:
input_channels = []
if 'H' in magdata.channels:
input_channels.append('H')
elif 'X' in magdata.channels:
input_channels.append('X')
if 'D' in magdata.channels:
input_channels.append('D')
elif 'E' in magdata.channels:
input_channels.append(
'E') # IAGA notation for D when in nT
elif 'Y' in magdata.channels:
input_channels.append('Y')
input_channels = np.array(input_channels).flatten()
cidx = magdata.get_channel_index(input_channels)
if magdata.nominal_cadence <= np.timedelta64(5, 's'):
# Throw away ~30 seconds
n = int(np.timedelta64(30, 's') / magdata.nominal_cadence)
else:
# Throw away up to 2.5 minutes
n = int(np.timedelta64(150, 's') / magdata.nominal_cadence)
nth_largest = ap.tools.NthLargest(n)
disturbance = magdata.extract(channels=input_channels)
disturbance.data = np.abs(magdata.data[cidx] - aligned.data[cidx])
disturbance.set_cadence(cadence,
inplace=True,
aggregate=nth_largest)
data = disturbance.data.copy()
if range_:
range_high = magdata.extract(channels=input_channels)
range_high.set_cadence(cadence,
inplace=True,
aggregate=nth_largest)
nth_smallest = ap.tools.NthLargest(n, smallest=True)
range_low = magdata.extract(channels=input_channels)
range_low.set_cadence(cadence,
inplace=True,
aggregate=nth_smallest)
hourly_range = copy.deepcopy(range_high)
hourly_range.data = range_high.data - range_low.data
assert np.all(hourly_range.sample_start_time ==
disturbance.sample_start_time), \
'sample start times differ'
data = np.vstack([data, hourly_range.data])
self.project = magdata.project
self.site = magdata.site
if range_:
self.channels = np.array(['Activity'])
else:
self.channels = np.array(['Disturbance'])
self.start_time = disturbance.start_time.astype('datetime64[%s]' %
cad_units)
self.end_time = disturbance.end_time.astype(self.start_time.dtype)
self.sample_start_time = disturbance.sample_start_time
self.sample_end_time = disturbance.sample_end_time
self.integration_interval = None
self.nominal_cadence = cadence
# Take the maximum value for all
self.data = np.max(data, axis=0).reshape([1, data.shape[1]])
self.units = magdata.units
if sort:
self.sort(inplace=True)
if self.nominal_cadence <= np.timedelta64(5, 's'):
# Throw away ~30 seconds
n = int(np.timedelta64(30, 's') / self.nominal_cadence)
# elif self.nominal_cadence < np.timedelta64(2, 'm'):
else:
# Throw away up to 2.5 minutes
n = int(np.timedelta64(150, 's') / self.nominal_cadence)
nth_largest = ap.tools.NthLargest(n)
self.set_cadence(np.timedelta64(60, 'm'),
inplace=True,
aggregate=nth_largest)
if thresholds is None:
self.thresholds = self.get_site_info('activity_thresholds')
else:
self.thresholds = thresholds
if colors is None:
self.colors = self.get_site_info('activity_colors')
else:
self.colors = colors
def plot(
self,
channels=None,
figure=None,
axes=None,
subplot=None,
units_prefix=None,
title=None,
# Our own options
start_time=None,
end_time=None,
time_units=None,
plot_func=plt.bar,
plot_thresholds=True,
**kwargs):
if units_prefix is None and self.units == 'T':
# Use nT for plotting
units_prefix = 'n'
if plot_func == plt.bar:
if time_units is None:
time_units = dt64.get_units(self.sample_start_time)
if 'width' not in kwargs:
kwargs['width'] = dt64.dt64_to(self.nominal_cadence,
time_units)
if 'align' not in kwargs:
kwargs['align'] = 'center'
if 'color' not in kwargs:
kwargs['color'] = self.get_color()
if 'edgecolor' not in kwargs:
kwargs['edgecolor'] = np.tile(0.6, (1, 3))
if 'linewidth' not in kwargs:
kwargs['linewidth'] = 0.5
if 'zorder' not in kwargs:
kwargs['zorder'] = 2
r = Data.plot(self,
channels=channels,
figure=figure,
axes=axes,
subplot=subplot,
units_prefix=units_prefix,
title=title,
start_time=start_time,
end_time=end_time,
time_units=time_units,
plot_func=plot_func,
**kwargs)
if plot_thresholds:
if axes is None:
axes = plt.gca()
mul = ap.str_units(0, 'T', units_prefix, wantstr=False)['mul']
tmp = self.extract(start_time, end_time, channels=channels)
ymax = np.nan
ylim = axes.get_ylim()
if np.size(tmp.data):
ymax = np.nanmax(tmp.data) / mul
if np.isnan(ymax):
ymax = axes.get_ylim()[1]
new_ylim_top = ylim[1]
for n in range(len(self.thresholds) - 1, 0, -1):
y = self.thresholds[n] / mul
if y >= ymax:
new_ylim_top = y * 1.05
axes.plot(axes.xaxis.get_view_interval(),
self.thresholds[[n, n]] / mul,
color=self.colors[n],
linestyle='-',
linewidth=2,
zorder=1)
axes.set_ylim(top=new_ylim_top)
return r