def save_starttime(self, tolast=False):
"""
If ``tolast`` is ``False``, loads the 'laststarttime' value from the
pklfile (FLAG `execution_time_file`) before saving the
``self.__timedict``. After this the 'laststarttime' should be
unchanged, and 'thisstarttime' should contain the starttime of this
exact program run.
If ``tolast`` is ``True``, the value of 'thisstarttime' is saved
to 'laststarttime'.
:type tolast: bool, optional
:param tolast: defaults to ``False``
"""
pklfile = self.flags['execution_time_file']
msg = self.__rlm
if pklfile is None:
msg += "FLAG \'execution_time_file\' undefined"
messenger(msg, "E")
return
if not os.path.isdir(os.path.dirname(pklfile)):
msg += "FLAG \'execution_time_file\' {}\n".format(pklfile)
msg += "Directory does not exist"
messenger(msg, "E")
return
if not os.path.isfile(pklfile) or tolast:
self._init_timedict()
fileutils.save_obj(pklfile, self.__timedict)
return
loaded_dict = fileutils.load_obj(pklfile)
if "laststarttime" in loaded_dict:
self.__timedict["laststarttime"] = loaded_dict["laststarttime"]
fileutils.save_obj(pklfile, self.__timedict)
def append_to_file(filename, text, header):
"""
If path defined in ``filename`` does not exist, new file is created and
``header`` is written in the file. If ``text`` is list, writes csv.
:type filename: str
:param filename:
:type text: str
:param text:
:type header: str
:param header:
"""
create_missing_folders(filename)
write_function = write_file
if isinstance(text, list):
if not isinstance(header, list):
header = [header]
write_function = functools.partial(write_csv, singlerow=True)
if not os.path.isfile(filename):
msg = "No previous file: {}".format(filename)
messenger(msg, "R")
write_function(filename, header, mode='w')
write_function(filename, text, mode='a')
def __init__(self):
"""
Sets the global program starttime, ``self.__now``.
Calls :meth:`~polyfemos.back.interpreter.Interpreter._init_timedict`
to initialize ``self.__timedict``.
Creates empty :class:`~polyfemos.back.station.Stations()` instance
(``self.station``).
"""
self._quit = False
self.__now = UTCDateTime(precision=0)
self.stations = Stations()
self.vars = {}
self.scopes = {"station": False}
self.__rlm = "" # reader location message
self.__current_conf_folder = ""
self._init_flags()
self.pool = None
self._init_timedict()
self._init_funcs_and_commands(self.__now)
messenger("", "B")
messenger("Program starttime: " + str(self.__now), "R")
def filepath(filepath, extension, force=False):
"""
Returns the filepath with the given extension if it exists.
If not, check if the 'retro' version is available and returns that.
If both are unavailable, returns an empty string.
:type filepath: str
:param filepath: original filepath without the extension
:type extension: str
:param extension: file extension, e.g. ``.stf``, ``.csv`` or ``.alert``
:type force: bool
:param force: If ``True``, returns filepath with the given extension and
if it does not exist, returns an empty string.
:rtype: str
:return:
"""
tempfilepath = filepath + extension
if os.path.isfile(tempfilepath):
return tempfilepath
msg = "File \'{}\' does not exist.".format(tempfilepath)
if not force:
tempfilepath = filepath + ".retro" + extension
if os.path.isfile(tempfilepath):
return tempfilepath
msg += "\nFile \'{}\' does not exist.".format(tempfilepath)
messenger(msg, "R")
return ""
def _read_stf_data(self, nez=False):
"""
Reads and parses the data from sohtextfiles.
:type nez: bool, optional
:param nez: defaults to ``False``. If selected sohpar is N, E or Z
offset, values for U, W and V offsets has to be read.
For information about UWV to NEZ conversion see
:class:`~polyfemos.front.sohplot.offsets.UWVOffsets`
"""
uwvo = offsets.UWVOffsets()
stored_data = []
runningdate = self.startdate
while (runningdate <= self.enddate):
filepath = self._get_stf_filepath(runningdate)
data_scope = False
runningdate += 86400
if not filepath:
continue
for row in fileutils.rowsof(filepath):
# Data values are read after 'DATA' text is encountered
if row[0] == "DATA":
data_scope = True
continue
if data_scope:
if len(row) < 3:
continue
stored_data.append(row)
for row in stored_data:
if nez and row[1] in uwvo.offsets:
uwvo.update(*row[:3])
# If U, W and V offset values are defined, conversion
# can be done
sensor = self.header["SENSOR"]
if uwvo and sensor is not None:
dt, value = \
uwvo.transform(sensor, self.sohpar_name[-1])
uwvo.clear()
else:
continue
elif row[1] == self.sohpar_name:
dt = to.utcdatetime(row[0])
if dt is None:
msg = "Could not convert '{}' to utcdatetime." \
.format(row[0])
messenger(msg, "W")
value = row[2]
else:
continue
dp = DataPoint(utcdatetime=dt, y=value)
if len(row) > 3:
dp.set_z(row[3])
self.data_container.append(dp)
def wrapper(path, *args, **kwargs):
quiet = False
if 'quiet' in kwargs:
quiet = kwargs['quiet']
messenger(path, "R", quiet=quiet)
if check_func(path):
return func_(path, *args, **kwargs)
msg = "{} \'{}\' DOES NOT EXIST".format(msgstr, path)
messenger(msg, "R", quiet=quiet)
return None
def print_warning(filename, msg):
"""
:type filename: str
:param filename: path to a file where the error was encountered
:type msg: str
:param msg: Message to be printed, if empty, nothing is printed.
"""
if msg:
msg = "In file: {}\n{}".format(filename, msg).strip()
messenger(msg, "W", showpid=False)
def index_alias(filename):
"""
Index site of the polyfemos web.
In addition provides the documentation files.
"""
msg = "Documentation filename: {}".format(filename)
messenger(msg, "R")
if not filename:
return render_template('index.htm')
folder = userdef.paths("doc_dir")
return send_from_directory(folder, filename)
def create_missing_folders(filename):
"""
Creates full path to given ``filename``.
:type filename: str
:param filename: file or folder
"""
folder = os.path.dirname(filename)
if not os.path.isdir(folder):
os.makedirs(folder)
msg = "Creating new folder: {}".format(folder)
messenger(msg, "R")
def data_timing_quality(path="",
average_calc_length=1,
scale=lambda x: x,
**kwargs):
"""
Reads timing quality values from miniseed file using
:func:`~obspy.io.mseed.util.get_record_information` function.
:type path: str
:param path: path to miniseed file
:type average_calc_length: int, optional
:param average_calc_length: defaults to one, setting this value greater
than one will result in a little bit smoothed timing quality curve,
since the returned datapoints will be averages over
``average_calc_length`` values.
:type scale: func, optional
:param scale: defaults to identity function,
scaling function applied to data values
:rtype: list or None
:return: list of lists containing timestamp (as
:class:`~obspy.core.utcdatetime.UTCDateTime` instance) and data value.
"""
imm = fileutils.invalid_mseed(path)
if imm:
messenger(imm, "M")
return None
timing_quality = []
tqtimes = []
last_qualities = []
offset = 0
# Codes from obspy
# Loop over each record.
# A valid record needs to have a record length of at
# least 256 bytes.
info = get_record_information(path)
while offset <= (info['filesize'] - 256):
this_info = get_record_information(path, offset)
if 'timing_quality' in this_info:
last_qualities.append(float(this_info['timing_quality']))
length = len(last_qualities)
ave = sum(last_qualities) / length
if length > average_calc_length:
last_qualities = last_qualities[1:]
timing_quality.append(ave)
tqtimes.append(this_info['starttime'])
offset += this_info['record_length']
return [[x, scale(y)] for x, y in zip(tqtimes, timing_quality)]
def pool_error_callback(arg, msg=""):
"""
A function to be called during pool worker error
:type arg:
:param arg: Error message
:type msg: str, optional
:param msg: Additional message
"""
msg += "while using multiprocessing\n"
msg += str(arg) + "\n"
messenger(msg, "E", showpid=False)
def define_flag(self, flag, value):
"""
:type flag: str
:param flag: name of the FLAG
:type value:
:param value: value of the FLAG, the type depends on the flag
"""
if flag not in self.__flags:
msg = self.__rlm
msg += "No FLAG \'{}\' available.".format(flag)
messenger(msg, "E")
return
self.flags[flag] = self.__flags[flag]["type"](value)
def main(conffile):
"""
:type conffile: str
:param conffile: Filepath of the conffile
"""
t0 = time.time()
interp = interpreter.Interpreter()
interp.readfile(conffile)
interp.stop()
t1 = time.time() - t0
messenger(str(t1), "N", showpid=False)
def wrapper(self, *args, **kwargs):
if self.track_datalen:
msg = "in method: {}".format(method.__name__)
messenger(msg, "R")
orig_len = len(self)
orig_nan_len = self.count_nans()
method(self, *args, **kwargs)
if self.track_datalen:
new_len = len(self)
new_nan_len = self.count_nans()
str_ = "{:*<19}**".format(method.__name__ + ":")
str_ += "dps:*{:*>7}*>*{:*<7}*".format(orig_len, new_len)
str_ += "nans:*{:*>7}*>*{:*<7}".format(orig_nan_len, new_nan_len)
self.add2info(str_)
def add_station(self, *args):
"""
Tries to define and add a station
(:class:`~polyfemos.back.station.Station`)
instance to ``self.stations``. ``args`` are passed straight to the
:meth:`~polyfemos.back.station.Station.__init__`.
"""
station = Station(*args)
success = self.stations.add_station(station)
if not success:
msg = self.__rlm
msg += "Unable to add station with id {}\n".format(station.id)
msg += "Start or endtimes overlap"
messenger(msg, "E")
return
self.scopes["station"] = True
def stop(self, quit_=True, save_starttime=True):
"""
:type quit\_: bool, optional
:param quit\_: defaults to ``True``, If ``True`` the program does
nothing anymore.
:type save_starttime: bool, optional
:param save_starttime: defaults to ``True``, In order to save
starttime value, ``save_starttime`` and FLAG 'save_starttime'
both are required to be ``True``.
"""
for k, v in self.__timedict.items():
msg = "{}: {}".format(k, v)
messenger(msg, "N")
self.close_and_join()
if save_starttime and self.flags["save_starttime"]:
self.save_starttime(tolast=True)
if quit_:
self._quit = True
def get_transform(from_epsg, to_epsg):
"""
:type from_epsg: str
:param from_epsg: EPSG code of the original coordinate system
:type to_epsg: str
:param to_epsg: EPSG code of the resulting coordinate system
:rtype: func
:return: A coordinate transformation function taking xy-coordinates
in ``from_epsg`` coordinate system and returning xy-coordinates in
``to_epsg`` coordinate system
"""
try:
from_coords = pyproj.Proj("+init=EPSG:{}".format(from_epsg))
to_coords = pyproj.Proj("+init=EPSG:{}".format(to_epsg))
except RuntimeError:
msg = "No coordinate conversion available for ESPGs from " \
"'{}' to '{}'.".format(from_epsg, to_epsg)
messenger(msg, "W")
return lambda x, y: (x, y)
return lambda x, y: pyproj.transform(from_coords, to_coords, x, y)
def write_file(filename, str_, mode='w', cmf=True):
r"""
:type filename: str
:param filename: filename of the file created
:type str\_: str
:param str\_: a text to written
:type mode: str, optional
:param mode: defaults to 'w' (write)
:type cmf: bool, optional
:param cmf: defaults to ``True``, If ``True`` missing folder
in ``filename`` aer created before writing
"""
if cmf:
create_missing_folders(filename)
if mode == "a" and not os.path.isfile(filename):
msg = "Trying to append to a non existing file: {}".format(filename)
messenger(msg, "W")
return
with open(filename, mode) as f:
f.write(str_)
def get_stream(path="", format_="MSEED"):
r"""
:type path: str
:param path: path to seismic data file
:type format\_: str, optional
:param format\_: defaults to ``MSEED``
:rtype: :class:`~obspy.core.stream.Stream` or None
:return: If invalid miniseed file(path) is provided,
empty stream is returned.
"""
msg = invalid_mseed(path)
if msg:
messenger(msg, "M")
return Stream()
else:
try:
st = read(path, format=format_)
return st
except InternalMSEEDError as e:
messenger(str(e), "W")
return Stream()
def _read_header(self, nez=False):
"""
Reads and parses the header information from the sohtextfile.
Header block ends when 'DATA' text is encountered.
:type nez: bool, optional
:param nez: defaults to ``False``, for UWV to NEZ conversion,
sensor information from the stf header is needed.
"""
for key, values in outfilefields.STF_HEADER.items():
# Fill the header with default values
self.header[key] = values[-1]
self.header["H_LINES"] = []
rowstart = self.sohpar_name if self.sohpar_name else "_"
if nez:
rowstart = rowstart[:-1]
filepath = self._get_stf_filepath(self.headerdate)
if not filepath:
msg = "Trying read the header of non existing 'stf' file."
messenger(msg, "W")
return
for row in fileutils.rowsof(filepath):
if len(row) >= 2:
key = row[0]
value = row[1]
if key.startswith(rowstart):
key = key.split("_")[-1]
if key in outfilefields.STF_HEADER:
self.header[key] = outfilefields.STF_HEADER[key][0](value)
if row[0] == "DATA":
break
h_lines = self.header["YELLOW"] \
+ self.header["ORANGE"] + self.header["RED"]
self.header["H_LINES"] = \
[line for line in h_lines if not to.isNaN(line)]
def read_file(fn, exclude_added_days=False):
if not fn:
return
if fn in redd_filepaths:
return
redd_filepaths.add(fn)
for row in fileutils.read_csv(fn)[1:]:
if len(row) < 2:
continue
dptimestamp = to.float_(row[0])
if dptimestamp is None:
msg = ""
messenger(msg, "W")
elif starttimestamp <= dptimestamp <= endtimestamp:
day = dptimestamp // 86400
if exclude_added_days and day in added_days:
continue
if not exclude_added_days:
added_days.add(day)
dp = DataPoint(timestamp=dptimestamp, y=row[1])
if len(row) > 2:
dp.set_z(row[2])
self.data_container.append(dp)
def transform_func(key):
"""
:type key: str
:param key: internal sensor code
:rtype: func
:return: UWV to NEZ transform function
"""
from math import sqrt
def func1(u, w, v):
# for streckeisen
x = -1. * u * sqrt(2. / 3.) + (v + w) / sqrt(6.)
y = (v - w) / sqrt(2.)
z = (u + v + w) / sqrt(3.)
return (x, y, z)
def func2(u, w, v):
# for nanometrics
x = (2. * u - v - w) / sqrt(6.)
y = (v - w) / sqrt(2.)
z = (u + v + w) / sqrt(3.)
return (x, y, z)
funcs = {
"STS-2": func1,
"Trillium_120PH": func2,
"Trillium_120PA": func2,
"Trillium_Co120": func2,
}
if key not in funcs:
msg = "Invalid key in transform_func"
messenger(msg, "W")
return lambda x, y, z: (x, y, z)
return funcs[key]
def get_summary(startdate="",
enddate="",
headerdate="",
combinations=[],
remove_irrationals=False,
advanced_outlier_removal=False,
fext="csv"):
"""
The function used to get the statistical summary of station/sohpar
combinations.
The execution is little bit slow because the combinations are
dealt separately, which means that, as all of the sohpars are in the same
file, the file is opened and closed multiple times during
the creation of the summary
:type startdate: str or :class:`~datetime.date`
:param startdate:
:type enddate: str or :class:`~datetime.date`
:param enddate:
:type headerdate: str or :class:`~datetime.date`
:param headerdate: The header information of this date is used in
calculations
:type combinations: list[tuple[str, str]]
:param combinations: list containing all unique station and sohpar
combinations
:type remove_irrationals: bool, optional
:param remove_irrationals: defaults to ``False``, If ``True``, irrational
values (limits defined in stf header) are removed from the data.
:type advanced_outlier_removal: bool, optional
:param advanced_outlier_removal: defaults to ``False``. If ``True``,
advanced outlier removal is used for specific station/sohpar
combination. The combinations and outlier removal are provided by
:func:`~polyfemos.front.userdef.summary_outlierremfuncs`, if the
YAML configuration is provided.
:type fext: str, optional
:param fext: defaults to "csv", select "stf" or "csv",
defines the datafile format which is read
:rtype: list, list
:return: rows and header, list of lists and a list.
"""
rows = []
header = []
max_loops = len(combinations)
for i, (station_id, sohpar_name) in enumerate(combinations):
sohplot = SOHPlot(station_id=station_id,
sohpar_name=sohpar_name,
startdate=startdate,
enddate=enddate,
headerdate=headerdate,
remove_irrationals=remove_irrationals,
advanced_outlier_removal=advanced_outlier_removal,
fext=fext)
dict_ = sohplot.get_statistics_dict()
if not header:
header = ["Station", "Sohpar"] + list(dict_.keys())
row = [station_id, sohpar_name]
# If every numerical value is nan, don't show the parameter
func_ = lambda x: isinstance(x, str) or math.isnan(x)
if all(map(func_, dict_.values())):
continue
rows.append(row + list(dict_.values()))
pr = 100.0 * float(i + 1) / max_loops
msg = "Summary table, percents ready: {:.2f}".format(pr)
messenger(msg, "R")
return rows, header
def sohmap():
"""
A map of the network area with stations. Alerts are sorted by their
priorities. The innermost circle consists of the alerts of the highest
priority.
"""
alertcolors = {
0: colors.ALERT_GREEN,
1: colors.ALERT_YELLOW,
2: colors.ALERT_RED,
}
filename = userdef.paths("map_file")
filename, extension = os.path.splitext(filename)
imgfile = "web_static/{}{}".format(filename, extension)
mapfile = "web_static/{}{}".format(filename, ".map")
msg = "Img and map files: {}, {}".format(imgfile, mapfile)
messenger(msg, "R")
if not os.path.isfile(imgfile):
return render_base(render_template)('sohmap.htm')
if not os.path.isfile(mapfile):
return render_base(render_template)('sohmap.htm')
# get transformation from WGS84 to pixels
pixel_transform = coordinator.transform_from_ozi_map(mapfile)
# open background map
pil_image = Image.open(imgfile)
draw = ImageDraw.Draw(pil_image)
today = UTCDateTime().now()
julday, year = get_jY(today)
sohpar_names = userdef.sohpars()
station_ids = userdef.station_ids()
# get alerts and priorities with each station and parameter combination
fpf = userdef.filepathformats("alert")
sohdict = get_sohdict(station_ids, year, julday, fpf)
for station_id in station_ids:
sohplot = SOHPlot(
station_id=station_id,
headerdate=today,
)
epsg = sohplot.header["EPSG"]
locx = sohplot.header["LOCX"]
locy = sohplot.header["LOCY"]
if epsg is None:
continue
# Convert stations coordinates into WGS84
transform = coordinator.get_transform(epsg, "4326")
px, py = pixel_transform(*transform(locx, locy))
def get_bbox(radius):
return (px - radius, py - radius, px + radius, py + radius)
# Alerts are stored in 3x3 matrix
# Priority in x axis
# Alert (red, yellow or green) in y axis
# Red alert with the highest priority is stored in [0,2]
alertcount = np.zeros((3, 3))
for sohpar_name in sohpar_names:
key = station_id + sohpar_name
if key not in sohdict['alerts']:
continue
alert = to.int_(sohdict['alerts'][key])
priority = to.int_(sohdict['priorities'][key])
if alert is None or priority is None or priority > 4:
continue
priority = min(3, priority) - 1
alertcount[alert, priority] += 1
# TODO clean plotting, own function?
# TODO Scalable map
radius = 21
for i in range(3)[::-1]:
bbox = get_bbox(radius)
draw.ellipse(bbox, fill=colors.BLACK)
radius -= 1
alerts = alertcount[:, i]
alertsum = np.sum(alerts)
bbox = get_bbox(radius)
if alertsum <= 0:
draw.pieslice(bbox, start=0, end=360, fill=colors.GREY_3)
else:
alerts *= 360. / alertsum
start = 0
for j in range(3):
alert = alerts[j]
color = alertcolors[j]
end = start + alert
draw.pieslice(bbox, start=start, end=end, fill=color)
start += alert
radius -= 5 - i
fontpath = userdef.paths("ttf_file")
font = None
if os.path.isfile(fontpath):
font = ImageFont.truetype(fontpath, 20)
str_ = "{} UTC".format(today.strftime("%Y-%m-%d %H:%M:%S"))
draw.text((10, 10), str_, font=font, fill=(0, 0, 0, 128))
max_height = 1500
width, height = pil_image.size
scale = int(max_height / height)
pil_image = pil_image.resize((scale * width, scale * height),
resample=Image.ANTIALIAS)
data = get_image_data(pil_image)
return render_base(render_template)('sohmap.htm', sohmapimg=data)
import os
import functools
from polyfemos.parser import typeoperator as to
from polyfemos.util.messenger import messenger
from polyfemos.util.randomizer import generate_secret_key
from polyfemos.util import fileutils
from polyfemos.front import request
# TODO command line and cwd
# TODO clean up, if yaml files are not found
# get current working directory
working_dir = os.getcwd()
msg = "Working directory: {}".format(working_dir)
messenger(msg, "R")
_global_config_folder = [working_dir, "conf", "front"]
_network_config_folder = [working_dir, "conf", "front", "networks"]
fn = os.path.join(*_global_config_folder, "global_config.yml")
GLOBAL_CONFIG = fileutils.load_yaml(fn)
def users():
"""
:rtype: dict
:return: ``users`` dictionary from global config yaml file
"""
if GLOBAL_CONFIG is None:
return []
def readfile(self, conffile):
r"""
Reads and interprets contents of the '\*.conf' files.
:type conffile: str
:param conffile: path to '\*.conf' file
"""
conffile = os.path.join(self.__current_conf_folder, conffile)
msg = "Reading conffile: {}".format(conffile)
messenger(msg, "R")
if not os.path.isfile(conffile):
msg = "Given conffile ({}) does not exist".format(conffile)
messenger(msg, "E")
return
self.__current_conf_folder = os.path.dirname(conffile)
rows = fileutils.read_file(conffile)
linenbr = 0
for row in rows:
linenbr += 1
# remove linebreaks and whitespaces at start and end of the row
row = row.strip()
# If row does not start with \ skip the row
if not row or row[0] != resources.SYMBOLS["CMD"]:
continue
# save file and line information for debugging
self.__rlm = "File \"{}\", line {}\n".format(conffile, linenbr)
self.__rlm += " {}\n".format(row)
# Remove everything from the row after #-symbol
if resources.SYMBOLS["COMMENT"] in row:
row = row[:row.index(resources.SYMBOLS["COMMENT"])]
# Split the row according the whitespaces
# and remove extra whitespaces
row = [r for r in row.split() if r]
command = row[0]
args = row[1:]
# Check if given command is defined
if command not in self.commands:
msg = self.__rlm
msg += "Unidentified command \'{}\'".format(command)
messenger(msg, "E")
continue
# Check if any station definition is before parameter definition
if command == resources.CMDS["PAR"] and not self.scopes["station"]:
msg = self.__rlm
msg += "Trying to add parameter before station."
messenger(msg, "E")
return
# extract list of functions to be applied to arguments
argtypes = self.commands[command]['argtypes'][:]
# If RUN command is used
# Check if called function exists and modify argument types
# accordingly
if command == resources.CMDS["RUN"]:
msg = self.__rlm
msg += "Using command \'{}\', ".format(resources.CMDS["RUN"])
if len(argtypes) <= 0:
msg += "no function name given"
messenger(msg, "E")
return
elif not args[0] in self.functions:
msg += "no function \'{}\' available".format(args[0])
messenger(msg, "E")
return
argtypes += self.functions[args[0]]['argtypes']
# Check if correct amount of arguments is given, raise error if not
if len(args) != len(argtypes):
msg = self.__rlm
msg += "Invalid amount of arguments with given " \
"command \'{}\'\n".format(command)
if command == resources.CMDS["RUN"]:
msg += "with function \'{}\'\n".format(args[0])
msg += "Arguments given {}, should be {}" \
.format(len(args), len(argtypes))
messenger(msg, "E")
return
newargs = [self.replace_var(arg) for arg in args]
for i, arg in enumerate(newargs):
if arg is None:
msg = self.__rlm
msg += "Variable \'{}\' at index {} is not defined." \
.format(args[i], i)
messenger(msg, "E")
return
# Convert arguments into their rightful types
# raise error if conversion is not possible
newargs = [f(arg) for f, arg in zip(argtypes, newargs)]
for i, arg in enumerate(newargs):
if arg is None:
msg = self.__rlm
msg += "Argument with invalid type or syntax\n"
msg += "Given \'{}\' at index {}, should be of type {}" \
.format(args[i], i, argtypes[i].__name__)
messenger(msg, "E")
return
self.commands[command]['func'](*newargs)
