def save_starttime(self, tolast=False): """ If ``tolast`` is ``False``, loads the 'laststarttime' value from the pklfile (FLAG `execution_time_file`) before saving the ``self.__timedict``. After this the 'laststarttime' should be unchanged, and 'thisstarttime' should contain the starttime of this exact program run. If ``tolast`` is ``True``, the value of 'thisstarttime' is saved to 'laststarttime'. :type tolast: bool, optional :param tolast: defaults to ``False`` """ pklfile = self.flags['execution_time_file'] msg = self.__rlm if pklfile is None: msg += "FLAG \'execution_time_file\' undefined" messenger(msg, "E") return if not os.path.isdir(os.path.dirname(pklfile)): msg += "FLAG \'execution_time_file\' {}\n".format(pklfile) msg += "Directory does not exist" messenger(msg, "E") return if not os.path.isfile(pklfile) or tolast: self._init_timedict() fileutils.save_obj(pklfile, self.__timedict) return loaded_dict = fileutils.load_obj(pklfile) if "laststarttime" in loaded_dict: self.__timedict["laststarttime"] = loaded_dict["laststarttime"] fileutils.save_obj(pklfile, self.__timedict)
def append_to_file(filename, text, header): """ If path defined in ``filename`` does not exist, new file is created and ``header`` is written in the file. If ``text`` is list, writes csv. :type filename: str :param filename: :type text: str :param text: :type header: str :param header: """ create_missing_folders(filename) write_function = write_file if isinstance(text, list): if not isinstance(header, list): header = [header] write_function = functools.partial(write_csv, singlerow=True) if not os.path.isfile(filename): msg = "No previous file: {}".format(filename) messenger(msg, "R") write_function(filename, header, mode='w') write_function(filename, text, mode='a')
def __init__(self): """ Sets the global program starttime, ``self.__now``. Calls :meth:`~polyfemos.back.interpreter.Interpreter._init_timedict` to initialize ``self.__timedict``. Creates empty :class:`~polyfemos.back.station.Stations()` instance (``self.station``). """ self._quit = False self.__now = UTCDateTime(precision=0) self.stations = Stations() self.vars = {} self.scopes = {"station": False} self.__rlm = "" # reader location message self.__current_conf_folder = "" self._init_flags() self.pool = None self._init_timedict() self._init_funcs_and_commands(self.__now) messenger("", "B") messenger("Program starttime: " + str(self.__now), "R")
def filepath(filepath, extension, force=False): """ Returns the filepath with the given extension if it exists. If not, check if the 'retro' version is available and returns that. If both are unavailable, returns an empty string. :type filepath: str :param filepath: original filepath without the extension :type extension: str :param extension: file extension, e.g. ``.stf``, ``.csv`` or ``.alert`` :type force: bool :param force: If ``True``, returns filepath with the given extension and if it does not exist, returns an empty string. :rtype: str :return: """ tempfilepath = filepath + extension if os.path.isfile(tempfilepath): return tempfilepath msg = "File \'{}\' does not exist.".format(tempfilepath) if not force: tempfilepath = filepath + ".retro" + extension if os.path.isfile(tempfilepath): return tempfilepath msg += "\nFile \'{}\' does not exist.".format(tempfilepath) messenger(msg, "R") return ""
def _read_stf_data(self, nez=False): """ Reads and parses the data from sohtextfiles. :type nez: bool, optional :param nez: defaults to ``False``. If selected sohpar is N, E or Z offset, values for U, W and V offsets has to be read. For information about UWV to NEZ conversion see :class:`~polyfemos.front.sohplot.offsets.UWVOffsets` """ uwvo = offsets.UWVOffsets() stored_data = [] runningdate = self.startdate while (runningdate <= self.enddate): filepath = self._get_stf_filepath(runningdate) data_scope = False runningdate += 86400 if not filepath: continue for row in fileutils.rowsof(filepath): # Data values are read after 'DATA' text is encountered if row[0] == "DATA": data_scope = True continue if data_scope: if len(row) < 3: continue stored_data.append(row) for row in stored_data: if nez and row[1] in uwvo.offsets: uwvo.update(*row[:3]) # If U, W and V offset values are defined, conversion # can be done sensor = self.header["SENSOR"] if uwvo and sensor is not None: dt, value = \ uwvo.transform(sensor, self.sohpar_name[-1]) uwvo.clear() else: continue elif row[1] == self.sohpar_name: dt = to.utcdatetime(row[0]) if dt is None: msg = "Could not convert '{}' to utcdatetime." \ .format(row[0]) messenger(msg, "W") value = row[2] else: continue dp = DataPoint(utcdatetime=dt, y=value) if len(row) > 3: dp.set_z(row[3]) self.data_container.append(dp)
def wrapper(path, *args, **kwargs): quiet = False if 'quiet' in kwargs: quiet = kwargs['quiet'] messenger(path, "R", quiet=quiet) if check_func(path): return func_(path, *args, **kwargs) msg = "{} \'{}\' DOES NOT EXIST".format(msgstr, path) messenger(msg, "R", quiet=quiet) return None
def print_warning(filename, msg): """ :type filename: str :param filename: path to a file where the error was encountered :type msg: str :param msg: Message to be printed, if empty, nothing is printed. """ if msg: msg = "In file: {}\n{}".format(filename, msg).strip() messenger(msg, "W", showpid=False)
def index_alias(filename): """ Index site of the polyfemos web. In addition provides the documentation files. """ msg = "Documentation filename: {}".format(filename) messenger(msg, "R") if not filename: return render_template('index.htm') folder = userdef.paths("doc_dir") return send_from_directory(folder, filename)
def create_missing_folders(filename): """ Creates full path to given ``filename``. :type filename: str :param filename: file or folder """ folder = os.path.dirname(filename) if not os.path.isdir(folder): os.makedirs(folder) msg = "Creating new folder: {}".format(folder) messenger(msg, "R")
def data_timing_quality(path="", average_calc_length=1, scale=lambda x: x, **kwargs): """ Reads timing quality values from miniseed file using :func:`~obspy.io.mseed.util.get_record_information` function. :type path: str :param path: path to miniseed file :type average_calc_length: int, optional :param average_calc_length: defaults to one, setting this value greater than one will result in a little bit smoothed timing quality curve, since the returned datapoints will be averages over ``average_calc_length`` values. :type scale: func, optional :param scale: defaults to identity function, scaling function applied to data values :rtype: list or None :return: list of lists containing timestamp (as :class:`~obspy.core.utcdatetime.UTCDateTime` instance) and data value. """ imm = fileutils.invalid_mseed(path) if imm: messenger(imm, "M") return None timing_quality = [] tqtimes = [] last_qualities = [] offset = 0 # Codes from obspy # Loop over each record. # A valid record needs to have a record length of at # least 256 bytes. info = get_record_information(path) while offset <= (info['filesize'] - 256): this_info = get_record_information(path, offset) if 'timing_quality' in this_info: last_qualities.append(float(this_info['timing_quality'])) length = len(last_qualities) ave = sum(last_qualities) / length if length > average_calc_length: last_qualities = last_qualities[1:] timing_quality.append(ave) tqtimes.append(this_info['starttime']) offset += this_info['record_length'] return [[x, scale(y)] for x, y in zip(tqtimes, timing_quality)]
def pool_error_callback(arg, msg=""): """ A function to be called during pool worker error :type arg: :param arg: Error message :type msg: str, optional :param msg: Additional message """ msg += "while using multiprocessing\n" msg += str(arg) + "\n" messenger(msg, "E", showpid=False)
def define_flag(self, flag, value): """ :type flag: str :param flag: name of the FLAG :type value: :param value: value of the FLAG, the type depends on the flag """ if flag not in self.__flags: msg = self.__rlm msg += "No FLAG \'{}\' available.".format(flag) messenger(msg, "E") return self.flags[flag] = self.__flags[flag]["type"](value)
def main(conffile): """ :type conffile: str :param conffile: Filepath of the conffile """ t0 = time.time() interp = interpreter.Interpreter() interp.readfile(conffile) interp.stop() t1 = time.time() - t0 messenger(str(t1), "N", showpid=False)
def wrapper(self, *args, **kwargs): if self.track_datalen: msg = "in method: {}".format(method.__name__) messenger(msg, "R") orig_len = len(self) orig_nan_len = self.count_nans() method(self, *args, **kwargs) if self.track_datalen: new_len = len(self) new_nan_len = self.count_nans() str_ = "{:*<19}**".format(method.__name__ + ":") str_ += "dps:*{:*>7}*>*{:*<7}*".format(orig_len, new_len) str_ += "nans:*{:*>7}*>*{:*<7}".format(orig_nan_len, new_nan_len) self.add2info(str_)
def add_station(self, *args): """ Tries to define and add a station (:class:`~polyfemos.back.station.Station`) instance to ``self.stations``. ``args`` are passed straight to the :meth:`~polyfemos.back.station.Station.__init__`. """ station = Station(*args) success = self.stations.add_station(station) if not success: msg = self.__rlm msg += "Unable to add station with id {}\n".format(station.id) msg += "Start or endtimes overlap" messenger(msg, "E") return self.scopes["station"] = True
def stop(self, quit_=True, save_starttime=True): """ :type quit\_: bool, optional :param quit\_: defaults to ``True``, If ``True`` the program does nothing anymore. :type save_starttime: bool, optional :param save_starttime: defaults to ``True``, In order to save starttime value, ``save_starttime`` and FLAG 'save_starttime' both are required to be ``True``. """ for k, v in self.__timedict.items(): msg = "{}: {}".format(k, v) messenger(msg, "N") self.close_and_join() if save_starttime and self.flags["save_starttime"]: self.save_starttime(tolast=True) if quit_: self._quit = True
def get_transform(from_epsg, to_epsg): """ :type from_epsg: str :param from_epsg: EPSG code of the original coordinate system :type to_epsg: str :param to_epsg: EPSG code of the resulting coordinate system :rtype: func :return: A coordinate transformation function taking xy-coordinates in ``from_epsg`` coordinate system and returning xy-coordinates in ``to_epsg`` coordinate system """ try: from_coords = pyproj.Proj("+init=EPSG:{}".format(from_epsg)) to_coords = pyproj.Proj("+init=EPSG:{}".format(to_epsg)) except RuntimeError: msg = "No coordinate conversion available for ESPGs from " \ "'{}' to '{}'.".format(from_epsg, to_epsg) messenger(msg, "W") return lambda x, y: (x, y) return lambda x, y: pyproj.transform(from_coords, to_coords, x, y)
def write_file(filename, str_, mode='w', cmf=True): r""" :type filename: str :param filename: filename of the file created :type str\_: str :param str\_: a text to written :type mode: str, optional :param mode: defaults to 'w' (write) :type cmf: bool, optional :param cmf: defaults to ``True``, If ``True`` missing folder in ``filename`` aer created before writing """ if cmf: create_missing_folders(filename) if mode == "a" and not os.path.isfile(filename): msg = "Trying to append to a non existing file: {}".format(filename) messenger(msg, "W") return with open(filename, mode) as f: f.write(str_)
def get_stream(path="", format_="MSEED"): r""" :type path: str :param path: path to seismic data file :type format\_: str, optional :param format\_: defaults to ``MSEED`` :rtype: :class:`~obspy.core.stream.Stream` or None :return: If invalid miniseed file(path) is provided, empty stream is returned. """ msg = invalid_mseed(path) if msg: messenger(msg, "M") return Stream() else: try: st = read(path, format=format_) return st except InternalMSEEDError as e: messenger(str(e), "W") return Stream()
def _read_header(self, nez=False): """ Reads and parses the header information from the sohtextfile. Header block ends when 'DATA' text is encountered. :type nez: bool, optional :param nez: defaults to ``False``, for UWV to NEZ conversion, sensor information from the stf header is needed. """ for key, values in outfilefields.STF_HEADER.items(): # Fill the header with default values self.header[key] = values[-1] self.header["H_LINES"] = [] rowstart = self.sohpar_name if self.sohpar_name else "_" if nez: rowstart = rowstart[:-1] filepath = self._get_stf_filepath(self.headerdate) if not filepath: msg = "Trying read the header of non existing 'stf' file." messenger(msg, "W") return for row in fileutils.rowsof(filepath): if len(row) >= 2: key = row[0] value = row[1] if key.startswith(rowstart): key = key.split("_")[-1] if key in outfilefields.STF_HEADER: self.header[key] = outfilefields.STF_HEADER[key][0](value) if row[0] == "DATA": break h_lines = self.header["YELLOW"] \ + self.header["ORANGE"] + self.header["RED"] self.header["H_LINES"] = \ [line for line in h_lines if not to.isNaN(line)]
def read_file(fn, exclude_added_days=False): if not fn: return if fn in redd_filepaths: return redd_filepaths.add(fn) for row in fileutils.read_csv(fn)[1:]: if len(row) < 2: continue dptimestamp = to.float_(row[0]) if dptimestamp is None: msg = "" messenger(msg, "W") elif starttimestamp <= dptimestamp <= endtimestamp: day = dptimestamp // 86400 if exclude_added_days and day in added_days: continue if not exclude_added_days: added_days.add(day) dp = DataPoint(timestamp=dptimestamp, y=row[1]) if len(row) > 2: dp.set_z(row[2]) self.data_container.append(dp)
def transform_func(key): """ :type key: str :param key: internal sensor code :rtype: func :return: UWV to NEZ transform function """ from math import sqrt def func1(u, w, v): # for streckeisen x = -1. * u * sqrt(2. / 3.) + (v + w) / sqrt(6.) y = (v - w) / sqrt(2.) z = (u + v + w) / sqrt(3.) return (x, y, z) def func2(u, w, v): # for nanometrics x = (2. * u - v - w) / sqrt(6.) y = (v - w) / sqrt(2.) z = (u + v + w) / sqrt(3.) return (x, y, z) funcs = { "STS-2": func1, "Trillium_120PH": func2, "Trillium_120PA": func2, "Trillium_Co120": func2, } if key not in funcs: msg = "Invalid key in transform_func" messenger(msg, "W") return lambda x, y, z: (x, y, z) return funcs[key]
def get_summary(startdate="", enddate="", headerdate="", combinations=[], remove_irrationals=False, advanced_outlier_removal=False, fext="csv"): """ The function used to get the statistical summary of station/sohpar combinations. The execution is little bit slow because the combinations are dealt separately, which means that, as all of the sohpars are in the same file, the file is opened and closed multiple times during the creation of the summary :type startdate: str or :class:`~datetime.date` :param startdate: :type enddate: str or :class:`~datetime.date` :param enddate: :type headerdate: str or :class:`~datetime.date` :param headerdate: The header information of this date is used in calculations :type combinations: list[tuple[str, str]] :param combinations: list containing all unique station and sohpar combinations :type remove_irrationals: bool, optional :param remove_irrationals: defaults to ``False``, If ``True``, irrational values (limits defined in stf header) are removed from the data. :type advanced_outlier_removal: bool, optional :param advanced_outlier_removal: defaults to ``False``. If ``True``, advanced outlier removal is used for specific station/sohpar combination. The combinations and outlier removal are provided by :func:`~polyfemos.front.userdef.summary_outlierremfuncs`, if the YAML configuration is provided. :type fext: str, optional :param fext: defaults to "csv", select "stf" or "csv", defines the datafile format which is read :rtype: list, list :return: rows and header, list of lists and a list. """ rows = [] header = [] max_loops = len(combinations) for i, (station_id, sohpar_name) in enumerate(combinations): sohplot = SOHPlot(station_id=station_id, sohpar_name=sohpar_name, startdate=startdate, enddate=enddate, headerdate=headerdate, remove_irrationals=remove_irrationals, advanced_outlier_removal=advanced_outlier_removal, fext=fext) dict_ = sohplot.get_statistics_dict() if not header: header = ["Station", "Sohpar"] + list(dict_.keys()) row = [station_id, sohpar_name] # If every numerical value is nan, don't show the parameter func_ = lambda x: isinstance(x, str) or math.isnan(x) if all(map(func_, dict_.values())): continue rows.append(row + list(dict_.values())) pr = 100.0 * float(i + 1) / max_loops msg = "Summary table, percents ready: {:.2f}".format(pr) messenger(msg, "R") return rows, header
def sohmap(): """ A map of the network area with stations. Alerts are sorted by their priorities. The innermost circle consists of the alerts of the highest priority. """ alertcolors = { 0: colors.ALERT_GREEN, 1: colors.ALERT_YELLOW, 2: colors.ALERT_RED, } filename = userdef.paths("map_file") filename, extension = os.path.splitext(filename) imgfile = "web_static/{}{}".format(filename, extension) mapfile = "web_static/{}{}".format(filename, ".map") msg = "Img and map files: {}, {}".format(imgfile, mapfile) messenger(msg, "R") if not os.path.isfile(imgfile): return render_base(render_template)('sohmap.htm') if not os.path.isfile(mapfile): return render_base(render_template)('sohmap.htm') # get transformation from WGS84 to pixels pixel_transform = coordinator.transform_from_ozi_map(mapfile) # open background map pil_image = Image.open(imgfile) draw = ImageDraw.Draw(pil_image) today = UTCDateTime().now() julday, year = get_jY(today) sohpar_names = userdef.sohpars() station_ids = userdef.station_ids() # get alerts and priorities with each station and parameter combination fpf = userdef.filepathformats("alert") sohdict = get_sohdict(station_ids, year, julday, fpf) for station_id in station_ids: sohplot = SOHPlot( station_id=station_id, headerdate=today, ) epsg = sohplot.header["EPSG"] locx = sohplot.header["LOCX"] locy = sohplot.header["LOCY"] if epsg is None: continue # Convert stations coordinates into WGS84 transform = coordinator.get_transform(epsg, "4326") px, py = pixel_transform(*transform(locx, locy)) def get_bbox(radius): return (px - radius, py - radius, px + radius, py + radius) # Alerts are stored in 3x3 matrix # Priority in x axis # Alert (red, yellow or green) in y axis # Red alert with the highest priority is stored in [0,2] alertcount = np.zeros((3, 3)) for sohpar_name in sohpar_names: key = station_id + sohpar_name if key not in sohdict['alerts']: continue alert = to.int_(sohdict['alerts'][key]) priority = to.int_(sohdict['priorities'][key]) if alert is None or priority is None or priority > 4: continue priority = min(3, priority) - 1 alertcount[alert, priority] += 1 # TODO clean plotting, own function? # TODO Scalable map radius = 21 for i in range(3)[::-1]: bbox = get_bbox(radius) draw.ellipse(bbox, fill=colors.BLACK) radius -= 1 alerts = alertcount[:, i] alertsum = np.sum(alerts) bbox = get_bbox(radius) if alertsum <= 0: draw.pieslice(bbox, start=0, end=360, fill=colors.GREY_3) else: alerts *= 360. / alertsum start = 0 for j in range(3): alert = alerts[j] color = alertcolors[j] end = start + alert draw.pieslice(bbox, start=start, end=end, fill=color) start += alert radius -= 5 - i fontpath = userdef.paths("ttf_file") font = None if os.path.isfile(fontpath): font = ImageFont.truetype(fontpath, 20) str_ = "{} UTC".format(today.strftime("%Y-%m-%d %H:%M:%S")) draw.text((10, 10), str_, font=font, fill=(0, 0, 0, 128)) max_height = 1500 width, height = pil_image.size scale = int(max_height / height) pil_image = pil_image.resize((scale * width, scale * height), resample=Image.ANTIALIAS) data = get_image_data(pil_image) return render_base(render_template)('sohmap.htm', sohmapimg=data)
import os import functools from polyfemos.parser import typeoperator as to from polyfemos.util.messenger import messenger from polyfemos.util.randomizer import generate_secret_key from polyfemos.util import fileutils from polyfemos.front import request # TODO command line and cwd # TODO clean up, if yaml files are not found # get current working directory working_dir = os.getcwd() msg = "Working directory: {}".format(working_dir) messenger(msg, "R") _global_config_folder = [working_dir, "conf", "front"] _network_config_folder = [working_dir, "conf", "front", "networks"] fn = os.path.join(*_global_config_folder, "global_config.yml") GLOBAL_CONFIG = fileutils.load_yaml(fn) def users(): """ :rtype: dict :return: ``users`` dictionary from global config yaml file """ if GLOBAL_CONFIG is None: return []
def readfile(self, conffile): r""" Reads and interprets contents of the '\*.conf' files. :type conffile: str :param conffile: path to '\*.conf' file """ conffile = os.path.join(self.__current_conf_folder, conffile) msg = "Reading conffile: {}".format(conffile) messenger(msg, "R") if not os.path.isfile(conffile): msg = "Given conffile ({}) does not exist".format(conffile) messenger(msg, "E") return self.__current_conf_folder = os.path.dirname(conffile) rows = fileutils.read_file(conffile) linenbr = 0 for row in rows: linenbr += 1 # remove linebreaks and whitespaces at start and end of the row row = row.strip() # If row does not start with \ skip the row if not row or row[0] != resources.SYMBOLS["CMD"]: continue # save file and line information for debugging self.__rlm = "File \"{}\", line {}\n".format(conffile, linenbr) self.__rlm += " {}\n".format(row) # Remove everything from the row after #-symbol if resources.SYMBOLS["COMMENT"] in row: row = row[:row.index(resources.SYMBOLS["COMMENT"])] # Split the row according the whitespaces # and remove extra whitespaces row = [r for r in row.split() if r] command = row[0] args = row[1:] # Check if given command is defined if command not in self.commands: msg = self.__rlm msg += "Unidentified command \'{}\'".format(command) messenger(msg, "E") continue # Check if any station definition is before parameter definition if command == resources.CMDS["PAR"] and not self.scopes["station"]: msg = self.__rlm msg += "Trying to add parameter before station." messenger(msg, "E") return # extract list of functions to be applied to arguments argtypes = self.commands[command]['argtypes'][:] # If RUN command is used # Check if called function exists and modify argument types # accordingly if command == resources.CMDS["RUN"]: msg = self.__rlm msg += "Using command \'{}\', ".format(resources.CMDS["RUN"]) if len(argtypes) <= 0: msg += "no function name given" messenger(msg, "E") return elif not args[0] in self.functions: msg += "no function \'{}\' available".format(args[0]) messenger(msg, "E") return argtypes += self.functions[args[0]]['argtypes'] # Check if correct amount of arguments is given, raise error if not if len(args) != len(argtypes): msg = self.__rlm msg += "Invalid amount of arguments with given " \ "command \'{}\'\n".format(command) if command == resources.CMDS["RUN"]: msg += "with function \'{}\'\n".format(args[0]) msg += "Arguments given {}, should be {}" \ .format(len(args), len(argtypes)) messenger(msg, "E") return newargs = [self.replace_var(arg) for arg in args] for i, arg in enumerate(newargs): if arg is None: msg = self.__rlm msg += "Variable \'{}\' at index {} is not defined." \ .format(args[i], i) messenger(msg, "E") return # Convert arguments into their rightful types # raise error if conversion is not possible newargs = [f(arg) for f, arg in zip(argtypes, newargs)] for i, arg in enumerate(newargs): if arg is None: msg = self.__rlm msg += "Argument with invalid type or syntax\n" msg += "Given \'{}\' at index {}, should be of type {}" \ .format(args[i], i, argtypes[i].__name__) messenger(msg, "E") return self.commands[command]['func'](*newargs)