def test_click_process(mock_process): runner = CliRunner() d_conffile, conffile, pyfile = \ get_templates_fpaths("download.yaml", "paramtable.yaml", "paramtable.py") # test no dburl supplied mock_process.reset_mock() result = runner.invoke(cli, ['process', '-c', conffile, '-p', pyfile, 'c']) assert "Missing option" in result.output assert result.exc_info # test dburl supplied mock_process.reset_mock() result = runner.invoke( cli, ['process', '-d', 'd', '-c', conffile, '-p', pyfile, 'c']) lst = list(mock_process.call_args_list[0][0]) assert lst == ['d', pyfile, None, conffile, 'c'] assert result.exit_code == 0 # test dburl supplied via config mock_process.reset_mock() result = runner.invoke( cli, ['process', '-d', d_conffile, '-c', conffile, '-p', pyfile, 'c']) lst = list(mock_process.call_args_list[0][0]) assert lst == [yaml_load(d_conffile)['dburl'], pyfile, None, conffile, 'c'] assert result.exit_code == 0 # test funcname supplied via cli: mock_process.reset_mock() result = runner.invoke(cli, [ 'process', '--funcname', 'wat?', '-d', d_conffile, '-c', conffile, '-p', pyfile, 'c' ]) lst = list(mock_process.call_args_list[0][0]) assert lst == [ yaml_load(d_conffile)['dburl'], pyfile, 'wat?', conffile, 'c' ] assert result.exit_code == 0 # test an error in params: -dburl instead of --dburl: mock_process.reset_mock() result = runner.invoke( cli, ['process', '-dburl', d_conffile, '-c', conffile, '-p', pyfile, 'c']) assert not mock_process.called assert result.exit_code != 0 # assert help works: mock_process.reset_mock() result = runner.invoke(cli, ['process', '--help']) assert not mock_process.called assert result.exit_code == 0
def test_click_show(mock_create_s2s_show_app, mock_open_in_browser, mock_show): runner = CliRunner() d_conffile, conffile, pyfile = \ get_templates_fpaths("download.yaml", "paramtable.yaml", "paramtable.py") # when asserting if we called open_in_browser, since tha latter is inside a thread which # executes with a delay of 1.5 seconds, we need to make our function here. Quite hacky, # but who cares def assert_opened_in_browser(url=None): # if None, assert time.sleep(2) # to be safe mock_open_in_browser.assert_called_once args = mock_open_in_browser.call_args_list[0][0] assert len(args) == 1 assert args[0].startswith('http://127.0.0.1:') # test no dburl supplied mock_show.reset_mock() mock_open_in_browser.reset_mock() result = runner.invoke(cli, ['show', '-c', conffile, '-p', pyfile]) assert "Missing option" in result.output assert result.exc_info assert not mock_open_in_browser.called # test dburl supplied mock_show.reset_mock() mock_open_in_browser.reset_mock() result = runner.invoke(cli, ['show', '-d', 'd', '-c', conffile, '-p', pyfile]) lst = list(mock_show.call_args_list[0][0]) assert lst == ['d', pyfile, conffile] assert result.exit_code == 0 assert_opened_in_browser('d') # test dburl supplied via config mock_show.reset_mock() mock_open_in_browser.reset_mock() result = runner.invoke( cli, ['show', '-d', d_conffile, '-c', conffile, '-p', pyfile]) lst = list(mock_show.call_args_list[0][0]) dburl = yaml_load(d_conffile)['dburl'] assert lst == [dburl, pyfile, conffile] assert result.exit_code == 0 assert_opened_in_browser(dburl) # test an error in params: -dburl instead of --dburl: mock_show.reset_mock() mock_open_in_browser.reset_mock() result = runner.invoke( cli, ['show', '-dburl', d_conffile, '-c', conffile, '-p', pyfile]) assert not mock_show.called assert result.exit_code != 0 assert not mock_open_in_browser.called # assert help works: mock_show.reset_mock() mock_open_in_browser.reset_mock() result = runner.invoke(cli, ['show', '--help']) assert not mock_show.called assert result.exit_code == 0 assert not mock_open_in_browser.called
def extract_dburl_if_yamlpath(value, param_name='dburl'): """ Returns the database path from 'value': 'value' can be a file (in that case is assumed to be a yaml file with the `param_name` key in it, which must denote a db path) or the database path otherwise """ if not isinstance(value, string_types) or not value: raise TypeError('please specify a string denoting either a path to a yaml file with the ' '`dburl` parameter defined, or a valid db path') return yaml_load(value)[param_name] if (os.path.isfile(value)) else value
def test_yaml_load(): # NB: all dic keys must be strings dic1 = {'a': 7, '5': 'h'} dic2 = {'a': 7, '7': 'h'} d = yaml_load(dic1, **dic2) assert d['a'] == 7 assert d['5'] == 'h' assert d['7'] == 'h' assert sorted(d.keys()) == sorted(['a', '5', '7']) dic1 = {'a': 7, '5': 'h', 'v': {1: 2, 3: 4}} dic2 = {'a': 7, '7': 'h', 'v': {1: 2, 3: 5}} d = yaml_load(dic1, **dic2) assert d['a'] == 7 assert d['5'] == 'h' assert d['7'] == 'h' assert d['v'][1] == 2 assert d['v'][3] == 5 assert sorted(d.keys()) == sorted(['a', '5', '7', 'v'])
def download_setup_func(filename, **params): yamldic = yaml_load(get_templates_fpath(filename)) for key, val in params.items(): if val is None: yamldic.pop(key, None) else: yamldic[key] = val path = os.path.join(basedir, filename) with open(path, 'w') as _opn: yaml.safe_dump(yamldic, _opn) return path, yamldic
def yaml_get(yaml_content): '''Returns the arguments used for the eventws query stored in the yaml, or an empty dict in case of errors :param yaml_content: yaml formatted string representing a download config''' try: dic = yaml_load(StringIO(yaml_content)) ret = {k: dic[k] for k in EVENTWS_SAFE_PARAMS if k in dic} additional_eventws_params = dic.get('eventws_query_args', None) or {} ret.update(additional_eventws_params) return ret except Exception as _: # pylint: disable=broad-except return {}
def load_config_for_process(dburl, pyfile, funcname=None, config=None, outfile=None, **param_overrides): '''checks process arguments. Returns the tuple session, pyfunc, config_dict, where session is the dql alchemy session from `dburl`, pyfunc is the python function loaded from `pyfile`, and config_dict is the dict loaded from `config` which must denote a path to a yaml file, or None (config_dict will be empty in this latter case) ''' try: session = get_session(dburl, True) except Exception as exc: raise BadArgument('dburl', exc) try: funcname = get_funcname(funcname) except Exception as exc: raise BadArgument('funcname', exc) try: # yaml_load accepts a file name or a dict config = yaml_load({} if config is None else config, **param_overrides) except Exception as exc: raise BadArgument('config', exc) # NOTE: contrarily to the download routine, we cannot check the types of the config because # no parameter is mandatory, and thus they might NOT be present in the config. try: pyfunc = load_pyfunc(pyfile, funcname) except Exception as exc: raise BadArgument('pyfile', exc) if outfile is not None: try: filewritable(outfile) except Exception as exc: raise BadArgument('outfile', exc) # nothing more to process return session, pyfunc, funcname, config
def init(self, request, db, data): # re-init a sqlite database (no-op if the db is not sqlite): db.create(to_file=False) # setup a run_id: rdw = Download() db.session.add(rdw) db.session.commit() self.run = rdw # side effects: self._evt_urlread_sideeffect = """#EventID | Time | Latitude | Longitude | Depth/km | Author | Catalog | Contributor | ContributorID | MagType | Magnitude | MagAuthor | EventLocationName 20160508_0000129|2016-05-08 05:17:11.500000|1|1|60.0|AZER|EMSC-RTS|AZER|505483|ml|3|AZER|CASPIAN SEA, OFFSHR TURKMENISTAN 20160508_0000004|2016-05-08 01:45:30.300000|90|90|2.0|EMSC|EMSC-RTS|EMSC|505183|ml|4|EMSC|CROATIA """ self._dc_urlread_sideeffect = """http://geofon.gfz-potsdam.de/fdsnws/dataselect/1/query ZZ * * * 2002-09-01T00:00:00 2005-10-20T00:00:00 UP ARJ * * 2013-08-01T00:00:00 2017-04-25 http://ws.resif.fr/fdsnws/dataselect/1/query ZU * * HHZ 2015-01-01T00:00:00 2016-12-31T23:59:59.999999 """ # Note: by default we set sta_urlsideeffect to return such a channels which result in 12 # segments (see lat and lon of channels vs lat and lon of events above) self._sta_urlread_sideeffect = [ """#Network|Station|Location|Channel|Latitude|Longitude|Elevation|Depth|Azimuth|Dip|SensorDescription|Scale|ScaleFreq|ScaleUnits|SampleRate|StartTime|EndTime GE|FLT1||HHE|1|1|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2003-01-01T00:00:00| GE|FLT1||HHN|1|1|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2003-01-01T00:00:00| GE|FLT1||HHZ|1|1|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2003-01-01T00:00:00| n1|s||c1|90|90|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2009-01-01T00:00:00| n1|s||c2|90|90|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2009-01-01T00:00:00| n1|s||c3|90|90|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2009-01-01T00:00:00| """, """#Network|Station|Location|Channel|Latitude|Longitude|Elevation|Depth|Azimuth|Dip|SensorDescription|Scale|ScaleFreq|ScaleUnits|SampleRate|StartTime|EndTime IA|BAKI||BHE|1|1|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2003-01-01T00:00:00| IA|BAKI||BHN|1|1|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2003-01-01T00:00:00| IA|BAKI||BHZ|1|1|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2003-01-01T00:00:00| n2|s||c1|90|90|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2009-01-01T00:00:00| n2|s||c2|90|90|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2009-01-01T00:00:00| n2|s||c3|90|90|485.0|0.0|90.0|0.0|GFZ:HT1980:CMG-3ESP/90/g=2000|838860800.0|0.1|M/S|100.0|2009-01-01T00:00:00| """ ] self._mintraveltime_sideeffect = cycle([1]) self._seg_data = data.read("GE.FLT1..HH?.mseed") self._seg_data_gaps = data.read("IA.BAKI..BHZ.D.2016.004.head") self._seg_data_empty = b'' self._seg_urlread_sideeffect = [ self._seg_data, self._seg_data_gaps, 413, 500, self._seg_data[:2], self._seg_data_empty, 413, URLError("++urlerror++"), socket.timeout() ] self.service = '' # so get_datacenters_df accepts any row by default self.db_buf_size = 1 self.routing_service = yaml_load(get_templates_fpath("download.yaml"))\ ['advanced_settings']['routing_service_url'] # NON db stuff (logging, patchers, pandas...): self.logout = StringIO() handler = StreamHandler(stream=self.logout) self._logout_cache = "" # THIS IS A HACK: query_logger.setLevel(logging.INFO) # necessary to forward to handlers # if we called closing (we are testing the whole chain) the level will be reset # (to level.INFO) otherwise it stays what we set two lines above. Problems might arise # if closing sets a different level, but for the moment who cares query_logger.addHandler(handler) # when debugging, I want the full dataframe with to_string(), not truncated # NOTE: this messes up right alignment of numbers in DownloadStats (see utils.py) # FIRST, remember current settings and restore them in cleanup: _pd_display_maxcolwidth = pd.get_option('display.max_colwidth') pd.set_option('display.max_colwidth', -1) # define class level patchers (we do not use a yiled as we need to do more stuff in the # finalizer, see below patchers = [] patchers.append(patch('stream2segment.utils.url.urlopen')) self.mock_urlopen = patchers[-1].start() # mock ThreadPool (tp) to run one instance at a time, so we get deterministic results: class MockThreadPool(object): def __init__(self, *a, **kw): pass def imap(self, func, iterable, *args): # make imap deterministic: same as standard python map: # everything is executed in a single thread the right input order return map(func, iterable) def imap_unordered(self, func_, iterable, *args): # make imap_unordered deterministic: same as standard python map: # everything is executed in a single thread in the right input order return map(func_, iterable) def close(self, *a, **kw): pass # assign patches and mocks: patchers.append(patch('stream2segment.utils.url.ThreadPool')) self.mock_tpool = patchers[-1].start() self.mock_tpool.side_effect = MockThreadPool # add finalizer: def delete(): pd.set_option('display.max_colwidth', _pd_display_maxcolwidth) for patcher in patchers: patcher.stop() hndls = query_logger.handlers[:] handler.close() for h in hndls: if h is handler: query_logger.removeHandler(h) request.addfinalizer(delete)
def test_download_eventws_query_args( mock_isfile, mock_run_download, # fixtures: run_cli_download): # pylint: disable=redefined-outer-name '''test different scenarios where we provide eventws query args from the command line''' d_yaml_file = get_templates_fpath("download.yaml") # FIRST SCENARIO: no eventws_params porovided mock_run_download.reset_mock() def_yaml_dict = yaml_load(d_yaml_file)['eventws_params'] assert not def_yaml_dict # None or empty dict result = run_cli_download() # invalid type assert result.exit_code == 0 # assert the yaml (as passed to the download function) has the correct value: real_eventws_params = mock_run_download.call_args_list[0][1][ 'eventws_params'] # just assert it has keys merged from the global event-related yaml keys assert 'maxmagnitude' not in real_eventws_params assert real_eventws_params # test by providing an eventsws param which is not optional: mock_run_download.reset_mock() def_yaml_dict = yaml_load(d_yaml_file)['eventws_params'] assert not def_yaml_dict # None or empty dict result = run_cli_download('--minmagnitude', '15.5') assert result.exit_code == 0 # assert the yaml (as passed to the download function) has the correct value: real_eventws_params = mock_run_download.call_args_list[0][1][ 'eventws_params'] # just assert it has keys merged from the global event-related yaml keys assert real_eventws_params['minmagnitude'] == 15.5 # test by providing a eventsws param which is optional: mock_run_download.reset_mock() def_yaml_dict = yaml_load(d_yaml_file)['eventws_params'] assert not def_yaml_dict # None or empty dict result = run_cli_download('--minmagnitude', '15.5', eventws_params={'format': 'abc'}) assert result.exit_code == 0 # assert the yaml (as passed to the download function) has the correct value: real_eventws_params = mock_run_download.call_args_list[0][1][ 'eventws_params'] # just assert it has keys merged from the global event-related yaml keys assert real_eventws_params['minmagnitude'] == 15.5 assert real_eventws_params['format'] == 'abc' # conflicting args (supplying a global non-optional param in eventws's config): for pars in [['--minlatitude', '-minlat'], ['--maxlatitude', '-maxlat'], ['--minlongitude', '-minlon'], ['--maxlongitude', '-maxlon'], ['--minmagnitude', '-minmag'], ['--maxmagnitude', '-maxmag'], ['--mindepth'], ['--maxdepth']]: for par1, par2 in product(pars, pars): mock_run_download.reset_mock() result = run_cli_download( par1, '15.5', eventws_params={par2.replace('-', ''): 15.5}) assert result.exit_code != 1 assert 'conflict' in result.output assert 'Invalid value for "eventws_params"' in result.output # test a eventws supplied as non existing file and not valid fdsnws: mock_isfile.reset_mock() assert not mock_isfile.called result = run_cli_download('--eventws', 'myfile') assert result.exit_code != 0 assert 'eventws' in result.output assert mock_isfile.called
def load_config_for_download(config, parseargs, **param_overrides): '''loads download arguments from the given config (yaml file or dict) after parsing and checking some of the dict keys. :return: a dict loaded from the given `config` and with parseed arguments (dict keys) Raises BadArgument in case of parsing errors, missisng arguments, conflicts etcetera ''' try: config_dict = yaml_load(config, **param_overrides) except Exception as exc: raise BadArgument('config', exc) if parseargs: # few variables: configfile = config if (isinstance(config, string_types) and os.path.isfile(config))\ else None # define first default event params in order to avoid typos def_evt_params = EVENTWS_SAFE_PARAMS # now, what we want to do here is basically convert config_dict keys # into suitable arguments for stream2segment functions: this includes # renaming params, parsing/converting their values, raising # BadArgument exceptions and so on # Let's configure a 'params' list, a list of dicts where each dict is a 'param checker' # with the following keys (at least one should be provided): # names: list of strings. provide it in order to check for optional names, # check that only one param is provided, and # replace whatever is found with the first item in the list # newname: string, provide it if you want to replace names above with this value # instead first item in 'names' # defvalue: if provided, then the parameter is optional and will be set to this value # if not provided, then the parameter is mandatory (BadArgument is raised in case) # newvalue: function accepting a value (the parameter value) raising whatever is # needed if the parameter is invalid, and returning the correct parameter value params = [ { 'names': def_evt_params[:2], # ['minlatitude', 'minlat'], 'defvalue': None, 'newvalue': between(-90.0, 90.0) }, { 'names': def_evt_params[2:4], # ['maxlatitude', 'maxlat'], 'defvalue': None, 'newvalue': between(-90.0, 90.0) }, { 'names': def_evt_params[4:6], # ['minlongitude', 'minlon'], 'defvalue': None, 'newvalue': between(-180.0, 180.0) }, { 'names': def_evt_params[6:8], # ['maxlongitude', 'maxlon'], 'defvalue': None, 'newvalue': between(-180.0, 180.0) }, { 'names': def_evt_params[8:10], # ['minmagnitude', 'minmag'], 'defvalue': None }, { 'names': def_evt_params[10:12], # ['maxmagnitude', 'maxmag'], 'defvalue': None }, { 'names': def_evt_params[12:13], # ['mindepth'], 'defvalue': None }, { 'names': def_evt_params[13:14], # ['maxdepth'], 'defvalue': None }, { 'names': ['update_metadata'], 'newvalue': parse_update_metadata }, { 'names': ['restricted_data'], 'newname': 'authorizer', 'newvalue': lambda val: create_auth(val, config_dict['dataws'], configfile) }, { 'names': ['dburl'], 'newname': 'session', 'newvalue': get_session }, { 'names': ['traveltimes_model'], 'newname': 'tt_table', 'newvalue': load_tt_table }, { 'names': ('starttime', 'start'), 'newvalue': valid_date }, { 'names': ('endtime', 'end'), 'newvalue': valid_date }, { 'names': ['eventws'], 'newvalue': lambda url: valid_fdsn(url, is_eventws=True, configfile=configfile) }, { 'names': ['dataws'], 'newvalue': lambda url: valid_fdsn(url, is_eventws=False) }, { 'names': ('network', 'net', 'networks'), 'defvalue': [], 'newvalue': nslc_param_value_aslist }, { 'names': ('station', 'sta', 'stations'), 'defvalue': [], 'newvalue': nslc_param_value_aslist }, { 'names': ('location', 'loc', 'locations'), 'defvalue': [], 'newvalue': nslc_param_value_aslist }, { 'names': ('channel', 'cha', 'channels'), 'defvalue': [], 'newvalue': nslc_param_value_aslist }, {'names': ['eventws_params', 'eventws_query_args']}, { 'names': ['advanced_settings'], 'newvalue': parse_download_advanced_settings }, { 'names': ['search_radius'], 'newvalue': check_search_radius } ] # store all keys now because we might change them (see below): all_keys = set(config_dict) # do the check (this MODIFIES config_dict in place!): parse_arguments(config_dict, *params) # Now check for: # 1a. parameter supplied here NOT in the default config # 1b. parameter supplied here with different type of the default config # 2. Parameters in the default config not supplied here # First, create some sets of params names: # the parsed keys (all names defined above): parsed_keys = set(chain(*(_['names'] for _ in params))) # load original configuration (default in this package): orig_config = yaml_load(get_templates_fpath("download.yaml")) # Check 1a. and 1b.: for key in all_keys - parsed_keys: try: other_value = orig_config[key] except KeyError: raise BadArgument(key, '', 'No such option') try: typesmatch(config_dict[key], other_value) except Exception as exc: raise BadArgument(key, exc) # Check 2. : missing_keys = set(orig_config) - all_keys - parsed_keys if missing_keys: raise BadArgument(list(missing_keys), KeyError()) # At last, put all event-related parameters (except starttime and endtime): # and in the eventws_params dict (the latter is an OPTIONAL dict # which can be set in the config for ADDITIONAL eventws parameters) # and check for conflicts: eventsearchparams = config_dict['eventws_params'] # eventsearchparams might be none if not eventsearchparams: config_dict['eventws_params'] = eventsearchparams = {} for par in def_evt_params: if par in eventsearchparams: # conflict: raise BadArgument('eventws_params', 'conflicting parameter "%s"' % par) value = config_dict.pop(par, None) if value is not None: eventsearchparams[par] = value return config_dict
def init(self, request, db, data): # re-init a sqlite database (no-op if the db is not sqlite): db.create(to_file=False, process=True) # init db: session = db.session dct = DataCenter(station_url="345fbgfnyhtgrefs", dataselect_url='edfawrefdc') session.add(dct) utcnow = datetime.utcnow() dwl = Download(run_time=utcnow) session.add(dwl) ws = WebService(url='webserviceurl') session.add(ws) session.commit() # id = 'firstevent' ev1 = Event(event_id='event1', webservice_id=ws.id, time=utcnow, latitude=89.5, longitude=6, depth_km=7.1, magnitude=56) # note: e2 not used, store in db here anyway... ev2 = Event(event_id='event2', webservice_id=ws.id, time=utcnow + timedelta(seconds=5), latitude=89.5, longitude=6, depth_km=7.1, magnitude=56) session.add_all([ev1, ev2]) session.commit() # refresh datacenter id (alo flush works) d = datetime.utcnow() s = Station(network='network', station='station', datacenter_id=dct.id, latitude=90, longitude=-45, start_time=d) session.add(s) channels = [ Channel(location='01', channel='HHE', sample_rate=6), Channel(location='01', channel='HHN', sample_rate=6), Channel(location='01', channel='HHZ', sample_rate=6), Channel(location='01', channel='HHW', sample_rate=6), Channel(location='02', channel='HHE', sample_rate=6), Channel(location='02', channel='HHN', sample_rate=6), Channel(location='02', channel='HHZ', sample_rate=6), Channel(location='04', channel='HHZ', sample_rate=6), Channel(location='05', channel='HHE', sample_rate=6), Channel(location='05gap_merged', channel='HHN', sample_rate=6), Channel(location='05err', channel='HHZ', sample_rate=6), Channel(location='05gap_unmerged', channel='HHZ', sample_rate=6) ] s.channels.extend(channels) session.commit() fixed_args = dict(datacenter_id=dct.id, download_id=dwl.id) # Note: data_gaps_merged is a stream where gaps can be merged via obspy.Stream.merge # data_gaps_unmerged is a stream where gaps cannot be merged (is a stream of three # different channels of the same event) data_gaps_unmerged = data.read("GE.FLT1..HH?.mseed") data_gaps_merged = data.read("IA.BAKI..BHZ.D.2016.004.head") data_ok = data.read("GE.FLT1..HH?.mseed") # create an 'ok' and 'error' Stream, the first by taking the first trace of # "GE.FLT1..HH?.mseed", the second by maipulating it obspy_stream = data.read_stream( "GE.FLT1..HH?.mseed") # read(BytesIO(data_ok)) obspy_trace = obspy_stream[0] # write data_ok is actually bytes data of 3 traces, write just the first one, we have # as it is it would be considered a trace with gaps, wwe have # another trace with gaps b = BytesIO() obspy_trace.write(b, format='MSEED') data_ok = b.getvalue() data_err = data_ok[:5] # whatever slice should be ok seedid_ok = seedid_err = obspy_trace.get_id() seedid_gaps_unmerged = None seedid_gaps_merged = read(BytesIO(data_gaps_merged))[0].get_id() for evt, cha in product([ev1], channels): val = int(cha.location[:2]) mseed = data_gaps_merged if "gap_merged" in cha.location else \ data_err if "err" in cha.location else \ data_gaps_unmerged if 'gap_unmerged' in cha.location else data_ok seedid = seedid_gaps_merged if "gap_merged" in cha.location else \ seedid_err if 'err' in cha.location else \ seedid_gaps_unmerged if 'gap_unmerged' in cha.location else seedid_ok # set times. For everything except data_ok, we set a out-of-bounds time: start_time = evt.time - timedelta(seconds=5) arrival_time = evt.time - timedelta(seconds=4) end_time = evt.time - timedelta(seconds=1) if "gap_merged" not in cha.location and 'err' not in cha.location and \ 'gap_unmerged' not in cha.location: start_time = obspy_trace.stats.starttime.datetime arrival_time = ( obspy_trace.stats.starttime + (obspy_trace.stats.endtime - obspy_trace.stats.starttime) / 2).datetime end_time = obspy_trace.stats.endtime.datetime seg = Segment(request_start=start_time, arrival_time=arrival_time, request_end=end_time, data=mseed, data_seed_id=seedid, event_distance_deg=val, event_id=evt.id, **fixed_args) cha.segments.append(seg) session.commit() self.inventory_bytes = data.read("GE.FLT1.xml") self.inventory = data.read_inv("GE.FLT1.xml") pfile, cfile = get_templates_fpaths('paramtable.py', 'paramtable.yaml') self.pymodule = load_source(pfile) self.config = yaml_load(cfile) # remove segment_select, we use all segments here: self.config.pop('segment_select', None)
def create_plot_manager(pyfile, configfile): pymodule = None if pyfile is None else load_source(pyfile) configdict = {} if configfile is None else yaml_load(configfile) global PLOT_MANAGER # pylint: disable=global-statement PLOT_MANAGER = PlotManager(pymodule, configdict) return PLOT_MANAGER
def test_click_template(mock_main_init, mock_input, pytestdir): runner = CliRunner() # assert help works: result = runner.invoke(cli, ['init', '--help']) assert not mock_main_init.called assert result.exit_code == 0 expected_files = [ 'download.yaml', 'paramtable.py', 'paramtable.yaml', 'save2fs.py', 'save2fs.yaml', 'jupyter.example.ipynb', 'jupyter.example.db' ] non_python_files = [ _ for _ in expected_files if os.path.splitext(_)[1] not in ('.py', '.yaml') ] dir_ = pytestdir.makedir() path = os.path.join(dir_, 'abc') def max_mod_time(): return max( os.path.getmtime(os.path.join(path, f)) for f in os.listdir(path)) result = runner.invoke(cli, ['init', path]) # FIXME: check how to mock os.path.isfile properly. This doesnot work: # assert mock_isfile.call_count == 5 assert result.exit_code == 0 assert mock_main_init.called files = os.listdir(path) assert sorted(files) == sorted(expected_files) assert not mock_input.called # assert we correctly wrote the files for fle in files: sourcepath = get_templates_fpath(fle) destpath = os.path.join(path, fle) if os.path.splitext(fle)[1] == '.yaml': # check loaded yaml, which also assures our templates are well formed: sourceconfig = yaml_load(sourcepath) destconfig = yaml_load(destpath) if os.path.basename(sourcepath) == 'download.yaml': assert sorted(sourceconfig.keys()) == sorted(destconfig.keys()) else: # assert we have all keys. Note that 'advanced_settings' is not in # sourceconfig (it is added via jinja2 templating system): assert sorted(['advanced_settings'] + list(sourceconfig.keys())) \ == sorted(destconfig.keys()) for key in sourceconfig.keys(): assert type(sourceconfig[key]) == type(destconfig[key]) elif os.path.splitext(fle)[1] == '.py': # check loaded python modules, which also assures our templates are well formed: sourcepy = load_source(sourcepath) destpy = load_source(destpath) # avoid comparing "__blabla__" methods as they are intended to be python # 'private' attributes and there are differences between py2 and py3 # we want to test OUR stuff is the same sourcekeys = [ a for a in dir(sourcepy) if (a[:2] + a[-2:]) != "____" ] destkeys = [a for a in dir(destpy) if (a[:2] + a[-2:]) != "____"] assert sorted(sourcekeys) == sorted(destkeys) for key in sourcekeys: assert type(getattr(sourcepy, key)) == type(getattr(destpy, key)) elif fle not in non_python_files: raise ValueError( 'The file "%s" is not supposed to be copied by `init`' % fle) # try to write to the same dir (1) mock_input.reset_mock() mock_input.side_effect = lambda arg: '1' # overwrite all files maxmodtime = max_mod_time() # we'll test that files are modified, but on mac timestamps are rounded to seconds # so wait 1 second to be safe time.sleep(1) result = runner.invoke(cli, ['init', path]) assert mock_input.called assert max_mod_time() > maxmodtime assert '%d file(s) copied in' % len(expected_files) in result.output # try to write to the same dir (2) for click_prompt_ret_val in ('', '2'): # '' => skip overwrite # '2' => overwrite only non existing # in thus case, both the above returned values produce the same result mock_input.reset_mock() mock_input.side_effect = lambda arg: click_prompt_ret_val maxmodtime = max_mod_time() time.sleep(1) # see comment above result = runner.invoke(cli, ['init', path]) assert mock_input.called assert max_mod_time() == maxmodtime assert 'No file copied' in result.output os.remove(os.path.join(path, expected_files[0])) # try to write to the same dir (2) mock_input.reset_mock() mock_input.side_effect = lambda arg: '2' # overwrite non-existing (1) file maxmodtime = max_mod_time() time.sleep(1) # see comment above result = runner.invoke(cli, ['init', path]) assert mock_input.called assert max_mod_time() > maxmodtime assert '1 file(s) copied in' in result.output
def init(self, request, db, data): # re-init a sqlite database (no-op if the db is not sqlite): db.create(to_file=False) # setup a run_id: rdw = Download() db.session.add(rdw) db.session.commit() self.run = rdw # side effects: self._dc_urlread_sideeffect = """http://geofon.gfz-potsdam.de/fdsnws/dataselect/1/query ZZ * * * 2002-09-01T00:00:00 2005-10-20T00:00:00 UP ARJ * * 2013-08-01T00:00:00 2017-04-25 http://ws.resif.fr/fdsnws/dataselect/1/query ZU * * HHZ 2015-01-01T00:00:00 2016-12-31T23:59:59.999999 """ self._mintraveltime_sideeffect = cycle([1]) self._seg_data = data.read("GE.FLT1..HH?.mseed") self._seg_data_gaps = data.read("IA.BAKI..BHZ.D.2016.004.head") self._seg_data_empty = b'' self._seg_urlread_sideeffect = [ self._seg_data, self._seg_data_gaps, 413, 500, self._seg_data[:2], self._seg_data_empty, 413, URLError("++urlerror++"), socket.timeout() ] self.service = '' # so get_datacenters_df accepts any row by default self.db_buf_size = 1 self.routing_service = yaml_load(get_templates_fpath("download.yaml"))\ ['advanced_settings']['routing_service_url'] # NON db stuff (logging, patchers, pandas...): self.logout = StringIO() handler = StreamHandler(stream=self.logout) self._logout_cache = "" # THIS IS A HACK: query_logger.setLevel(logging.INFO) # necessary to forward to handlers # if we called closing (we are testing the whole chain) the level will be reset # (to level.INFO) otherwise it stays what we set two lines above. Problems might arise # if closing sets a different level, but for the moment who cares query_logger.addHandler(handler) # when debugging, I want the full dataframe with to_string(), not truncated # NOTE: this messes up right alignment of numbers in DownloadStats (see utils.py) # FIRST, remember current settings and restore them in cleanup: _pd_display_maxcolwidth = pd.get_option('display.max_colwidth') pd.set_option('display.max_colwidth', -1) # define class level patchers (we do not use a yiled as we need to do more stuff in the # finalizer, see below patchers = [] patchers.append(patch('stream2segment.utils.url.urlopen')) self.mock_urlopen = patchers[-1].start() # mock ThreadPool (tp) to run one instance at a time, so we get deterministic results: class MockThreadPool(object): def __init__(self, *a, **kw): pass def imap(self, func, iterable, *args): # make imap deterministic: same as standard python map: # everything is executed in a single thread the right input order return map(func, iterable) def imap_unordered(self, func_, iterable, *args): # make imap_unordered deterministic: same as standard python map: # everything is executed in a single thread in the right input order return map(func_, iterable) def close(self, *a, **kw): pass # assign patches and mocks: patchers.append(patch('stream2segment.utils.url.ThreadPool')) self.mock_tpool = patchers[-1].start() self.mock_tpool.side_effect = MockThreadPool # add finalizer: def delete(): pd.set_option('display.max_colwidth', _pd_display_maxcolwidth) for patcher in patchers: patcher.stop() hndls = query_logger.handlers[:] handler.close() for h in hndls: if h is handler: query_logger.removeHandler(h) request.addfinalizer(delete)
def init(self, request, db, data): # re-init a sqlite database (no-op if the db is not sqlite): db.create(to_file=False) # setup a run_id: rdw = Download() db.session.add(rdw) db.session.commit() self.run = rdw # side effects: self._evt_urlread_sideeffect = """#EventID | Time | Latitude | Longitude | Depth/km | Author | Catalog | Contributor | ContributorID | MagType | Magnitude | MagAuthor | EventLocationName 20160508_0000129|2016-05-08 05:17:11.500000|1|1|60.0|AZER|EMSC-RTS|AZER|505483|ml|3|AZER|CASPIAN SEA, OFFSHR TURKMENISTAN 20160508_0000004|2016-05-08 01:45:30.300000|90|90|2.0|EMSC|EMSC-RTS|EMSC|505183|ml|4|EMSC|CROATIA """ self._mintraveltime_sideeffect = cycle([1]) self._seg_data = data.read("GE.FLT1..HH?.mseed") self._seg_data_gaps = data.read("IA.BAKI..BHZ.D.2016.004.head") self._seg_data_empty = b'' self._seg_urlread_sideeffect = [ self._seg_data, self._seg_data_gaps, 413, 500, self._seg_data[:2], self._seg_data_empty, 413, URLError("++urlerror++"), socket.timeout() ] self.service = '' # so get_datacenters_df accepts any row by default self.db_buf_size = 1 self.routing_service = yaml_load(get_templates_fpath("download.yaml"))\ ['advanced_settings']['routing_service_url'] # NON db stuff (logging, patchers, pandas...): self.loghandler = StreamHandler(stream=StringIO()) # THIS IS A HACK: query_logger.setLevel(logging.INFO) # necessary to forward to handlers # if we called closing (we are testing the whole chain) the level will be reset # (to level.INFO) otherwise it stays what we set two lines above. Problems might arise # if closing sets a different level, but for the moment who cares query_logger.addHandler(self.loghandler) # when debugging, I want the full dataframe with to_string(), not truncated # NOTE: this messes up right alignment of numbers in DownloadStats (see utils.py) # FIRST, remember current settings and restore them in cleanup: _pd_display_maxcolwidth = pd.get_option('display.max_colwidth') pd.set_option('display.max_colwidth', -1) # define class level patchers (we do not use a yiled as we need to do more stuff in the # finalizer, see below patchers = [] patchers.append(patch('stream2segment.utils.url.urlopen')) self.mock_urlopen = patchers[-1].start() # mock ThreadPool (tp) to run one instance at a time, so we get deterministic results: class MockThreadPool(object): def __init__(self, *a, **kw): pass def imap(self, func, iterable, *args): # make imap deterministic: same as standard python map: # everything is executed in a single thread the right input order return map(func, iterable) def imap_unordered(self, func_, iterable, *args): # make imap_unordered deterministic: same as standard python map: # everything is executed in a single thread in the right input order return map(func_, iterable) def close(self, *a, **kw): pass # assign patches and mocks: patchers.append(patch('stream2segment.utils.url.ThreadPool')) self.mock_tpool = patchers[-1].start() self.mock_tpool.side_effect = MockThreadPool # add finalizer: def delete(): pd.set_option('display.max_colwidth', _pd_display_maxcolwidth) for patcher in patchers: patcher.stop() hndls = query_logger.handlers[:] for h in hndls: if h is self.loghandler: self.loghandler.close() query_logger.removeHandler(h) request.addfinalizer(delete)