def config(set, sync):
"""This command should now only be used to use the command line to set
a parameter value in the data base. It used to launch the Configurator but
the recommended way to configure MSNoise is to use the "msnoise admin" web
interface."""
if set:
from ..default import default
if not set.count("="):
click.echo("!! format of the set command is name=value !!")
return
name, value = set.split("=")
if name not in default:
click.echo("!! unknown parameter %s !!" % name)
return
from ..api import connect, update_config
db = connect()
update_config(db, name, value)
db.commit()
db.close()
click.echo("Successfully updated parameter %s = %s" % (name, value))
elif sync:
import glob
from ..api import connect, get_config, get_stations, update_station,\
preload_instrument_responses
db = connect()
responses = preload_instrument_responses(db)
netsta = []
for id, row in responses.iterrows():
net, sta, loc, chan = row["channel_id"].split(".")
netsta.append("%s.%s" % (net, sta))
responses["netsta"] = netsta
for station in get_stations(db):
id = "%s.%s" % (station.net, station.sta)
coords = responses[responses["netsta"] == id]
lon = float(coords["longitude"].values[0])
lat = float(coords["latitude"].values[0])
update_station(
db,
station.net,
station.sta,
lon,
lat,
0,
"DEG",
)
logging.info("Added coordinates (%.5f %.5f) for station %s.%s" %
(lon, lat, station.net, station.sta))
db.close()
else:
from ..s001configurator import main
click.echo('Let\'s Configure MSNoise !')
main()
def config_sync():
"""
Synchronise station metadata from inventory/dataless.
"""
import glob
from ..api import connect, get_config, get_stations, update_station,\
preload_instrument_responses
db = connect()
responses = preload_instrument_responses(db)
netsta = []
for id, row in responses.iterrows():
net, sta, loc, chan = row["channel_id"].split(".")
netsta.append("%s.%s"%(net,sta))
responses["netsta"] = netsta
for station in get_stations(db):
id = "%s.%s" % (station.net, station.sta)
coords = responses[responses["netsta"] == id]
lon = float(coords["longitude"].values[0])
lat = float(coords["latitude"].values[0])
update_station(db, station.net, station.sta, lon, lat, 0, "DEG", )
logging.info("Added coordinates (%.5f %.5f) for station %s.%s" %
(lon, lat, station.net, station.sta))
db.close()
def scan_archive(ctx, init, path, recursively):
"""Scan the archive and insert into the Data Availability table."""
if path:
logging.info("Overriding workflow...")
from obspy import UTCDateTime
from ..s01scan_archive import worker
db = connect()
startdate = UTCDateTime(get_config(db, "startdate"))
enddate = UTCDateTime(get_config(db, "enddate"))
cc_sampling_rate = float(get_config(db, "cc_sampling_rate"))
db.close()
if recursively:
for root, dirs, _ in os.walk(path):
for d in dirs:
tmppath = os.path.join(root, d)
_ = os.listdir(tmppath)
if not len(_):
continue
worker(sorted(_),
tmppath,
startdate,
enddate,
cc_sampling_rate,
init=True)
worker(sorted(os.listdir(path)),
path,
startdate,
enddate,
cc_sampling_rate,
init=True)
else:
from ..s01scan_archive import main
main(init, threads=ctx.obj['MSNOISE_threads'])
def config_sync():
"""
Synchronise station metadata from inventory/dataless.
"""
import glob
from ..api import connect, get_config, get_stations, update_station,\
preload_instrument_responses
db = connect()
responses = preload_instrument_responses(db)
netsta = []
for id, row in responses.iterrows():
net, sta, loc, chan = row["channel_id"].split(".")
netsta.append("%s.%s"%(net,sta))
responses["netsta"] = netsta
for station in get_stations(db):
id = "%s.%s" % (station.net, station.sta)
coords = responses[responses["netsta"] == id]
lon = float(coords["longitude"].values[0])
lat = float(coords["latitude"].values[0])
update_station(db, station.net, station.sta, lon, lat, 0, "DEG", )
logging.info("Added coordinates (%.5f %.5f) for station %s.%s" %
(lon, lat, station.net, station.sta))
db.close()
def execute(sql_command, outfile=None, show=True):
"""EXPERT MODE: Executes 'sql_command' on the database. Use this command
at your own risk!!"""
from msnoise.api import connect
db = connect()
for cmd in sql_command.split(";"):
if not len(cmd):
continue
print("Executing '%s'" % cmd)
r = db.execute(cmd)
if cmd.count("select") or cmd.count("SELECT"):
result = r.fetchall()
if not len(result):
print("The query returned no results, sorry.")
else:
import pandas as pd
df = pd.DataFrame(result, columns=r.keys())
if show:
pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)
pd.set_option('display.max_colwidth', None)
print(df)
if outfile:
if outfile == "?":
df.to_csv("request.csv")
else:
df.to_csv("%s" % outfile)
db.commit()
db.close()
def populate(fromda):
"""Rapidly scan the archive filenames and find Network/Stations"""
if fromda:
logging.info("Overriding workflow...")
from ..msnoise_table_def import DataAvailability
from ..api import update_station
db = connect()
stations = db.query(DataAvailability.net, DataAvailability.sta). \
group_by(DataAvailability.net, DataAvailability.sta)
for net, sta in stations:
print('Adding:', net, sta)
X = 0.0
Y = 0.0
altitude = 0.0
coordinates = 'UTM'
instrument = 'N/A'
update_station(db,
net,
sta,
X,
Y,
altitude,
coordinates=coordinates,
instrument=instrument)
else:
from ..s002populate_station_table import main
main()
def main(smooth, show, outfile):
db = connect()
fig = plt.figure()
for sta1, sta2 in get_station_pairs(db):
sta1 = "%s.%s" % (sta1.net, sta1.sta)
sta2 = "%s.%s" % (sta2.net, sta2.sta)
pair = "%s_%s" % (sta1, sta2)
try:
st = read(os.path.join('SARA', 'RATIO', pair, "*"))
except:
print("No data for %s" % pair)
continue
st.merge(fill_value=np.nan)
t = pd.date_range(st[0].stats.starttime.datetime,
periods=st[0].stats.npts,
freq="%ims" % (st[0].stats.delta * 1000))
s = pd.Series(data=st[0].data, index=t, dtype=st[0].data.dtype)
s = s.resample('1T').median()
s.to_csv("%s.csv" % pair)
if smooth != 1:
s = s.rolling(window=smooth, min_periods=1, center=True).median()
plt.plot(s.index, s, lw=1.0, label='%s' % pair)
plt.ylim(0, 200)
plt.legend()
fig.autofmt_xdate()
plt.tight_layout()
if outfile:
plt.savefig(outfile)
if show:
plt.show()
def config_get(names):
"""
Display the value of the given configuration variable(s).
"""
from ..api import connect, get_config
db = connect()
show_config_values(db, names)
db.close()
def config_get(names):
"""
Display the value of the given configuration variable(s).
"""
from ..api import connect, get_config
db = connect()
show_config_values(db, names)
db.close()
def execute(sql_command):
"""EXPERT MODE: Executes 'sql_command' on the database. Use this command
at your own risk!!"""
from msnoise.api import connect
db = connect()
r = db.execute(sql_command)
if sql_command.count("select") or sql_command.count("SELECT"):
print(r.fetchall())
db.commit()
db.close()
def main(per, a1, b1, l1, s1, a2, b2, l2, s2, filterid, comp, show):
# Smoothing and damping parameters
db = connect()
alpha1 = a1 if a1 else float(get_config(db, "alpha1", plugin="Tomo"))
beta1 = b1 if b1 else float(get_config(db, "beta1", plugin="Tomo"))
lambda1 = l1 if l1 else float(get_config(db, "lambda1", plugin="Tomo"))
sigma1 = s1 if s1 else float(get_config(db, "sigma1", plugin="Tomo"))
alpha2 = a2 if a2 else float(get_config(db, "alpha2", plugin="Tomo"))
beta2 = b2 if b2 else float(get_config(db, "beta2", plugin="Tomo"))
lambda2 = l2 if l2 else float(get_config(db, "lambda2", plugin="Tomo"))
sigma2 = s2 if s2 else float(get_config(db, "sigma2", plugin="Tomo"))
v_cmap = get_config(db, "v_cmap", plugin="Tomo")
d_cmap = get_config(db, "d_cmap", plugin="Tomo")
if per is None:
PER = get_config(db, "ftan_periods", plugin="Tomo")
periods = np.array([float(pi) for pi in PER.split(',')])
else:
periods = [
float(per),
]
# ANSWT inputs
gridfile = os.path.join("TOMO_FILES", "%02i" % filterid, comp, "Grid.dat")
stacoordfile = os.path.join("TOMO_FILES", "%02i" % filterid, comp,
"STACoord.dat")
for per in periods:
DCfile = os.path.join("TOMO_FILES", "%02i" % filterid, comp,
"TestGroupVel_%.3fs.dat" % float(per))
PERIOD = per
paramfile = os.path.join("TOMO_FILES", "%02i" % filterid, comp,
'ParamFile.txt')
fid = open(paramfile, 'w')
fid.write(
'%% alpha1 \t beta1 \t lambda1 \t Lcorr1 \t alpha2 \t beta2 \t lambda2 \t Lcorr2\n'
)
fid.write(
'%f %f %f %f %f %f %f %f\n' %
(alpha1, beta1, lambda1, sigma1, alpha2, beta2, lambda2, sigma2))
fid.close()
try:
ANSWT(gridfile, stacoordfile, DCfile, paramfile, PERIOD, show,
v_cmap, d_cmap)
except:
traceback.print_exc()
print("!" * 80)
print("Can't compute tomo for period=", per)
print("!" * 80)
def info():
from msnoise.api import connect, get_config
from .default import default
db = connect()
click.echo('')
click.echo('Raw config bits: "D"efault or "M"odified (green)')
for key in default.keys():
tmp = get_config(db, key, plugin='Tomo')
if tmp == default[key][1]:
click.secho(" D %s: %s" % (key, tmp))
else:
click.secho(" M %s: %s" % (key, tmp), fg='green')
def clean_duplicates():
"""Checks the Jobs table and deletes duplicate entries"""
from msnoise.api import connect, get_tech
db = connect()
if get_tech() == 1:
query = "DELETE FROM jobs WHERE rowid NOT IN (SELECT MIN(rowid) FROM jobs GROUP BY day,pair,jobtype)"
else:
query = "DELETE from jobs USING jobs, jobs as vtable WHERE (jobs.ref > vtable.ref) AND (jobs.day=vtable.day) AND (jobs.pair=vtable.pair) AND (jobs.jobtype=vtable.jobtype)"
db.execute(query)
db.commit()
db.close()
def execute(sql_command):
"""EXPERT MODE: Executes 'sql_command' on the database. Use this command
at your own risk!!"""
from msnoise.api import connect, get_tech
db = connect()
r = db.execute(sql_command)
if sql_command.count("select") or sql_command.count("SELECT"):
print(r.fetchall())
db.commit()
db.close()
def reset(jobtype, all, rule):
"""Resets the job to "T"odo. ARG is [CC] or [DTT]. By default
only resets jobs "I"n progress. --all resets all jobs, whatever
the flag value"""
from ..api import connect, reset_jobs
session = connect()
if jobtype == "DA":
session.execute("update data_availability set flag='M' where 1")
elif jobtype != jobtype.upper():
logging.info("The jobtype %s is not uppercase (usually jobtypes"
" are uppercase...)" % jobtype)
reset_jobs(session, jobtype, all, rule)
session.close()
def new_jobs(init, nocc, hpc=""):
"""Determines if new CC jobs are to be defined"""
if not hpc:
from ..s02new_jobs import main
main(init, nocc)
if hpc:
from ..api import connect
left, right = hpc.split(":")
db = connect()
db.execute(
"INSERT INTO jobs (pair, day, jobtype, flag) SELECT pair, day, '%s', 'T' from jobs where jobtype='%s' and flag='D';"
% (right, left))
db.commit()
db.close()
def main():
db = connect()
while is_next_job(db, jobtype='AMAZ1'):
jobs = get_next_job(db, jobtype='AMAZ1')
for job in jobs:
net, sta = job.pair.split('.')
gd = UTCDateTime(job.day).datetime
print("Processing %s.%s for day %s"%(net,sta, job.day))
files = get_data_availability(
db, net=net, sta=sta, starttime=gd, endtime=gd,
comp="Z")
for file in files:
fn = os.path.join(file.path, file.file)
st = read(fn, starttime=UTCDateTime(job.day), endtime=UTCDateTime(job.day)+86400)
print st
def upgrade():
"""Upgrade the database from previous to a new version.\n
This procedure adds new parameters with their default value
in the config database.
"""
from ..api import connect, Config, read_database_inifile
from ..default import default
db = connect()
tech, hostname, database, username, password, prefix = \
read_database_inifile()
if prefix != "":
prefix = prefix + "_"
for name in default.keys():
try:
db.add(Config(name=name, value=default[name][-1]))
db.commit()
except:
db.rollback()
# print("Passing %s: already in DB" % name)
continue
try:
db.execute("CREATE UNIQUE INDEX job_index ON %sjobs (day, pair, "
"jobtype)" % prefix)
db.commit()
except:
logging.info("It looks like the v1.5 'job_index' is already in the DB")
traceback.print_exc()
db.rollback()
try:
db.execute("CREATE INDEX job_index2 ON %sjobs (jobtype, flag)" %
prefix)
db.commit()
except:
logging.info(
"It looks like the v1.6 'job_index2' is already in the DB")
db.rollback()
try:
db.execute(
"CREATE UNIQUE INDEX da_index ON %sdata_availability (path, "
"file, net, sta, comp)" % prefix)
db.commit()
except:
logging.info("It looks like the v1.5 'da_index' is already in the DB")
db.rollback()
db.close()
def reset(jobtype, all, rule):
"""Resets the job to "T"odo. ARG is [CC] or [DTT]. By default
only resets jobs "I"n progress. --all resets all jobs, whatever
the flag value"""
from ..api import connect, reset_jobs, read_db_inifile
dbini = read_db_inifile()
prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
session = connect()
if jobtype == "DA":
session.execute("UPDATE {0}data_availability SET flag='M'"
.format(prefix))
elif jobtype != jobtype.upper():
logging.info("The jobtype %s is not uppercase (usually jobtypes"
" are uppercase...)"%jobtype)
reset_jobs(session, jobtype, all, rule)
session.close()
def reset(jobtype, all, rule):
"""Resets the job to "T"odo. ARG is [CC] or [DTT]. By default
only resets jobs "I"n progress. --all resets all jobs, whatever
the flag value"""
from ..api import connect, reset_jobs, read_db_inifile
dbini = read_db_inifile()
prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
session = connect()
if jobtype == "DA":
session.execute("UPDATE {0}data_availability SET flag='M'"
.format(prefix))
elif jobtype != jobtype.upper():
logging.info("The jobtype %s is not uppercase (usually jobtypes"
" are uppercase...)"%jobtype)
reset_jobs(session, jobtype, all, rule)
session.close()
def new_jobs(init, nocc, hpc=""):
"""Determines if new CC jobs are to be defined"""
if not hpc:
from ..s02new_jobs import main
main(init, nocc)
if hpc:
from ..api import connect, read_db_inifile
dbini = read_db_inifile()
prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
left, right = hpc.split(':')
db = connect()
db.execute("INSERT INTO {prefix}jobs (pair, day, jobtype, flag) "
"SELECT pair, day, '{right_type}', 'T' FROM {prefix}jobs "
"WHERE jobtype='{left_type}' AND flag='D';"
.format(prefix=prefix, right_type=right, left_type=left))
db.commit()
db.close()
def new_jobs(init, nocc, hpc=""):
"""Determines if new CC jobs are to be defined"""
if not hpc:
from ..s02new_jobs import main
main(init, nocc)
if hpc:
from ..api import connect, read_db_inifile
dbini = read_db_inifile()
prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
left, right = hpc.split(':')
db = connect()
db.execute("INSERT INTO {prefix}jobs (pair, day, jobtype, flag) "
"SELECT pair, day, '{right_type}', 'T' FROM {prefix}jobs "
"WHERE jobtype='{left_type}' AND flag='D';"
.format(prefix=prefix, right_type=right, left_type=left))
db.commit()
db.close()
def reset(jobtype, all, rule):
"""Resets the job to "T"odo. JOBTYPE is the acronym of the job type.
By default only resets jobs "I"n progress. --all resets all jobs, whatever
the flag value. Standard Job Types are CC, STACK, MWCS and DTT, but
plugins can define their own."""
from ..api import connect, reset_jobs, read_db_inifile
dbini = read_db_inifile()
prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
session = connect()
if jobtype == "DA":
session.execute("UPDATE {0}data_availability SET flag='M'"
.format(prefix))
elif jobtype != jobtype.upper():
logging.info("The jobtype %s is not uppercase (usually jobtypes"
" are uppercase...)"%jobtype)
reset_jobs(session, jobtype, all, rule)
session.close()
def upgrade():
"""Upgrade the database from previous to a new version.\n
This procedure adds new parameters with their default value
in the config database.
"""
from ..api import connect, Config, read_db_inifile
from ..default import default
db = connect()
dbini = read_db_inifile()
prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
for name in default.keys():
try:
db.add(Config(name=name, value=default[name].default))
db.commit()
except:
db.rollback()
# print("Passing %s: already in DB" % name)
continue
try:
db.execute("CREATE UNIQUE INDEX job_index ON %sjobs (day, pair, "
"jobtype)" %
prefix)
db.commit()
except:
logging.info("It looks like the v1.5 'job_index' is already in the DB")
db.rollback()
try:
db.execute("CREATE INDEX job_index2 ON %sjobs (jobtype, flag)" %
prefix)
db.commit()
except:
logging.info("It looks like the v1.6 'job_index2' is already in the DB")
db.rollback()
try:
db.execute("CREATE UNIQUE INDEX da_index ON %sdata_availability (path, "
"file, net, sta, loc, chan)" %
prefix)
db.commit()
except:
logging.info("It looks like the v1.5 'da_index' is already in the DB")
db.rollback()
db.close()
def upgrade():
"""Upgrade the database from previous to a new version.\n
This procedure adds new parameters with their default value
in the config database.
"""
from ..api import connect, Config, read_db_inifile
from ..default import default
db = connect()
dbini = read_db_inifile()
prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
for name in default.keys():
try:
db.add(Config(name=name, value=default[name][1]))
db.commit()
except:
db.rollback()
# print("Passing %s: already in DB" % name)
continue
try:
db.execute("CREATE UNIQUE INDEX job_index ON %sjobs (day, pair, "
"jobtype)" %
prefix)
db.commit()
except:
logging.info("It looks like the v1.5 'job_index' is already in the DB")
db.rollback()
try:
db.execute("CREATE INDEX job_index2 ON %sjobs (jobtype, flag)" %
prefix)
db.commit()
except:
logging.info("It looks like the v1.6 'job_index2' is already in the DB")
db.rollback()
try:
db.execute("CREATE UNIQUE INDEX da_index ON %sdata_availability (path, "
"file, net, sta, comp)" %
prefix)
db.commit()
except:
logging.info("It looks like the v1.5 'da_index' is already in the DB")
db.rollback()
db.close()
def execute(sql_command):
"""EXPERT MODE: Executes 'sql_command' on the database. Use this command
at your own risk!!"""
from msnoise.api import connect
db = connect()
r = db.execute(sql_command)
if sql_command.count("select") or sql_command.count("SELECT"):
result = r.fetchall()
if not len(result):
print("The query returned no results, sorry.")
else:
import pandas as pd
df = pd.DataFrame(result)
print(df)
db.commit()
db.close()
def info(jobs):
"""Outputs general information about the current install and config, plus
information about jobs and their status."""
from ..api import connect
if not os.path.isfile('db.ini'):
click.secho(' - db.ini is not present, is MSNoise installed here ?',
fg='red')
return
db = connect()
if not jobs:
info_folders(db)
info_parameters(db)
info_stations(db)
info_jobs(db)
info_plugins(db)
db.close()
def info(jobs):
"""Outputs general information about the current install and config, plus
information about jobs and their status."""
from ..api import connect
if not os.path.isfile('db.ini'):
click.secho(' - db.ini is not present, is MSNoise installed here ?',
fg='red')
return
db = connect()
if not jobs:
info_folders(db)
info_parameters(db)
info_stations(db)
info_jobs(db)
info_plugins(db)
db.close()
def clean_duplicates():
"""Checks the Jobs table and deletes duplicate entries"""
from msnoise.api import connect, read_db_inifile
dbini = read_db_inifile()
prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
db = connect()
if dbini.tech == 1:
query = 'DELETE FROM {0}jobs WHERE rowid NOT IN '\
'(SELECT MIN(rowid) FROM {0}jobs GROUP BY day,pair,jobtype)'\
.format(prefix)
else:
query = 'DELETE from {0}jobs USING {0}jobs as j1, {0}jobs as j2 '\
'WHERE (j1.ref > j2.ref) AND (j1.day=j2.day) '\
'AND (j1.pair=j2.pair) AND (j1.jobtype=j2.jobtype)'\
.format(prefix)
db.execute(query)
db.commit()
db.close()
def config_set(name_value):
"""
Set a configuration value. The argument should be of the form
'variable=value'.
"""
from ..default import default
if not name_value.count("="):
click.echo("!! format of the set command is name=value !!")
return
name, value = name_value.split("=")
if name not in default:
click.echo("!! unknown parameter %s !!" % name)
return
from ..api import connect, update_config
db = connect()
update_config(db, name, value)
db.commit()
db.close()
click.echo("Successfully updated parameter %s = %s" % (name, value))
def clean_duplicates():
"""Checks the Jobs table and deletes duplicate entries"""
from msnoise.api import connect, read_db_inifile
dbini = read_db_inifile()
prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
db = connect()
if dbini.tech == 1:
query = 'DELETE FROM {0}jobs WHERE rowid NOT IN '\
'(SELECT MIN(rowid) FROM {0}jobs GROUP BY day,pair,jobtype)'\
.format(prefix)
else:
query = 'DELETE from {0}jobs USING {0}jobs as j1, {0}jobs as j2 '\
'WHERE (j1.ref > j2.ref) AND (j1.day=j2.day) '\
'AND (j1.pair=j2.pair) AND (j1.jobtype=j2.jobtype)'\
.format(prefix)
db.execute(query)
db.commit()
db.close()
def config_set(name_value):
"""
Set a configuration value. The argument should be of the form
'variable=value'.
"""
from ..default import default
if not name_value.count("="):
click.echo("!! format of the set command is name=value !!")
return
name, value = name_value.split("=")
if name not in default:
click.echo("!! unknown parameter %s !!" % name)
return
from ..api import connect, update_config
db = connect()
update_config(db, name, value)
db.commit()
db.close()
click.echo("Successfully updated parameter %s = %s" % (name, value))
def populate(fromda):
"""Rapidly scan the archive filenames and find Network/Stations"""
if fromda:
logging.info("Overriding workflow...")
db = connect()
stations = db.query(DataAvailability.net, DataAvailability.sta). \
group_by(DataAvailability.net, DataAvailability.sta)
for net, sta in stations:
print('Adding:', net, sta)
X = 0.0
Y = 0.0
altitude = 0.0
coordinates = 'UTM'
instrument = 'N/A'
update_station(db, net, sta, X, Y, altitude,
coordinates=coordinates, instrument=instrument)
else:
from ..s002populate_station_table import main
main()
def da_stations_update_loc_chan():
"""EXPERT MODE: Populates the Location & Channel from the Data Availability
table. Warning: rewrites automatically, no confirmation."""
from msnoise.api import connect, get_stations
session = connect()
stations = get_stations(session)
for sta in stations:
data = session.query(DataAvailability). \
filter(text("net=:net")). \
filter(text("sta=:sta")). \
group_by(DataAvailability.net, DataAvailability.sta,
DataAvailability.loc, DataAvailability.chan). \
params(net=sta.net, sta=sta.sta).all()
locids = sorted([d.loc for d in data])
chans = sorted([d.chan for d in data])
print("%s.%s has locids:%s and chans:%s" % (sta.net, sta.sta,
locids, chans))
sta.used_location_codes = ",".join(locids)
sta.used_channel_names = ",".join(chans)
session.commit()
def info(jobs):
"""Outputs general information about the current install and config, plus
information about jobs and their status."""
from ..api import connect, get_config, get_job_types, get_filters, \
get_stations
from ..default import default
def d(path):
return os.path.split(path)[0]
if not os.path.isfile('db.ini'):
click.secho(' - db.ini is not present, is MSNoise installed here ?',
fg='red')
return
db = connect()
if not jobs:
click.echo('')
click.echo('General:')
click.echo('MSNoise is installed in: %s' %
d(d(d(os.path.abspath(__file__)))))
if os.path.isfile('db.ini'):
click.echo(' - db.ini is present')
else:
click.secho(
' - db.ini is not present, is MSNoise installed here ?',
fg='red')
return
click.echo('')
click.echo('')
click.echo('Configuration:')
data_folder = get_config(db, "data_folder")
if os.path.isdir(data_folder):
click.echo(" - %s exists" % data_folder)
else:
click.secho(" - %s does not exists !" % data_folder, fg='red')
output_folder = get_config(db, "output_folder")
if os.path.isdir(output_folder):
click.echo(" - %s exists" % output_folder)
else:
if get_config(db, 'keep_all') in ['Y', 'y']:
for job in get_job_types(db):
if job[1] == 'D':
if job[0] > 0:
click.secho(
" - %s does not exists and that is not normal"
" (%i CC jobs done)" % (output_folder, job[0]),
fg='red')
else:
click.secho(
" - %s does not exists and that is normal"
" (%i CC jobs done)" % (output_folder, job[0]))
else:
click.secho(" - %s does not exists (and that is normal because"
" keep_all=False)" % output_folder)
click.echo('')
click.echo('Raw config bits: "D"efault or "M"odified (green)')
for key in default.keys():
tmp = get_config(db, key)
if tmp == default[key][1]:
click.secho(" D %s: %s" % (key, tmp))
else:
click.secho(" M %s: %s" % (key, tmp), fg='green')
# TODO add plugins params
click.echo('')
click.echo('Filters:')
print('ID: [low:high] [mwcs_low:mwcs_high] mwcs_wlen mwcs_step'
' used')
for f in get_filters(db, all=True):
data = (f.ref, f.low, f.high, f.mwcs_low, f.mwcs_high, f.mwcs_wlen,
f.mwcs_step, ['N', 'Y'][f.used])
print('%02i: [%.03f:%.03f] [%.03f:%.03f] %.03i %.03i %s' % data)
click.echo('')
click.echo('Stations:')
for s in get_stations(db, all=True):
data = (s.net, s.sta, s.X, s.Y, s.altitude, s.coordinates,
['N', 'Y'][s.used])
print('%s.%s %.4f %.4f %.1f %s %s' % data)
click.echo("MSNoise")
for jobtype in ["CC", "STACK", "MWCS", "DTT"]:
click.echo(' %s:' % jobtype)
for (n, jobtype) in get_job_types(db, jobtype):
click.echo(" %s : %i" % (jobtype, n))
plugins = get_config(db, "plugins")
if plugins:
plugins = plugins.split(",")
for ep in pkg_resources.iter_entry_points(
group='msnoise.plugins.jobtypes'):
module_name = ep.module_name.split(".")[0]
if module_name in plugins:
click.echo('')
click.echo('Plugin: %s' % module_name)
for row in ep.load()():
click.echo(' %s:' % row["name"])
for (n, jobtype) in get_job_types(db, row["name"]):
click.echo(" %s : %i" % (jobtype, n))
def main(sta1, sta2, filterid, components, mov_stack=1, ampli=5, show=False,
outfile=False, refilter=None, startdate=None, enddate=None):
db = connect()
cc_sampling_rate = float(get_config(db, 'cc_sampling_rate'))
start, end, datelist = build_movstack_datelist(db)
base = mdates.date2num(start)
sta1 = sta1.replace('.', '_')
sta2 = sta2.replace('.', '_')
# TODO: Height adjustment of the plot for large number of stacks.
# Preferably interactive
fig = plt.figure(figsize=(12, 9))
if refilter:
freqmin, freqmax = refilter.split(':')
freqmin = float(freqmin)
freqmax = float(freqmax)
if sta2 >= sta1:
pair = "%s:%s" % (sta1, sta2)
print("New Data for %s-%s-%i-%i" % (pair, components, filterid,
mov_stack))
nstack, stack_total = get_results(db, sta1, sta2, filterid, components,
datelist, mov_stack, format="matrix")
ax = fig.add_subplot(111)
for i, line in enumerate(stack_total):
if np.all(np.isnan(line)):
continue
if refilter:
line = bandpass(line, freqmin, freqmax, cc_sampling_rate,
zerophase=True)
freq, line = prepare_abs_postitive_fft(line, cc_sampling_rate)
line /= line.max()
ax.plot(freq, line * ampli + i + base, c='k')
for filterdb in get_filters(db, all=True):
if filterid == filterdb.ref:
low = float(filterdb.low)
high = float(filterdb.high)
break
ax.set_ylim(start-datetime.timedelta(days=ampli),
end+datetime.timedelta(days=ampli))
ax.yaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
ax.set_xlabel("Frequency [Hz]")
ax.set_xscale('log')
ax.grid()
title = '%s : %s, %s, Filter %d (%.2f - %.2f Hz), Stack %d' %\
(sta1.replace('_', '.'), sta2.replace('_', '.'), components,
filterid, low, high, mov_stack)
if refilter:
title += ", Re-filtered (%.2f - %.2f Hz)" % (freqmin, freqmax)
ax.set_title(title)
cursor = Cursor(ax, useblit=True, color='red', linewidth=1.2)
print(outfile)
if outfile:
if outfile.startswith("?"):
pair = pair.replace(':', '-')
outfile = outfile.replace('?', '%s-%s-f%i-m%i' % (pair,
components,
filterid,
mov_stack))
outfile = "spectime" + outfile
print("output to:", outfile)
plt.savefig(outfile)
if show:
plt.show()
else:
plt.close(fig)
def main(sta1, sta2, filterid, components, mov_stack=1, ampli=5, show=False,
outfile=False, refilter=None, startdate=None, enddate=None, **kwargs):
db = connect()
cc_sampling_rate = float(get_config(db, 'cc_sampling_rate'))
start, end, datelist = build_movstack_datelist(db)
base = mdates.date2num(start)
sta1 = sta1.replace('.', '_')
sta2 = sta2.replace('.', '_')
# TODO: Height adjustment of the plot for large number of stacks.
# Preferably interactive
fig = plt.figure(figsize=(12, 9))
if refilter:
freqmin, freqmax = refilter.split(':')
freqmin = float(freqmin)
freqmax = float(freqmax)
if sta2 >= sta1:
pair = "%s:%s" % (sta1, sta2)
print("New Data for %s-%s-%i-%i" % (pair, components, filterid,
mov_stack))
nstack, stack_total = get_results(db, sta1, sta2, filterid, components,
datelist, mov_stack, format="matrix")
ax = fig.add_subplot(111)
for i, line in enumerate(stack_total):
if np.all(np.isnan(line)):
continue
if refilter:
line = bandpass(line, freqmin, freqmax, cc_sampling_rate,
zerophase=True)
freq, line = prepare_abs_postitive_fft(line, cc_sampling_rate)
line /= line.max()
ax.plot(freq, line * ampli + i + base, c='k')
for filterdb in get_filters(db, all=True):
if filterid == filterdb.ref:
low = float(filterdb.low)
high = float(filterdb.high)
break
ax.set_ylim(start-datetime.timedelta(days=ampli),
end+datetime.timedelta(days=ampli))
ax.yaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
if "xlim" in kwargs:
plt.xlim(kwargs["xlim"][0],kwargs["xlim"][1])
ax.set_xlabel("Frequency [Hz]")
ax.set_xscale('log')
ax.grid()
title = '%s : %s, %s, Filter %d (%.2f - %.2f Hz), Stack %d' %\
(sta1.replace('_', '.'), sta2.replace('_', '.'), components,
filterid, low, high, mov_stack)
if refilter:
title += ", Re-filtered (%.2f - %.2f Hz)" % (freqmin, freqmax)
ax.set_title(title)
cursor = Cursor(ax, useblit=True, color='red', linewidth=1.2)
print(outfile)
if outfile:
if outfile.startswith("?"):
pair = pair.replace(':', '-')
outfile = outfile.replace('?', '%s-%s-f%i-m%i' % (pair,
components,
filterid,
mov_stack))
outfile = "spectime" + outfile
print("output to:", outfile)
plt.savefig(outfile)
if show:
plt.show()
else:
plt.close(fig)
cli.add_command(stack)
cli.add_command(compute_mwcs)
cli.add_command(compute_stretching)
cli.add_command(compute_dtt)
cli.add_command(compute_dvv)
cli.add_command(reset)
cli.add_command(db)
cli.add_command(jupyter)
cli.add_command(test)
# Finally add the plot group too:
cli.add_command(plot)
try:
from ..api import connect, get_config
db = connect()
plugins = get_config(db, "plugins")
db.close()
except:
plugins = None
if plugins:
plugins = plugins.split(",")
for ep in pkg_resources.iter_entry_points(
group='msnoise.plugins.commands'):
module_name = ep.module_name.split(".")[0]
if module_name in plugins:
plugin.add_command(ep.load())
p.add_command(ep.load())
cli.add_command(plugin)
STA1 and STA2 must be provided with this format: NET.STA !\n
DAY must be provided in the ISO format: YYYY-MM-DD"""
if sta1 > sta2:
click.echo("Stations STA1 and STA2 must be sorted alphabetically.")
return
if ctx.obj['MSNOISE_custom']:
from dtt import main
else:
from ..plots.dtt import main
main(sta1, sta2, filterid, comp, day, mov_stack, show, outfile)
## Main script
try:
db = connect()
plugins = get_config(db, "plugins")
db.close()
except DBConfigNotFoundError:
plugins = None
except sqlalchemy.exc.OperationalError as e:
logging.critical('Unable to read project configuration: error connecting to the database:\n{}'.format(str(e)))
sys.exit(1)
if plugins:
plugins = plugins.split(",")
for ep in pkg_resources.iter_entry_points(group='msnoise.plugins.commands'):
module_name = ep.module_name.split(".")[0]
if module_name in plugins:
plugin.add_command(ep.load())
p.add_command(ep.load())
def main(sta1,
sta2,
filterid,
components,
mov_stack=1,
ampli=5,
seismic=False,
show=False,
outfile=None,
envelope=False,
refilter=None,
startdate=None,
enddate=None):
db = connect()
maxlag = float(get_config(db, 'maxlag'))
samples = get_maxlag_samples(db)
cc_sampling_rate = float(get_config(db, 'cc_sampling_rate'))
start, end, datelist = build_movstack_datelist(db, startdate, enddate)
base = mdates.date2num(start)
fig = plt.figure(figsize=(12, 9))
sta1 = sta1.replace('.', '_')
sta2 = sta2.replace('.', '_')
t = np.arange(samples) / cc_sampling_rate - maxlag
if refilter:
freqmin, freqmax = refilter.split(':')
freqmin = float(freqmin)
freqmax = float(freqmax)
if sta2 >= sta1:
pair = "%s:%s" % (sta1, sta2)
print("New Data for %s-%s-%i-%i" %
(pair, components, filterid, mov_stack))
nstack, stack_total = get_results(db,
sta1,
sta2,
filterid,
components,
datelist,
mov_stack,
format="matrix")
ax = fig.add_subplot(111)
for i, line in enumerate(stack_total):
if np.all(np.isnan(line)):
continue
if refilter:
line = bandpass(line,
freqmin,
freqmax,
cc_sampling_rate,
zerophase=True)
if envelope:
line = obspy_envelope(line)
line /= line.max()
ax.plot(t, line * ampli + i + base, c='k')
if seismic:
y1 = np.ones(len(line)) * i
y2 = line * ampli + i + base
ax.fill_between(t,
y1,
y2,
where=y2 >= y1,
facecolor='k',
interpolate=True)
for filterdb in get_filters(db, all=True):
if filterid == filterdb.ref:
low = float(filterdb.low)
high = float(filterdb.high)
break
ax.set_xlabel("Lag Time (s)")
ax.axhline(0, lw=0.5, c='k')
ax.grid()
ax.scatter(0, [
start,
], alpha=0)
ax.set_ylim(start - datetime.timedelta(days=ampli),
end + datetime.timedelta(days=ampli))
ax.set_xlim(-maxlag, maxlag)
ax.fmt_ydata = mdates.DateFormatter('%Y-%m-%d')
cursor = Cursor(ax, useblit=True, color='red', linewidth=1.2)
title = '%s : %s, %s, Filter %d (%.2f - %.2f Hz), Stack %d' % \
(sta1.replace('_', '.'), sta2.replace('_', '.'), components,
filterid, low, high, mov_stack)
if refilter:
title += ", Re-filtered (%.2f - %.2f Hz)" % (freqmin, freqmax)
ax.set_title(title)
if outfile:
if outfile.startswith("?"):
pair = pair.replace(':', '-')
outfile = outfile.replace(
'?',
'%s-%s-f%i-m%i' % (pair, components, filterid, mov_stack))
outfile = "ccftime " + outfile
print("output to:", outfile)
fig.savefig(outfile)
if show:
fig.show()
else:
plt.close(fig)
def main(loglevel="INFO", njobs_per_worker=9999):
logger = logbook.Logger("msnoise")
# Reconfigure logger to show the pid number in log records
logger = get_logger('msnoise.compute_psd_child', loglevel, with_pid=True)
logger.info('*** Starting: Compute PPSD ***')
db = connect()
logger.debug('Preloading all instrument response')
responses = preload_instrument_responses(db, return_format="inventory")
params = get_params(db)
ppsd_components = params.qc_components
ppsd_length = params.qc_ppsd_length
ppsd_overlap = params.qc_ppsd_overlap
ppsd_period_smoothing_width_octaves = params.qc_ppsd_period_smoothing_width_octaves
ppsd_period_step_octaves = params.qc_ppsd_period_step_octaves
ppsd_period_limits = params.qc_ppsd_period_limits
ppsd_db_bins = params.qc_ppsd_db_bins
while is_next_job(db, jobtype='PSD'):
logger.info("Getting the next job")
jobs = get_next_job(db, jobtype='PSD', limit=njobs_per_worker)
logger.debug("I will process %i jobs" % len(jobs))
if len(jobs) == 0:
# edge case, should only occur when is_next returns true, but
# get_next receives no jobs (heavily parallelised code)
continue
for job in jobs:
net, sta, loc = job.pair.split('.')
print("Processing %s" % job.pair)
gd = UTCDateTime(job.day).datetime
files = get_data_availability(
db,
net=net,
sta=sta,
loc=loc,
starttime=(UTCDateTime(job.day) - 1.5 * ppsd_length).datetime,
endtime=gd)
if len(files) == 0:
print("No files found for %s" % job.day)
continue
for comp in ppsd_components:
toprocess = []
for file in files:
if file.chan[-1] != comp:
continue
tmp = os.path.join(file.path, file.file)
toprocess.append(tmp)
if len(toprocess) == 0:
continue
st = Stream()
for tmp in np.unique(toprocess):
logger.debug("Reading %s" % tmp)
try:
st += read(
tmp,
starttime=UTCDateTime(gd) - 1.5 * ppsd_length,
endtime=UTCDateTime(gd +
datetime.timedelta(days=1)) -
0.001)
except:
logger.debug("Problem loading %s" % tmp)
if not len(st):
continue
try:
st.merge()
except:
logger.info("Failed merging streams:")
traceback.print_exc()
continue
st = st.split()
for tr in st:
tr.stats.network = tr.stats.network.upper()
tr.stats.station = tr.stats.station.upper()
tr.stats.channel = tr.stats.channel.upper()
tr = st.select(component=comp)[0]
out = to_sds(tr.stats, gd.year, int(gd.strftime('%j')))
npzdout = os.path.join("PSD", "NPZ", out)
logger.debug("ppsd will be output to: %s" % npzdout)
ppsd = PPSD(tr.stats,
metadata=responses,
ppsd_length=ppsd_length,
overlap=ppsd_overlap,
period_smoothing_width_octaves=
ppsd_period_smoothing_width_octaves,
period_step_octaves=ppsd_period_step_octaves,
period_limits=ppsd_period_limits,
db_bins=ppsd_db_bins)
# TODO handle when the response for this trace is not in the inv
ppsd.add(st)
out = to_sds(tr.stats, gd.year, int(gd.strftime('%j')))
pngout = os.path.join("PSD", "PNG", out)
if not os.path.isdir(os.path.split(npzdout)[0]):
os.makedirs(os.path.split(npzdout)[0])
os.makedirs(os.path.split(pngout)[0])
ppsd.save_npz(npzdout + ".npz")
update_job(db, job.day, job.pair, 'PSD', 'D', ref=job.ref)
if not params.hpc:
for job in jobs:
update_job(db, job.day, job.pair, 'PSD2HDF', 'T')
try:
ppsd.plot(pngout + ".png")
except:
logger.debug("Error saving PNG image")
traceback.print_exc()
del ppsd
logger.debug('Day (job) "D"one')
