def get(self):
""" Search for stored Observations """
db = get_session(DB_URL)
if all(k in request.args for k in ("network", "station", "channel")):
r = db.query(ObservationRecord) \
.filter_by(**{k: request.args[k] for k in ("network", "station", "channel")}) \
.order_by(ObservationRecord.start) \
.all()
evt_counts = {}
for o in r:
evt_counts[o.obs_id] = \
db.query(EventRecord).filter(EventRecord.obs_id == o.obs_id).count()
return [{
"id": o.obs_id,
"network": o.network,
"station": o.station,
"channel": o.channel,
"start": o.start.timestamp(),
"end": o.end.timestamp(),
"sampling_rate": o.sampling_rate,
"filename": o.filename,
"events": evt_counts[o.obs_id]
} for o in r]
elif all(k in request.args for k in ("network", "station")):
r = db.query(ObservationRecord.channel) \
.filter_by(**{k: request.args[k] for k in ("network", "station")}) \
.distinct().all()
elif "network" in request.args:
r = db.query(ObservationRecord.station) \
.filter_by(network=request.args["network"]).distinct().all()
else:
r = db.query(ObservationRecord.network).distinct().all()
app.logger.debug(r)
# return [i for j in r for i in j] # Flatten r
return r
def post(self):
try:
if request.headers.get("Content-Type",
None) != "application/octet":
raise BadRequest("Content-Type should be application/octet")
filename = request.args.get("filename", "", type=str)
# Database connections
db = get_session(DB_URL)
ds = Datastore(MINIO_HOST, MINIO_ACCESS_KEY, MINIO_SECRET_KEY,
MINIO_BUCKET)
# Read data from request
data = BytesIO(request.data)
obs = ObservationDAO(data, filename)
obs_rec = obs.observation_record()
db.add(obs_rec)
ds.put(obs_rec.obs_id, data, data.getbuffer().nbytes)
app.logger.info("Stored {} as {}".format(obs_rec.filename,
obs_rec.obs_id))
db.commit()
return {
"status": "ok",
"obs_id": obs_rec.obs_id
}, status.HTTP_201_CREATED
except ObservationDAOError as e:
raise BadRequest(e)
except DatastoreError as e:
raise InternalServerError("Datastore Error: {}".format(e))
def get(self, obs_id):
# TODO - Exceptions
db = get_session(DB_URL)
r = db.query(EventRecord).\
filter_by(obs_id=obs_id).\
order_by(EventRecord.start).\
all()
return [{
"evt_id": e.evt_id,
"obs_id": e.obs_id,
"start": e.start.replace(tzinfo=pytz.UTC).timestamp(),
"end": e.end.replace(tzinfo=pytz.UTC).timestamp(),
"duration": int((e.end - e.start).total_seconds() * 1000)
} for e in r]
def get(self, evt_id):
""" Return SAX data for a detected Event """
try:
parser = reqparse.RequestParser()
parser.add_argument("offset", type=int, default=0)
parser.add_argument("length", type=int, required=True)
parser.add_argument("absolute", type=str)
parser.add_argument("bandpass", type=str)
parser.add_argument("bandpassLow", type=int)
parser.add_argument("bandpassHigh", type=int)
parser.add_argument("sax", type=str)
parser.add_argument("paaInt", type=int, required=True)
parser.add_argument("alphabet", type=str, required=True)
p = parser.parse_args()
ds = Datastore(MINIO_HOST, MINIO_ACCESS_KEY, MINIO_SECRET_KEY,
MINIO_BUCKET)
db = get_session(DB_URL)
evt = db.query(EventRecord).filter_by(evt_id=evt_id).one()
raw = ds.get(evt.obs_id)
obs = ObservationDAO(raw)
start = (evt.start + timedelta(milliseconds=p["offset"]))
end = (start + timedelta(milliseconds=p["length"]))
obs.slice(start=start.timestamp(), end=end.timestamp())
obs.normalise()
if is_true(p["bandpass"]):
obs.bandpass(p["bandpassLow"], p["bandpassHigh"])
if is_true(p["absolute"]):
obs.absolute()
paa = Paa(obs.series())
paa_data = paa(p["paaInt"])
paa_results = [{
"x": int(k),
"y": v
} for k, v in json.loads(paa_data.to_json(orient="index")).items()]
paa_results = sorted(paa_results, key=lambda x: x["x"])
sax = Sax(paa_data)
return {
"original": obs.view(),
"paa": paa_results,
"sax": "".join([i for i in sax(p["alphabet"])])
}
except (PaaError, SaxError) as e:
raise InternalServerError(e)
except NoResultFound:
raise NotFound
except MultipleResultsFound:
raise InternalServerError("Multiple results found, this is bad")
except DatastoreError as e:
raise InternalServerError(e)
def delete(self, obs_id):
""" Delete an Observation """
try:
ds = Datastore(MINIO_HOST, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_BUCKET)
db = get_session(DB_URL)
events_only = True if request.args.get("eventsOnly") == "true" else False
# Delete DB entry
db.query(EventRecord).filter(EventRecord.obs_id == obs_id).delete()
if not events_only:
db.query(ObservationRecord).filter(ObservationRecord.obs_id == obs_id).delete()
# Delete Object
ds.delete(obs_id)
# Commit DB if all went well
db.commit()
return "", status.HTTP_204_NO_CONTENT
except DatastoreError as e:
raise InternalServerError(e)
def get(self, obs_id):
""" Retrieve a RAW observation file """
ds = Datastore(MINIO_HOST, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_BUCKET)
db = get_session(DB_URL)
try:
obs_rec = db.query(ObservationRecord).filter_by(obs_id=obs_id).one()
filename = obs_rec.filename if len(obs_rec.filename) > 0 else obs_id
data = ds.get(obs_id)
return send_file(
data,
as_attachment=True,
attachment_filename=filename,
mimetype="application/octet"
)
except (DatastoreError, MultipleResultsFound) as e:
raise InternalServerError(e)
except NoResultFound:
raise NotFound("{} not found".format(obs_id))
def get(self, obs_id):
""" Return the trigger data for Event Detection on an Observation """
try:
ds = Datastore(MINIO_HOST, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_BUCKET)
db = get_session(DB_URL)
obs_rec = db.query(ObservationRecord).filter_by(obs_id=obs_id).one()
raw = ds.get("trigger_data/{}.json".format(obs_id))
df = pd.read_json(raw)
start_ms = obs_rec.start.timestamp() * 1000
end_ms = obs_rec.end.timestamp() * 1000
app.logger.info("Start: {}".format(start_ms))
df["t"] = df["t"].add(start_ms)
df["t"] = pd.to_datetime(df["t"], unit="ms")
downsample_freq = int((end_ms - start_ms) / 1000)
df.set_index("t", inplace=True)
downsample = df.resample("{}L".format(downsample_freq)).mean()
downsample["t"] = downsample.index.astype(np.int64) // 10 ** 6
return json.loads(downsample.to_json(orient="records"))
except DatastoreError as e:
return str(e), status.HTTP_404_NOT_FOUND
def get(self, obs_id, evt_id):
""" Retrieve a SAC file for an event """
try:
parser = reqparse.RequestParser()
parser.add_argument("offset", type=int, default=0)
parser.add_argument("length", type=int)
parser.add_argument("filename", type=str)
p = parser.parse_args()
# Get observation
ds = Datastore(MINIO_HOST, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_BUCKET)
db = get_session(DB_URL)
evt = db.query(EventRecord).filter_by(obs_id=obs_id, evt_id=evt_id).one()
obs = ObservationDAO(ds.get(obs_id))
# Slice event from observation
start = (evt.start + timedelta(milliseconds=p["offset"]))
if p["length"]:
end = (start + timedelta(milliseconds=p["length"]))
else:
end = evt.end
obs.slice(start.timestamp(), end.timestamp())
# Prepare download
filename = p["filename"] if p["filename"] else "{}.SAC".format(evt_id)
b = BytesIO()
obs.stream.write(b, format="SAC")
b.seek(0)
return send_file(
b,
as_attachment=True,
attachment_filename=filename,
mimetype="application/octet"
)
except NoResultFound:
raise NotFound
except MultipleResultsFound:
raise InternalServerError("Multiple results found, this is bad")
except DatastoreError as e:
raise InternalServerError(e)
