def queue_mosaic(batch_obs_ids, job_id, task_id, host_cluster, submission_time,
user, subband):
"""Creates a new item in the `mosaic` table to signify that a new batch
of `obs_ids` are being `swarp`ed together
"""
conn = mdb.connect()
cur = conn.cursor()
for obs_id in batch_obs_ids:
cur.execute(
"""
INSERT INTO mosaic
(obs_id, job_id, task_id, host_cluster, submission_time, user, subband, status)
VALUES
(%s, %s, %s, %s, %s, %s, %s, %s)
""",
(
obs_id,
job_id,
task_id,
host_cluster,
submission_time,
user,
subband,
"queued",
),
)
conn.commit()
conn.close()
def obtain_cen_chan(obsids):
"""Retrieve the cenchan for a set of specified pointings. This will be done through the
meta-database, but should be expanded in the future to dial home to the mwa meta-data service
Args:
obsids (iterable): collection of obsids to check
Returns:
cenchans (numpy.ndarray): the cenchan of each obsid
"""
cen_chan = np.array([1 for i in obsids])
if gxdb is None:
return cen_chan
try:
con = gxdb.connect()
obsids = [int(o) for o in obsids]
cursor = con.cursor()
# Need to wrangle the formatting so we can use the WHERE a IN b syntax
cursor.execute(
f"SELECT cenchan FROM observation WHERE obs_id IN ({', '.join(['%s' for _ in obsids])}) ",
(*obsids, ),
)
return np.squeeze(np.array([o[0] for o in cursor.fetchall()]))
except:
print("WARNING: Database lookup failed. Setting all cenchans to 1")
return cen_chan
def create_conn_cur():
"""Creates a database connection and cursor object
Returns:
{mysql.connector.Connection} -- an open connection to the database
{mysql.connector.Cursor} -- a cursor object read for work
"""
conn = mdb.connect()
cur = conn.cursor()
return conn, cur
def fail_job(job_id, task_id, host_cluster, time):
conn = mdb.connect()
cur = conn.cursor()
cur.execute(
"""UPDATE processing
SET status='failed', end_time=%s
WHERE job_id =%s AND task_id=%s and host_cluster=%s""",
(time, job_id, task_id, host_cluster),
)
conn.commit()
conn.close()
def get_all_obsids():
"""Helper function to create a single database connection to get all obsids
from the database without reinventing the whell. The database connection is
killed to ensure no funny business with the multi-processing.
Returns:
{list} -- a iterable with each element being an int
"""
conn = mdb.connect()
cur = conn.cursor()
print('Getting all obsids from the user database...')
obsids = get_obs(cur)
print('...{0} obsids found'.format(len(obsids)))
return [o[0] for o in obsids]
def fail_mosaic(job_id, task_id, host_cluster, end_time):
"""Update all rows that form a `mos_id` job that their mosaic operation has failed
"""
conn = mdb.connect()
cur = conn.cursor()
cur.execute(
"""
UPDATE mosaic
SET status='failed', end_time=%s
WHERE job_id=%s AND task_id=%s AND host_cluster=%s
""",
(end_time, job_id, task_id, host_cluster),
)
conn.commit()
conn.close()
def observation_status(obs_id, status):
"""Update the observation table to inform it that obsid has been downloaded
Args:
obs_id (int): observation id to update the status of
status (str): the status to insert for the observation
"""
conn = mdb.connect()
cur = conn.cursor()
cur.execute(
"""
UPDATE observation
SET status=%s
WHERE obs_id=%s
""",
(
status.lower(),
obs_id,
),
)
conn.commit()
conn.close()
def observation_calibrator_id(obs_id, cal_id):
"""A general update function that will modify a spectied key value for a specific observation ID
Args:
obs_id (int): observation id to update the status of
cali_id (int): observation id of the calibrator to insert
value (str):
"""
conn = mdb.connect()
cur = conn.cursor()
cur.execute(
"""
UPDATE observation
SET cal_obs_id=%s
WHERE obs_id=%s
""",
(
cal_id,
obs_id,
),
)
conn.commit()
conn.close()
def queue_job(
job_id,
task_id,
host_cluster,
submission_time,
obs_id,
user,
batch_file,
stderr,
stdout,
task,
):
conn = mdb.connect()
cur = conn.cursor()
cur.execute(
"""
INSERT INTO processing
(job_id, task_id, host_cluster, submission_time, obs_id, user, batch_file, stderr, stdout, task, status)
VALUES
( %s,%s,%s,%s,%s,%s,%s,%s,%s, %s, 'queued')
""",
(
job_id,
task_id,
host_cluster,
submission_time,
obs_id,
user,
batch_file,
stderr,
stdout,
task,
),
)
conn.commit()
conn.close()
def obsids_from_db(obsids):
"""Obtain the observation details from the gleam-x database
Args:
obsids (Iterable): List of observations ids to obtain
Returns:
pandas.DataFrame: Constructed properties based on GLEAM-X observations table
"""
if gxdb is None:
raise ValueError("GLEAM-X database module not available")
dbconn = gxdb.connect()
cursor = dbconn.cursor()
cursor.execute(
f"SELECT * FROM observation WHERE obs_id IN ({', '.join(['%s' for _ in obsids])})",
(*obsids, ),
)
columns = [c[0] for c in cursor.description]
df = pd.DataFrame(cursor.fetchall(), columns=columns)
return df
import requests
import json
import time
import sys
import os
import time
import numpy as np
import argparse
import gleam_x.db.mysql_db as mdb
__author__ = ["PaulHancock", "Natasha Hurley-Walker", "Tim Galvin"]
# Append the service name to this base URL, eg 'con', 'obs', etc.
BASEURL = 'http://ws.mwatelescope.org/metadata'
dbconn = mdb.connect()
# Function to call a JSON web service and return a dictionary: This function by Andrew Williams
# Function to call a JSON web service and return a dictionary: This function by Andrew Williams
def getmeta(service='obs', params=None, level=0):
# Validate the service name
if service.strip().lower() in ['obs', 'find', 'con']:
service = service.strip().lower()
else:
print("invalid service name: {0}".format(service))
return
service_url = "{0}/{1}".format(BASEURL, service)
try: