def build_master_table(input_table,
output_table,
ftype="fitacf",
coords="mlt",
config_filename="../mysql_dbconfig_files/config.ini",
section="midlat",
input_dbname=None,
output_dbname=None):
""" combines all the ten-min median filtered gridded iscat data into
one master table.
The results are stored in a different db file.
Parameters
----------
input_table : str
A table name from input_dbname
output_table : str
A table name from output_dbname
ftype : str
SuperDARN file type
coords : str
Coordinates in which the binning process took place.
Default to "mlt, can be "geo" as well.
config_filename: str
name and path of the configuration file
section: str, default to "midlat"
section of database configuration
input_dbname : str, default to None
Name of the MySQL db where ten-min median data are stored.
output_dbname : str, default to None
Name of the master db
Returns
-------
Nothing
"""
import numpy as np
import datetime as dt
from mysql.connector import MySQLConnection
from month_to_season import get_season_by_month
import sys
sys.path.append("../")
from mysql_dbutils.db_config import db_config
from mysql_dbutils import db_tools
import logging
# create db name
if input_dbname is None:
input_dbname = "ten_min_median_" + coords + "_" + ftype
if output_dbname is None:
output_dbname = "master_" + coords + "_" + ftype
# create a db (if not exist) that combines all the data
try:
# create a db
db_tools.create_db(output_dbname)
except Exception, e:
logging.error(e, exc_info=True)
def main():
""" Call the functions above. Acts as an example code.
Multiprocessing has been implemented to do parallel computing.
The unit process is for reading a day worth of data for a given radar
NOTE: NO need to worry about the sd data in tmp dir because they will
be removed after using them.
"""
import datetime as dt
import multiprocessing as mp
import os
import sys
sys.path.append("../")
from mysql_dbutils import db_tools, db_config
import logging
from mysql.connector import MySQLConnection
# create a log file to which any error occured between client and
# MySQL server communication will be written
logging.basicConfig(filename="./log_files/boxcar_filtered_data_to_db_six_rads.log",
level=logging.INFO)
# input parameters
sdate = dt.datetime(2017, 1, 1) # includes sdate
# sdate = dt.datetime(2016, 6, 21) # includes sdate
# edate = dt.datetime(2015, 1, 1) # does not include edate
edate = dt.datetime(2018, 7, 1) # does not include edate
channel = None
params=['velocity']
ftype = "fitacf"
ffname = None
# run the code for the following radars in parallel
#rad_list = ["hok", "hkw", "ade", "adw"]
#rad_list = ["ade", "adw"]
# rad_list = ["tig", "unw"]
rad_list = ["bks", "wal", "fhe", "fhw", "cve", "cvw"]
#rad_list = ["bpk"]
# create tmpdirs to store dmap files temporarily
for rad in rad_list:
tmpdir = "../data/" + rad + "_tmp"
os.system("mkdir -p " + tmpdir)
# create dbs (if not exist) for radars
for rad in rad_list:
db_name = rad + "_boxcar_" + ftype
try:
# create a db
db_tools.create_db(db_name)
except Exception, e:
logging.error(e, exc_info=True)
def main():
""" Call the functions above. Acts as an example code.
Multiprocessing has been implemented to do parallel computing.
The unit process is for reading a day worth of data for a given radar"""
import datetime as dt
import multiprocessing as mp
import sys
sys.path.append("../")
from mysql_dbutils import db_tools
import logging
# create a log file to which any error occured between client and
# MySQL server communication will be written.
logging.basicConfig(filename="./log_files/iscat_to_db_six_rads.log",
level=logging.INFO)
# input parameters
sdate = dt.datetime(2017, 1, 1) # includes sdate
# sdate = dt.datetime(2011, 8, 1) # includes sdate
edate = dt.datetime(2018, 7, 1) # does not include edate
channel = None
params = ['velocity']
ftype = "fitacf"
low_vel_iscat_event_only = True
search_allbeams = True
no_gscat = True
data_from_db = True
ffname = None
tmpdir = None
# run the code for the following radars in parallel
#rad_list = ["hok", "hkw"]
#rad_list = ["ade", "adw"]
#rad_list = ["tig", "unw"]
#rad_list = ["bpk"]
rad_list = ["bks", "wal", "fhe", "fhw", "cve", "cvw"]
# create dbs (if not exist) for radars
for rad in rad_list:
db_name = rad + "_iscat_" + ftype
try:
# create a db
db_tools.create_db(db_name)
except Exception, e:
logging.error(e, exc_info=True)
def ten_min_median(rad,
stm,
etm,
ftype="fitacf",
coords="mlt",
config_filename="../mysql_dbconfig_files/config.ini",
section="midlat",
iscat_dbname=None,
output_dbname=None):
""" Bins the gridded data from all beams of a single radar into ten-minute intervals.
e.g., at each ten-minute interval, median vector in each azimuth bin within a grid cell is
selected as the representative velocity for that bin.
The results are stored in a different db such that data from all beams
of a given radar are written into a single table named by the radar name.
Parameters
----------
rad : str
Three-letter radar code
stm : datetime.datetime
The start time.
etm : datetime.datetime
The end time.
ftype : str
SuperDARN file type
coords : str
Coordinates in which the binning process takes place.
Default to "mlt. Can be "geo" as well.
config_filename: str
name and path of the configuration file
section: str, default to "midlat"
section of database configuration
iscat_dbname : str, default to None
Name of the MySQL db to which iscat data has been written
output_dbname : str, default to None
Name of the MySQL db to which ten-min median filtered data will be written
Returns
-------
Nothing
"""
import numpy as np
import datetime as dt
from mysql.connector import MySQLConnection
import sys
sys.path.append("../")
from mysql_dbutils.db_config import db_config
from mysql_dbutils import db_tools
import logging
# create db names
if iscat_dbname is None:
iscat_dbname = rad + "_iscat_" + ftype
if output_dbname is None:
output_dbname = "ten_min_median_" + coords + "_" + ftype
output_table = rad + "_" + ftype
# create a db (if not exist) for ten-min median data
try:
# create a db
db_tools.create_db(output_dbname)
except Exception, e:
logging.error(e, exc_info=True)
def imf_based_filter(imf_table,
output_table,
stm,
etm,
ftype="fitacf",
coords="mlt",
kp_lim=[0.0, 9.0],
config_filename="../mysql_dbconfig_files/config.ini",
section="midlat",
input_dbname=None,
input_table=None,
output_dbname=None,
imf_dbname=None,
imf_db_location="../../data/sqlite3/"):
""" Selects the data for stable IMF intervals.
Parameters
----------
stm : datetime.datetime
The start time
etm : datetime.datetime
The end time
imf_table : list
The binned imf table from which binned imf data will be taken.
output_table : str
Name of the table where filtered iscat data will be stored.
ftype : str
SuperDARN LOS data file type
coords : str
Coordinate systems. valid inputs are "geo" or "mlt".
kp_lim : list
The range of Kp. The range bondaries are inclusive.
config_filename: str
name and path of the configuration file
section: str, default to "midlat"
section of database configuration
input_dbname : str
Name of the input db.
Default to None.
input_table : str
Name of the table where the iscat data will be extracted.
output_dbname : str
Name of the output db.
Default to None.
imf_dbname : str
Name of the sqlite3 db where imf data are stored.
Default to None.
imf_db_location : str
The path to imf_dbname sqlite db.
Default to None.
Returns
-------
Nothing
"""
import datetime as dt
import sqlite3
import os
import sys
sys.path.append("../../")
from mysql.connector import MySQLConnection
from mysql_dbutils.db_config import db_config
from mysql_dbutils import db_tools
import logging
# construct db names and table names
if input_dbname is None:
input_dbname = "master_" + coords + "_" + ftype
if output_dbname is None:
output_dbname = "master_" + coords + "_" + ftype + "_binned_by_imf_clock_angle"
if imf_dbname is None:
imf_dbname = "binned_imf.sqlite"
# create the output db (if not exist)
try:
# create a db
db_tools.create_db(output_dbname, config_filename=config_filename)
except Exception, e:
logging.error(e, exc_info=True)