def updateSHA(dbu, filename):
"""
update the shasum in the db
"""
file = dbu.getEntry('File', dbu.getFileID(os.path.basename(filename)))
file.shasum = DButils.calcDigest(filename)
dbu.session.commit()
def getStuff(dbname, proc_name, filename):
cfg = {}
dbu = DButils.DButils(dbname) # TODO don't assume RBSP later
dbu.openDB()
dbu._createTableObjects()
# are we trying to write files for all the products?
# if prod_name.lower() == 'all':
# prods = dbu.getAllProducts()
# for prod in prods:
# getStuff(prod.product_name, prod.product_name + filename)
# return
proc_id = dbu.getProcessID(proc_name)
# get instances of all the tables in a product traceback
sq = dbu.getTraceback('Process', proc_id)
for section in sq:
attrs = dir(sq[section])
if section in ['input_product',
'productprocesslink']: # special case this is a list
for ii, sec in enumerate(sq[section]):
cfg[section + str(ii)] = {}
attrs = dir(
sec[0]) # [0] is the object, [1] is boolean for optional
for val in attrs:
if val[0] != '_':
cfg[section +
str(ii)][val] = sec[0].__getattribute__(val)
continue
cfg[section] = {}
for val in attrs:
if val[0] != '_':
cfg[section][val] = sq[section].__getattribute__(val)
writeconfig(cfg, filename)
def getStuff(prod_name, filename):
cfg = {}
dbu = DButils.DButils('rbsp') # TODO don't assume RBSP later
dbu.openDB()
dbu._createTableObjects()
# are we trying to write files for all the products?
if prod_name.lower() == 'all':
prods = dbu.getAllProducts()
for prod in prods:
getStuff(prod.product_name, prod.product_name + filename)
return
try:
prod_name = dbu.getProductID(int(prod_name))
except ValueError:
prod_name = dbu.getProductID(prod_name)
# get instances of all the tables in a product traceback
sq = dbu.getTraceback('Product', prod_name)
for section in sq:
attrs = dir(sq[section])
cfg[section] = {}
for val in attrs:
if val[0] != '_':
cfg[section][val] = sq[section].__getattribute__(val)
writeconfig(cfg, filename)
def setUp(self):
super(TestSetup, self).setUp()
sqpath = os.path.join(os.path.dirname(__file__), 'RBSP_MAGEIS.sqlite')
self.sqlworking = sqpath.replace('RBSP_MAGEIS.sqlite',
'working.sqlite')
shutil.copy(sqpath, self.sqlworking)
os.chmod(self.sqlworking, stat.S_IRUSR | stat.S_IWUSR)
self.dbu = DButils.DButils(self.sqlworking)
def setUp(self):
super(UtilsTests, self).setUp()
# Not changing the DB now but use a copy anyway
# Would need to at least update DB path if we wanted to
# do more than vanilla dirSubs
self.tempD = tempfile.mkdtemp()
copy_tree(
os.path.dirname(__file__) + '/../functional_test/', self.tempD)
self.dbu = DButils.DButils(self.tempD + '/testDB.sqlite')
def setUp(self):
"""Makes a copy of the DB to run tests on without altering the original."""
super(InspectorClass, self).setUp()
# These tests shouldn't change the DB but use a copy anyway
# Would need to at least update DB path if we wanted to
# use DB
self.tempD = tempfile.mkdtemp()
copy_tree(os.path.dirname(__file__) + '/../functional_test/', self.tempD)
self.dbu = DButils.DButils(self.tempD + '/testDB.sqlite')
self.inspect = imp.load_source('inspect', os.path.dirname(__file__) + '/inspector/rot13_L1.py')
def setUp(self):
super(DBfileTests, self).setUp()
self.tempD = tempfile.mkdtemp()
copy_tree(
os.path.dirname(__file__) + '/../functional_test/', self.tempD)
copy_tree(os.path.dirname(__file__) + '/tars/', self.tempD)
self.dbu = DButils.DButils(self.tempD + '/testDB.sqlite')
#Update the mission path to the tmp dir
self.dbu.getEntry('Mission', 1).rootdir = self.tempD
self.dbu.commitDB()
self.dbu.MissionDirectory = self.dbu.getMissionDirectory()
def setUp(self):
"""Makes a copy of the DB to run tests on without altering the original."""
super(InspectorClass, self).setUp()
# These tests shouldn't change the DB but use a copy anyway
# Would need to at least update DB path if we wanted to
# use DB
self.tempD = tempfile.mkdtemp()
copy_tree(os.path.join(
dbp_testing.testsdir, '..', 'functional_test'), self.tempD)
self.dbu = DButils.DButils(os.path.join(self.tempD, 'testDB.sqlite'))
self.inspect = imp.load_source('inspect', os.path.join(
dbp_testing.testsdir, 'inspector', 'rot13_L1.py'))
def copyDB(DBname = None):
usage = \
"""
Usage: %prog -m
-m -> selects mission
"""
parser = OptionParser(usage=usage)
parser.add_option("-m", "--mission", dest="mission",
help="selected mission database", default=None)
(options, args) = parser.parse_args()
if len(args) != 0:
parser.error("incorrect number of arguments")
DButils.CreateDB(copiedDB_test.db)
def makeHTML(mission, filename):
header = """
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
<html><head>
<meta content="text/html; charset=ISO-8859-1" http-equiv="content-type"><title>DBprocessing</title>
<style type="text/css">
table, td, th
{
border:1px solid green;
padding:3px 7px 2px 7px;
}
th
{
background-color:green;
color:white;
}
tr.alt td
{
color:#000000;
background-color:#EAF2D3;
}
</style>
</head>
<body>
"""
footer = \
"""
<br>
<br>
</body></html>
"""
dbu = DButils.DButils(mission) # TODO don't assume RBSP later
dbu.openDB()
dbu._createTableObjects()
output = open(filename, 'w')
output.writelines(header)
_writeProducts(dbu, output)
_writeProcesses(dbu, output)
output.writelines(footer)
output.close()
def makeHTML(mission, filename):
global dbu
header = """
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
<html><head>
<meta content="text/html; charset=ISO-8859-1" http-equiv="content-type"><title>HOPE Coverage</title>
<style type="text/css">
table, td, th
{
border:1px solid green;
padding:3px 7px 2px 7px;
}
th
{
background-color:green;
color:white;
}
tr.alt td
{
color:#000000;
background-color:#EAF2D3;
}
</style>
</head>
<body>
"""
footer = \
"""
<br>
<br>
</body></html>
"""
dbu = DButils.DButils(mission)
a, b, = gather_files()
output = open(filename, 'w')
output.writelines(header)
_writeTable(dbu, output, a, b)
output.writelines(footer)
output.close()
def _updateSections(conf):
"""
go through each section and update the db is there is a change
"""
dbu = DButils.DButils('rbsp') # TODO don't assume RBSP later
dbu.openDB()
dbu._createTableObjects()
sections = [
'mission', 'satellite', 'instrument', 'product',
'instrumentproductlink', 'inspector'
]
succ = 0
for section in sections:
# get the object for the section
try:
obj = dbu.session.query(dbu.__getattribute__(section.title())).get(
conf[section][section + '_id'])
succ += 1
except KeyError: # happens for instrumentproductlink where there is not an id
continue
# get the attributes
attrs = [v for v in dir(obj) if v[0] != '_']
# check if everything is the same or not
same = [conf[section][v1] == obj.__getattribute__(v1) for v1 in attrs]
# print out what is different DB --- file
if sum(same) != len(same): # means they are different
for i, v1 in enumerate(same):
if not v1:
print('{0}[{1}] {2} ==> {3}'.format(
section, attrs[i], obj.__getattribute__(attrs[i]),
conf[section][attrs[i]]))
obj.__setattr__(attrs[i], conf[section][attrs[i]])
dbu.session.add(obj)
dbu.commitDB()
if succ == 0:
raise (ValueError(
'using {0} on a product that is not in the DB'.format(
sys.argv[0])))
startDate = startDate.date()
if options.endDate is not None:
endDate = dup.parse(options.endDate)
else:
endDate = datetime.datetime.now()
endDate = endDate.date()
if endDate < startDate:
parser.error("endDate must be >= to startDate")
db = dbprocessing.ProcessQueue(options.mission,)
dbu = DButils.DButils(options.mission, echo=options.echo)
dates = [startDate + datetime.timedelta(days=v) for v in range((endDate-startDate).days +1)]
if not dates: # only one day
dates = [startDate]
# this is the possible dates for a product
dates = sorted(dates)
product_id = args[0]
dbfiles = dbu.getFilesByProductDate(product_id,
[startDate, endDate],
newest_version=True)
# this returns filenames
# make it file objects
def test1(self):
print('111111111')
dbu = DButils.DButils(self.tfile)
print('222222222')
del dbu
print('333333333')
import datetime
import glob
import os
import sys
import re
import dateutil
import rbsp #rbsp.mission_day_to_UTC
from dbprocessing import Utils, inspector
from rbsp import Version
from dbprocessing import DButils
dbu = DButils.DButils('~ectsoc/MagEphem_processing.sqlite')
prods = dbu.getAllProducts()
ids = [v.product_id for v in prods]
names = [v.product_name for v in prods]
sc = [dbu.getTraceback('Product', v.product_id)['satellite'].satellite_name for v in prods]
a_files = []
b_files = []
a_dates = []
b_dates = []
a_products = []
b_products = []
a_versions = []
b_versions = []
import re
import shutil
import subprocess
import dbprocessing.DButils as DBUtils
import dbprocessing.DBlogging as DBlogging
"""
make a meta kernel each day that is dated 2 weeks out
1) figure out what files we have
2) figure out what 2 weeks from today is
3) make a list of all dates that would be today through 2 weeks
4) if we do not have that file make it
- if yes, done, if no runnewMetaKernel.py and drop in incoming
"""
dbu = DBUtils.DButils(
os.path.expanduser('/home/ectsoc/PROCESSING_DB/magephem_pre.sqlite'))
mission_path = dbu.getMissionDirectory()
g_inc_path = dbu.getIncomingPath()
filesa = dbu.getFilesByProduct('rbspa_pre_kernel')
filesb = dbu.getFilesByProduct('rbspb_pre_kernel')
#files = sorted(files, key=lambda x: x.utc_file_date)[-1]
dbu.closeDB()
filesa = set(v.filename for v in filesa)
filesb = set(v.filename for v in filesb)
files = filesa.union(filesb)
dates = [
def test1(self):
dbu = DButils.DButils(self.tfile)
del dbu
"-e",
"--endDate",
dest="endDate",
type="string",
help='Date to end printing, only used for "File" (e.g. 2012-10-25)',
default=None)
(options, args) = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
mission = args[0]
field = args[1].capitalize()
dbu = DButils.DButils(mission)
basepath = dbu.session.query(dbu.Mission).filter_by(
mission_name=dbu.getMissions()[0]).all()[0].rootdir
if not hasattr(dbu, field):
dbu.closeDB()
parser.error('Field: "{0}" not found in database: "{1}"'.format(
field, mission))
if field == 'Product':
print("{0:4} {1:5} {2:40} {3}".format("ID", "LEVEL", "NAME", "PATH"))
for p in dbu.getAllProducts():
print(getProductString(p, basepath))
elif field == 'Mission':
print("{0:4} {1:40} {2:40} {3:40}".format("ID", "NAME", "ROOT",
help="The file to test the inspector on")
parser.add_option("-i",
"--inspector",
dest="inspector",
help="The inspector to test")
parser.add_option("-p",
"--product",
dest="product",
help="The product the file belongs to")
parser.add_option("-a",
"--args",
dest="args",
help="kwargs to pass to the inspector(optional)",
default=None)
(options, args) = parser.parse_args()
dbu = DButils.DButils(options.mission)
inspect = imp.load_source('inspect', options.inspector)
if options.args:
kwargs = strargs_to_args(options.args)
df = inspect.Inspector(options.file, dbu, options.product, **kwargs)
else:
df = inspect.Inspector(
options.file,
dbu,
options.product,
)
print(df)
def addMission(self, filename):
"""utility to add a mission"""
self.dbu = DButils.DButils(filename)
self.mission = self.dbu.addMission(
'rbsp', os.path.join('/', 'n', 'space_data', 'cda', 'rbsp'))
def addStuff(cfg, options):
# setup the db
dbu = DButils.DButils(options.mission)
# is the mission in the DB? If not add it
if cfg['mission']['mission_name'] not in dbu.getMissions(
): # was it there?
# add it
mission_id = dbu.addMission(**cfg['mission'])
print('Added Mission: {0} {1}'.format(
mission_id,
dbu.getEntry('Mission', mission_id).mission_name))
else:
mission_id = dbu.getMissionID(cfg['mission']['mission_name'])
print('Found Mission: {0} {1}'.format(
mission_id,
dbu.getEntry('Mission', mission_id).mission_name))
# is the satellite in the DB? If not add it
try:
satellite_id = dbu.getEntry(
'Satellite', cfg['satellite']['satellite_name']).satellite_id
print('Found Satellite: {0} {1}'.format(
satellite_id,
dbu.getEntry('Satellite', satellite_id).satellite_name))
except (DButils.DBNoData, NoResultFound):
# add it
satellite_id = dbu.addSatellite(mission_id=mission_id,
**cfg['satellite'])
print('Added Satellite: {0} {1}'.format(
satellite_id,
dbu.getEntry('Satellite', satellite_id).satellite_name))
# is the instrument in the DB? If not add it
try:
inst_id = dbu.getInstrumentID(cfg['instrument']['instrument_name'],
satellite_id)
instrument = dbu.session.query(dbu.Instrument).get(inst_id)
if instrument.satellite_id != satellite_id:
raise (
ValueError()
) # this means it is the same name on a different sat, need to add
instrument_id = instrument.instrument_id
print('Found Instrument: {0} {1}'.format(
instrument_id,
dbu.getEntry('Instrument', instrument_id).instrument_name))
except (DButils.DBNoData, ValueError, NoResultFound):
# add it
instrument_id = dbu.addInstrument(satellite_id=satellite_id,
**cfg['instrument'])
print('Added Instrument: {0} {1}'.format(
instrument_id,
dbu.getEntry('Instrument', instrument_id).instrument_name))
# loop over all the products, check if they are there and add them if not
products = [k for k in cfg if k.startswith('product')]
db_products = [v.product_name for v in dbu.getAllProducts()]
for p in products:
# is the product in the DB? If not add it
if cfg[p]['product_name'] in db_products:
p_id = dbu.getEntry('Product', cfg[p]['product_name']).product_id
cfg[p]['product_id'] = p_id
print('Found Product: {0} {1}'.format(
p_id,
dbu.getEntry('Product', p_id).product_name))
else:
tmp = dict((k, cfg[p][k]) for k in cfg[p]
if not k.startswith('inspector'))
p_id = dbu.addProduct(instrument_id=instrument_id, **tmp)
print('Added Product: {0} {1}'.format(
p_id,
dbu.getEntry('Product', p_id).product_name))
cfg[p]['product_id'] = p_id
ippl = dbu.addInstrumentproductlink(instrument_id, p_id)
print('Added Instrumentproductlink: {0}'.format(ippl))
dbu.updateProductSubs(p_id)
# if the product was not there we will assume the inspector is not either (requires a product_id)
tmp = dict(
(k, cfg[p][k]) for k in cfg[p] if k.startswith('inspector'))
replace_dict = {
'inspector_output_interface': 'output_interface_version',
'inspector_version': 'version',
'inspector_arguments': 'arguments',
'inspector_description': 'description',
'inspector_newest_version': 'newest_version',
'inspector_relative_path': 'relative_path',
'inspector_date_written': 'date_written',
'inspector_filename': 'filename',
'inspector_active': 'active_code'
}
for rd in replace_dict:
tmp[replace_dict[rd]] = tmp.pop(rd)
insp_id = dbu.addInspector(product=p_id, **tmp)
print('Added Inspector: {0} {1}'.format(
insp_id,
dbu.getEntry('Inspector', insp_id).filename))
dbu.updateInspectorSubs(insp_id)
# loop over all the processes, check if they are there and add them if not
processes = [k for k in cfg if k.startswith('process')]
db_processes = [v.process_name for v in dbu.getAllProcesses()]
for p in processes:
# is the process in the DB? If not add it
if cfg[p]['process_name'] in db_processes:
p_id = dbu.getEntry('Process', cfg[p]['process_name']).process_id
print('Found Process: {0} {1}'.format(
p_id,
dbu.getEntry('Process', p_id).process_name))
else:
tmp = dict((k, cfg[p][k]) for k in cfg[p]
if not k.startswith('code') and 'input' not in k)
# need to replace the output product with the right ID
# if it is a key then have to get the name from cfg, or it is a name itself
if tmp['output_product'] is not '':
tmp['output_product'] = cfg[
tmp['output_product']]['product_id']
p_id = dbu.addProcess(**tmp)
print('Added Process: {0} {1}'.format(
p_id,
dbu.getEntry('Process', p_id).process_name))
# now add the productprocesslink
tmp = dict((k, cfg[p][k]) for k in cfg[p] if 'input' in k)
for k in tmp:
ppl = dbu.addproductprocesslink(cfg[tmp[k][0]]['product_id'],
p_id, 'optional' in k,
tmp[k][1], tmp[k][2])
print('Added Productprocesslink: {0}'.format(ppl))
# if the process was not there we will assume the code is not either (requires a process_id)
tmp = dict((k, cfg[p][k]) for k in cfg[p] if k.startswith('code'))
replace_dict = {
'code_filename': 'filename',
'code_arguments': 'arguments',
'code_relative_path': 'relative_path',
'code_version': 'version',
'code_output_interface': 'output_interface_version',
'code_newest_version': 'newest_version',
'code_date_written': 'date_written',
'code_active': 'active_code',
'code_ram': 'ram',
'code_cpu': 'cpu'
}
for rd in replace_dict:
tmp[replace_dict[rd]] = tmp.pop(rd)
code_id = dbu.addCode(process_id=p_id, **tmp)
print('Added Code: {0} {1}'.format(
code_id,
dbu.getEntry('Code', code_id).filename))
#!/usr/bin/env python2.6
from __future__ import print_function
import sys
from dbprocessing import DButils
if len(sys.argv) != 2:
print("Usage: {0} database".format(sys.argv[0]))
sys.exit(-1)
if __name__ == "__main__":
a = DButils.DButils(sys.argv[1])
n_items = a.Processqueue.len()
a.Processqueue.flush()
print('Flushed {0} items from queue'.format(n_items))
#!/usr/bin/env python2.6
from __future__ import print_function
from dbprocessing import DButils
if __name__ == "__main__":
a = DButils.DButils('rbsp')
a.openDB()
a._createTableObjects()
prod_ids = zip(*a.getProductNames())[4]
for ff in f:
a._purgeFileFromDB(ff[0])
print('deleted {0} files'.format(len(f)))
parser.error("incorrect number of arguments")
try:
version = Version.Version.fromString(args[1])
except ValueError:
parser.error("Invalid version: {0}, must be X.Y.Z".format(args[1]))
try:
code_id = int(args[0])
except ValueError:
parser.error("Invalid code_id: {0}, must be an int".format(args[0]))
newname = args[2]
dbu = DButils.DButils(os.path.expanduser(options.mission))
try:
code = dbu.getEntry('Code', code_id)
except DButils.DBNoData:
parser.error("Invalid code_id: {0}, must be in the database".format(code_id))
old_version = Version.Version(code.interface_version, code.quality_version, code.revision_version)
if version <= old_version:
parser.error("New version, {0}, must be larger than old version {1}".format(version,old_version ))
# make a new code and copy across the needed keys
attrs = [ u'active_code',
u'arguments',
u'code_description',
if not os.path.isfile(conffile):
parser.error("could not read config file: {0}".format(conffile))
conf = readconfig(conffile)
conf = _processSubs(conf)
conf = _processDates(conf)
print('Read and parsed config file: {0}'.format(conffile))
# figure out the dates we are going to use
# make a range of dates
dates = list(Utils.expandDates(conf['settings']['startdate'], conf['settings']['enddate']))
dates = [v.date() for v in dates]
dates = list(Utils.chunker(dates, int(conf['panel']['daysperplot'])))
# make the arrays that hold the files
dbu = DButils.DButils(conf['settings']['mission'], echo=echo)
# go through the panel and see how many plots there are
nplots = _get_nplots(conf)
ans = []
for pnum, ind_pnum in enumerate(range(nplots), 1):
ans.append([])
n_prods = _get_nproducts(conf, pnum)
prods = []
for ind_d, d in enumerate(dates):
ans[ind_pnum].append([])
for pn, ind_pn in enumerate(range(n_prods), 1):
ans[ind_pnum][ind_d].append([])
return output
if __name__ == '__main__':
parser = OptionParser()
parser.add_option("", "--html", dest="html", action="store_true",
help="Output in html", default=False)
parser.add_option("-o", "--output", dest="output",
help="Write output to a file")
(options, args) = parser.parse_args()
if( len(args) != 1 ):
parser.error("Must pass a mission DB")
dbu = DButils.DButils(args[0])
items = dbu.Processqueue.getAll()
traceback = []
for v in items:
traceback.append(dbu.getTraceback('File', v))
if( options.html ):
out = output_html(traceback)
else:
out = output_text(traceback)
if( options.output is None):
print(out)
else:
output = open(options.output, 'w')
output.write(out)
help='Want yesterday. Default is false.',
required=False)
parser.add_argument('-t',
'--tmrw',
dest='tomorrow',
action='store_true',
help='Want tomorrow. Default is false.',
required=False)
args = parser.parse_args(argv)
kwargs = vars(args)
return [], kwargs
if __name__ == '__main__':
args, kwargs = parse_args()
db = DButils.DButils(kwargs['name'])
try:
proc_id = db.getProcessID(kwargs['process'])
except:
raise ValueError('Could not find process in database: {}'.format(
kwargs['process']))
try:
prod_id = db.getProductID(kwargs['product'])
except:
raise ValueError('Could not find product in database: {}'.format(
kwargs['product']))
if kwargs['yesterday']:
yesterday = 1
else:
