def updateSHA(dbu, filename): """ update the shasum in the db """ file = dbu.getEntry('File', dbu.getFileID(os.path.basename(filename))) file.shasum = DButils.calcDigest(filename) dbu.session.commit()
def getStuff(dbname, proc_name, filename): cfg = {} dbu = DButils.DButils(dbname) # TODO don't assume RBSP later dbu.openDB() dbu._createTableObjects() # are we trying to write files for all the products? # if prod_name.lower() == 'all': # prods = dbu.getAllProducts() # for prod in prods: # getStuff(prod.product_name, prod.product_name + filename) # return proc_id = dbu.getProcessID(proc_name) # get instances of all the tables in a product traceback sq = dbu.getTraceback('Process', proc_id) for section in sq: attrs = dir(sq[section]) if section in ['input_product', 'productprocesslink']: # special case this is a list for ii, sec in enumerate(sq[section]): cfg[section + str(ii)] = {} attrs = dir( sec[0]) # [0] is the object, [1] is boolean for optional for val in attrs: if val[0] != '_': cfg[section + str(ii)][val] = sec[0].__getattribute__(val) continue cfg[section] = {} for val in attrs: if val[0] != '_': cfg[section][val] = sq[section].__getattribute__(val) writeconfig(cfg, filename)
def getStuff(prod_name, filename): cfg = {} dbu = DButils.DButils('rbsp') # TODO don't assume RBSP later dbu.openDB() dbu._createTableObjects() # are we trying to write files for all the products? if prod_name.lower() == 'all': prods = dbu.getAllProducts() for prod in prods: getStuff(prod.product_name, prod.product_name + filename) return try: prod_name = dbu.getProductID(int(prod_name)) except ValueError: prod_name = dbu.getProductID(prod_name) # get instances of all the tables in a product traceback sq = dbu.getTraceback('Product', prod_name) for section in sq: attrs = dir(sq[section]) cfg[section] = {} for val in attrs: if val[0] != '_': cfg[section][val] = sq[section].__getattribute__(val) writeconfig(cfg, filename)
def setUp(self): super(TestSetup, self).setUp() sqpath = os.path.join(os.path.dirname(__file__), 'RBSP_MAGEIS.sqlite') self.sqlworking = sqpath.replace('RBSP_MAGEIS.sqlite', 'working.sqlite') shutil.copy(sqpath, self.sqlworking) os.chmod(self.sqlworking, stat.S_IRUSR | stat.S_IWUSR) self.dbu = DButils.DButils(self.sqlworking)
def setUp(self): super(UtilsTests, self).setUp() # Not changing the DB now but use a copy anyway # Would need to at least update DB path if we wanted to # do more than vanilla dirSubs self.tempD = tempfile.mkdtemp() copy_tree( os.path.dirname(__file__) + '/../functional_test/', self.tempD) self.dbu = DButils.DButils(self.tempD + '/testDB.sqlite')
def setUp(self): """Makes a copy of the DB to run tests on without altering the original.""" super(InspectorClass, self).setUp() # These tests shouldn't change the DB but use a copy anyway # Would need to at least update DB path if we wanted to # use DB self.tempD = tempfile.mkdtemp() copy_tree(os.path.dirname(__file__) + '/../functional_test/', self.tempD) self.dbu = DButils.DButils(self.tempD + '/testDB.sqlite') self.inspect = imp.load_source('inspect', os.path.dirname(__file__) + '/inspector/rot13_L1.py')
def setUp(self): super(DBfileTests, self).setUp() self.tempD = tempfile.mkdtemp() copy_tree( os.path.dirname(__file__) + '/../functional_test/', self.tempD) copy_tree(os.path.dirname(__file__) + '/tars/', self.tempD) self.dbu = DButils.DButils(self.tempD + '/testDB.sqlite') #Update the mission path to the tmp dir self.dbu.getEntry('Mission', 1).rootdir = self.tempD self.dbu.commitDB() self.dbu.MissionDirectory = self.dbu.getMissionDirectory()
def setUp(self): """Makes a copy of the DB to run tests on without altering the original.""" super(InspectorClass, self).setUp() # These tests shouldn't change the DB but use a copy anyway # Would need to at least update DB path if we wanted to # use DB self.tempD = tempfile.mkdtemp() copy_tree(os.path.join( dbp_testing.testsdir, '..', 'functional_test'), self.tempD) self.dbu = DButils.DButils(os.path.join(self.tempD, 'testDB.sqlite')) self.inspect = imp.load_source('inspect', os.path.join( dbp_testing.testsdir, 'inspector', 'rot13_L1.py'))
def copyDB(DBname = None): usage = \ """ Usage: %prog -m -m -> selects mission """ parser = OptionParser(usage=usage) parser.add_option("-m", "--mission", dest="mission", help="selected mission database", default=None) (options, args) = parser.parse_args() if len(args) != 0: parser.error("incorrect number of arguments") DButils.CreateDB(copiedDB_test.db)
def makeHTML(mission, filename): header = """ <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html><head> <meta content="text/html; charset=ISO-8859-1" http-equiv="content-type"><title>DBprocessing</title> <style type="text/css"> table, td, th { border:1px solid green; padding:3px 7px 2px 7px; } th { background-color:green; color:white; } tr.alt td { color:#000000; background-color:#EAF2D3; } </style> </head> <body> """ footer = \ """ <br> <br> </body></html> """ dbu = DButils.DButils(mission) # TODO don't assume RBSP later dbu.openDB() dbu._createTableObjects() output = open(filename, 'w') output.writelines(header) _writeProducts(dbu, output) _writeProcesses(dbu, output) output.writelines(footer) output.close()
def makeHTML(mission, filename): global dbu header = """ <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html><head> <meta content="text/html; charset=ISO-8859-1" http-equiv="content-type"><title>HOPE Coverage</title> <style type="text/css"> table, td, th { border:1px solid green; padding:3px 7px 2px 7px; } th { background-color:green; color:white; } tr.alt td { color:#000000; background-color:#EAF2D3; } </style> </head> <body> """ footer = \ """ <br> <br> </body></html> """ dbu = DButils.DButils(mission) a, b, = gather_files() output = open(filename, 'w') output.writelines(header) _writeTable(dbu, output, a, b) output.writelines(footer) output.close()
def _updateSections(conf): """ go through each section and update the db is there is a change """ dbu = DButils.DButils('rbsp') # TODO don't assume RBSP later dbu.openDB() dbu._createTableObjects() sections = [ 'mission', 'satellite', 'instrument', 'product', 'instrumentproductlink', 'inspector' ] succ = 0 for section in sections: # get the object for the section try: obj = dbu.session.query(dbu.__getattribute__(section.title())).get( conf[section][section + '_id']) succ += 1 except KeyError: # happens for instrumentproductlink where there is not an id continue # get the attributes attrs = [v for v in dir(obj) if v[0] != '_'] # check if everything is the same or not same = [conf[section][v1] == obj.__getattribute__(v1) for v1 in attrs] # print out what is different DB --- file if sum(same) != len(same): # means they are different for i, v1 in enumerate(same): if not v1: print('{0}[{1}] {2} ==> {3}'.format( section, attrs[i], obj.__getattribute__(attrs[i]), conf[section][attrs[i]])) obj.__setattr__(attrs[i], conf[section][attrs[i]]) dbu.session.add(obj) dbu.commitDB() if succ == 0: raise (ValueError( 'using {0} on a product that is not in the DB'.format( sys.argv[0])))
startDate = startDate.date() if options.endDate is not None: endDate = dup.parse(options.endDate) else: endDate = datetime.datetime.now() endDate = endDate.date() if endDate < startDate: parser.error("endDate must be >= to startDate") db = dbprocessing.ProcessQueue(options.mission,) dbu = DButils.DButils(options.mission, echo=options.echo) dates = [startDate + datetime.timedelta(days=v) for v in range((endDate-startDate).days +1)] if not dates: # only one day dates = [startDate] # this is the possible dates for a product dates = sorted(dates) product_id = args[0] dbfiles = dbu.getFilesByProductDate(product_id, [startDate, endDate], newest_version=True) # this returns filenames # make it file objects
def test1(self): print('111111111') dbu = DButils.DButils(self.tfile) print('222222222') del dbu print('333333333')
import datetime import glob import os import sys import re import dateutil import rbsp #rbsp.mission_day_to_UTC from dbprocessing import Utils, inspector from rbsp import Version from dbprocessing import DButils dbu = DButils.DButils('~ectsoc/MagEphem_processing.sqlite') prods = dbu.getAllProducts() ids = [v.product_id for v in prods] names = [v.product_name for v in prods] sc = [dbu.getTraceback('Product', v.product_id)['satellite'].satellite_name for v in prods] a_files = [] b_files = [] a_dates = [] b_dates = [] a_products = [] b_products = [] a_versions = [] b_versions = []
import re import shutil import subprocess import dbprocessing.DButils as DBUtils import dbprocessing.DBlogging as DBlogging """ make a meta kernel each day that is dated 2 weeks out 1) figure out what files we have 2) figure out what 2 weeks from today is 3) make a list of all dates that would be today through 2 weeks 4) if we do not have that file make it - if yes, done, if no runnewMetaKernel.py and drop in incoming """ dbu = DBUtils.DButils( os.path.expanduser('/home/ectsoc/PROCESSING_DB/magephem_pre.sqlite')) mission_path = dbu.getMissionDirectory() g_inc_path = dbu.getIncomingPath() filesa = dbu.getFilesByProduct('rbspa_pre_kernel') filesb = dbu.getFilesByProduct('rbspb_pre_kernel') #files = sorted(files, key=lambda x: x.utc_file_date)[-1] dbu.closeDB() filesa = set(v.filename for v in filesa) filesb = set(v.filename for v in filesb) files = filesa.union(filesb) dates = [
def test1(self): dbu = DButils.DButils(self.tfile) del dbu
"-e", "--endDate", dest="endDate", type="string", help='Date to end printing, only used for "File" (e.g. 2012-10-25)', default=None) (options, args) = parser.parse_args() if len(args) != 2: parser.error("incorrect number of arguments") mission = args[0] field = args[1].capitalize() dbu = DButils.DButils(mission) basepath = dbu.session.query(dbu.Mission).filter_by( mission_name=dbu.getMissions()[0]).all()[0].rootdir if not hasattr(dbu, field): dbu.closeDB() parser.error('Field: "{0}" not found in database: "{1}"'.format( field, mission)) if field == 'Product': print("{0:4} {1:5} {2:40} {3}".format("ID", "LEVEL", "NAME", "PATH")) for p in dbu.getAllProducts(): print(getProductString(p, basepath)) elif field == 'Mission': print("{0:4} {1:40} {2:40} {3:40}".format("ID", "NAME", "ROOT",
help="The file to test the inspector on") parser.add_option("-i", "--inspector", dest="inspector", help="The inspector to test") parser.add_option("-p", "--product", dest="product", help="The product the file belongs to") parser.add_option("-a", "--args", dest="args", help="kwargs to pass to the inspector(optional)", default=None) (options, args) = parser.parse_args() dbu = DButils.DButils(options.mission) inspect = imp.load_source('inspect', options.inspector) if options.args: kwargs = strargs_to_args(options.args) df = inspect.Inspector(options.file, dbu, options.product, **kwargs) else: df = inspect.Inspector( options.file, dbu, options.product, ) print(df)
def addMission(self, filename): """utility to add a mission""" self.dbu = DButils.DButils(filename) self.mission = self.dbu.addMission( 'rbsp', os.path.join('/', 'n', 'space_data', 'cda', 'rbsp'))
def addStuff(cfg, options): # setup the db dbu = DButils.DButils(options.mission) # is the mission in the DB? If not add it if cfg['mission']['mission_name'] not in dbu.getMissions( ): # was it there? # add it mission_id = dbu.addMission(**cfg['mission']) print('Added Mission: {0} {1}'.format( mission_id, dbu.getEntry('Mission', mission_id).mission_name)) else: mission_id = dbu.getMissionID(cfg['mission']['mission_name']) print('Found Mission: {0} {1}'.format( mission_id, dbu.getEntry('Mission', mission_id).mission_name)) # is the satellite in the DB? If not add it try: satellite_id = dbu.getEntry( 'Satellite', cfg['satellite']['satellite_name']).satellite_id print('Found Satellite: {0} {1}'.format( satellite_id, dbu.getEntry('Satellite', satellite_id).satellite_name)) except (DButils.DBNoData, NoResultFound): # add it satellite_id = dbu.addSatellite(mission_id=mission_id, **cfg['satellite']) print('Added Satellite: {0} {1}'.format( satellite_id, dbu.getEntry('Satellite', satellite_id).satellite_name)) # is the instrument in the DB? If not add it try: inst_id = dbu.getInstrumentID(cfg['instrument']['instrument_name'], satellite_id) instrument = dbu.session.query(dbu.Instrument).get(inst_id) if instrument.satellite_id != satellite_id: raise ( ValueError() ) # this means it is the same name on a different sat, need to add instrument_id = instrument.instrument_id print('Found Instrument: {0} {1}'.format( instrument_id, dbu.getEntry('Instrument', instrument_id).instrument_name)) except (DButils.DBNoData, ValueError, NoResultFound): # add it instrument_id = dbu.addInstrument(satellite_id=satellite_id, **cfg['instrument']) print('Added Instrument: {0} {1}'.format( instrument_id, dbu.getEntry('Instrument', instrument_id).instrument_name)) # loop over all the products, check if they are there and add them if not products = [k for k in cfg if k.startswith('product')] db_products = [v.product_name for v in dbu.getAllProducts()] for p in products: # is the product in the DB? If not add it if cfg[p]['product_name'] in db_products: p_id = dbu.getEntry('Product', cfg[p]['product_name']).product_id cfg[p]['product_id'] = p_id print('Found Product: {0} {1}'.format( p_id, dbu.getEntry('Product', p_id).product_name)) else: tmp = dict((k, cfg[p][k]) for k in cfg[p] if not k.startswith('inspector')) p_id = dbu.addProduct(instrument_id=instrument_id, **tmp) print('Added Product: {0} {1}'.format( p_id, dbu.getEntry('Product', p_id).product_name)) cfg[p]['product_id'] = p_id ippl = dbu.addInstrumentproductlink(instrument_id, p_id) print('Added Instrumentproductlink: {0}'.format(ippl)) dbu.updateProductSubs(p_id) # if the product was not there we will assume the inspector is not either (requires a product_id) tmp = dict( (k, cfg[p][k]) for k in cfg[p] if k.startswith('inspector')) replace_dict = { 'inspector_output_interface': 'output_interface_version', 'inspector_version': 'version', 'inspector_arguments': 'arguments', 'inspector_description': 'description', 'inspector_newest_version': 'newest_version', 'inspector_relative_path': 'relative_path', 'inspector_date_written': 'date_written', 'inspector_filename': 'filename', 'inspector_active': 'active_code' } for rd in replace_dict: tmp[replace_dict[rd]] = tmp.pop(rd) insp_id = dbu.addInspector(product=p_id, **tmp) print('Added Inspector: {0} {1}'.format( insp_id, dbu.getEntry('Inspector', insp_id).filename)) dbu.updateInspectorSubs(insp_id) # loop over all the processes, check if they are there and add them if not processes = [k for k in cfg if k.startswith('process')] db_processes = [v.process_name for v in dbu.getAllProcesses()] for p in processes: # is the process in the DB? If not add it if cfg[p]['process_name'] in db_processes: p_id = dbu.getEntry('Process', cfg[p]['process_name']).process_id print('Found Process: {0} {1}'.format( p_id, dbu.getEntry('Process', p_id).process_name)) else: tmp = dict((k, cfg[p][k]) for k in cfg[p] if not k.startswith('code') and 'input' not in k) # need to replace the output product with the right ID # if it is a key then have to get the name from cfg, or it is a name itself if tmp['output_product'] is not '': tmp['output_product'] = cfg[ tmp['output_product']]['product_id'] p_id = dbu.addProcess(**tmp) print('Added Process: {0} {1}'.format( p_id, dbu.getEntry('Process', p_id).process_name)) # now add the productprocesslink tmp = dict((k, cfg[p][k]) for k in cfg[p] if 'input' in k) for k in tmp: ppl = dbu.addproductprocesslink(cfg[tmp[k][0]]['product_id'], p_id, 'optional' in k, tmp[k][1], tmp[k][2]) print('Added Productprocesslink: {0}'.format(ppl)) # if the process was not there we will assume the code is not either (requires a process_id) tmp = dict((k, cfg[p][k]) for k in cfg[p] if k.startswith('code')) replace_dict = { 'code_filename': 'filename', 'code_arguments': 'arguments', 'code_relative_path': 'relative_path', 'code_version': 'version', 'code_output_interface': 'output_interface_version', 'code_newest_version': 'newest_version', 'code_date_written': 'date_written', 'code_active': 'active_code', 'code_ram': 'ram', 'code_cpu': 'cpu' } for rd in replace_dict: tmp[replace_dict[rd]] = tmp.pop(rd) code_id = dbu.addCode(process_id=p_id, **tmp) print('Added Code: {0} {1}'.format( code_id, dbu.getEntry('Code', code_id).filename))
#!/usr/bin/env python2.6 from __future__ import print_function import sys from dbprocessing import DButils if len(sys.argv) != 2: print("Usage: {0} database".format(sys.argv[0])) sys.exit(-1) if __name__ == "__main__": a = DButils.DButils(sys.argv[1]) n_items = a.Processqueue.len() a.Processqueue.flush() print('Flushed {0} items from queue'.format(n_items))
#!/usr/bin/env python2.6 from __future__ import print_function from dbprocessing import DButils if __name__ == "__main__": a = DButils.DButils('rbsp') a.openDB() a._createTableObjects() prod_ids = zip(*a.getProductNames())[4] for ff in f: a._purgeFileFromDB(ff[0]) print('deleted {0} files'.format(len(f)))
parser.error("incorrect number of arguments") try: version = Version.Version.fromString(args[1]) except ValueError: parser.error("Invalid version: {0}, must be X.Y.Z".format(args[1])) try: code_id = int(args[0]) except ValueError: parser.error("Invalid code_id: {0}, must be an int".format(args[0])) newname = args[2] dbu = DButils.DButils(os.path.expanduser(options.mission)) try: code = dbu.getEntry('Code', code_id) except DButils.DBNoData: parser.error("Invalid code_id: {0}, must be in the database".format(code_id)) old_version = Version.Version(code.interface_version, code.quality_version, code.revision_version) if version <= old_version: parser.error("New version, {0}, must be larger than old version {1}".format(version,old_version )) # make a new code and copy across the needed keys attrs = [ u'active_code', u'arguments', u'code_description',
if not os.path.isfile(conffile): parser.error("could not read config file: {0}".format(conffile)) conf = readconfig(conffile) conf = _processSubs(conf) conf = _processDates(conf) print('Read and parsed config file: {0}'.format(conffile)) # figure out the dates we are going to use # make a range of dates dates = list(Utils.expandDates(conf['settings']['startdate'], conf['settings']['enddate'])) dates = [v.date() for v in dates] dates = list(Utils.chunker(dates, int(conf['panel']['daysperplot']))) # make the arrays that hold the files dbu = DButils.DButils(conf['settings']['mission'], echo=echo) # go through the panel and see how many plots there are nplots = _get_nplots(conf) ans = [] for pnum, ind_pnum in enumerate(range(nplots), 1): ans.append([]) n_prods = _get_nproducts(conf, pnum) prods = [] for ind_d, d in enumerate(dates): ans[ind_pnum].append([]) for pn, ind_pn in enumerate(range(n_prods), 1): ans[ind_pnum][ind_d].append([])
return output if __name__ == '__main__': parser = OptionParser() parser.add_option("", "--html", dest="html", action="store_true", help="Output in html", default=False) parser.add_option("-o", "--output", dest="output", help="Write output to a file") (options, args) = parser.parse_args() if( len(args) != 1 ): parser.error("Must pass a mission DB") dbu = DButils.DButils(args[0]) items = dbu.Processqueue.getAll() traceback = [] for v in items: traceback.append(dbu.getTraceback('File', v)) if( options.html ): out = output_html(traceback) else: out = output_text(traceback) if( options.output is None): print(out) else: output = open(options.output, 'w') output.write(out)
help='Want yesterday. Default is false.', required=False) parser.add_argument('-t', '--tmrw', dest='tomorrow', action='store_true', help='Want tomorrow. Default is false.', required=False) args = parser.parse_args(argv) kwargs = vars(args) return [], kwargs if __name__ == '__main__': args, kwargs = parse_args() db = DButils.DButils(kwargs['name']) try: proc_id = db.getProcessID(kwargs['process']) except: raise ValueError('Could not find process in database: {}'.format( kwargs['process'])) try: prod_id = db.getProductID(kwargs['product']) except: raise ValueError('Could not find product in database: {}'.format( kwargs['product'])) if kwargs['yesterday']: yesterday = 1 else: