示例#1
0
文件: test_qc.py 项目: zqtzt/MarineQC
 def test_one_year_difference(self):
     j1 = qc.jul_day(2001, 1, 1)
     j2 = qc.jul_day(2002, 1, 1)
     self.assertEqual(365.0, j2 - j1)
示例#2
0
文件: test_qc.py 项目: zqtzt/MarineQC
 def test_one_leapyear_difference(self):
     j1 = qc.jul_day(2004, 1, 1)
     j2 = qc.jul_day(2005, 1, 1)
     self.assertEqual(366.0, j2 - j1)
示例#3
0
文件: test_qc.py 项目: zqtzt/MarineQC
 def test_one_day_difference(self):
     j1 = qc.jul_day(2001, 1, 1)
     j2 = qc.jul_day(2001, 1, 2)
     self.assertEqual(1.0, j2 - j1)
def add_marine_report_to_db(cursor,year,rep):
    '''
    Add a marine report to the data base
    
    :param cursor: Database cursor for adding the report to the database
    :param year: year of the observation
    :param rep: :class:`.MarineReport`
    :type cursor: database cursor
    :type year: integer
    :type rep: :class:`.MarineReport`
    '''
    
    #calculate the Julian day of the observattion   
    jultime = None
    if rep.year != None and rep.month != None and rep.day != None:
        jultime = qc.jul_day(rep.year,rep.month,rep.day)

    cursor.execute('INSERT INTO marinereports'+str(year)+
                   ' (uid,id, year,month,day,hour,lat,lon,sst,mat,c1,si,dck,sid,pt,icoads_ds,icoads_vs,jul_day)'+
                   ' VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
                   (rep.uid, rep.id,  rep.year, rep.month, rep.day, 
                    rep.hour, rep.lat, rep.lon,  rep.sst,  rep.mat,
                    rep.c1,   rep.si,  rep.dck,  rep.sid,   rep.pt, 
                    rep.ds,   rep.vs,  jultime))

#initialise QC flags to be null - neither pass nor fail
    cursor.execute('INSERT INTO base_qc'+str(year)+
                   ' (uid,bad_date,bad_position,land,bad_track,fewsome_check,day_check,blacklist)'+
                   ' VALUES (%s,%s,%s,%s,%s,%s,%s,%s)'
                   ,(rep.uid, rep.bad_date, rep.bad_position, 
                     rep.land, rep.bad_track, rep.fewsome_check, 
                     rep.day_check, rep.blacklist))
    
    cursor.execute('INSERT INTO sst_qc'+str(year)+
                   ' (uid, sst_buddy_fail, no_sst, sst_below_freezing, no_sst_normal, sst_climatology_fail)'+
                   ' VALUES (%s,%s,%s,%s,%s,%s)'
                   ,(rep.uid, rep.sst_buddy_fail, rep.no_sst, 
                     rep.sst_below_freezing, rep.no_sst_normal, 
                     rep.sst_climatology_fail))
    
    cursor.execute('INSERT INTO mat_qc'+str(year)+
                   ' (uid, mat_buddy_fail, no_mat, no_mat_normal, mat_climatology_fail)'+
                   ' VALUES (%s,%s,%s,%s,%s)'
                   ,(rep.uid, rep.mat_buddy_fail, rep.no_mat, 
                     rep.no_mat_normal, rep.mat_climatology_fail))

    cursor.execute('INSERT INTO extra_qc'+str(year)+
                   ' (uid,new_track_check,fewsome_check,new_buddy_check,'+
                   'bayesian_sst_buddy_check,bayesian_mat_buddy_check,'+
                   'bayesian_track_check,spike_check,iquam_spike_check,repeated_value)'+
                   ' VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                   ,(rep.uid, 0, 0, 0, 0, 0, 0, 0, 0, 0))

#set some default values for the ob extra table from Kent and Berry ASMOS report
    if rep.pt <= 5:
        cursor.execute('INSERT INTO ob_extras'+str(year)+
                       ' (uid,'+
                       ' random_unc_sst, micro_bias_sst, micro_bias_sst_unc, bias_sst,bias_unc_sst,sst_norm,'+
                       ' random_unc_mat, micro_bias_mat, micro_bias_mat_unc, bias_mat,bias_unc_mat,mat_norm)'+
                       ' VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                       ,(rep.uid, 
                         0.7, 0.0, 0.8, 0.0, 0.0, rep.sst_norm, 
                         0.9, 0.0, 0.6, 0.0, 0.0, rep.mat_norm))
#Moored buoy
    if rep.pt == 7:
        cursor.execute('INSERT INTO ob_extras'+str(year)+
                       ' (uid,'+
                       ' random_unc_sst, micro_bias_sst, micro_bias_sst_unc, bias_sst,bias_unc_sst,sst_norm,'+
                       ' random_unc_mat, micro_bias_mat, micro_bias_mat_unc, bias_mat,bias_unc_mat,mat_norm)'+
                       ' VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                       ,(rep.uid, 
                         0.5, 0.0, 0.3, 0.0, 0.0, rep.sst_norm, 
                         0.3, 0.0, 0.5, 0.0, 0.0, rep.mat_norm))
#Drifting buoy
    if rep.pt == 6:
        cursor.execute('INSERT INTO ob_extras'+str(year)+
                       ' (uid,'+
                       ' random_unc_sst, micro_bias_sst, micro_bias_sst_unc, bias_sst,bias_unc_sst,sst_norm,'+
                       ' random_unc_mat, micro_bias_mat, micro_bias_mat_unc, bias_mat,bias_unc_mat,mat_norm)'+
                       ' VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                       ,(rep.uid, 
                         0.5, 0.0, 0.3, 0.0, 0.0, rep.sst_norm, 
                         1.0, 0.0, 0.9, 0.0, 0.0, rep.mat_norm))

    return
def add_multiple_marine_reports_to_db(cursor, year, reps):

    datas_marinereports = []
    datas_base_qc = []
    datas_sst_qc = []
    datas_mat_qc = []
    datas_ob_extras = []
    datas_extra_qc = []

    for rep in reps:
        jultime = None
        if rep.year != None and rep.month != None and rep.day != None:
            jultime = qc.jul_day(rep.year,rep.month,rep.day)
 
        datas_marinereports.append((rep.uid, rep.id,  rep.year, rep.month, rep.day, 
                                    rep.hour, rep.lat, rep.lon,  rep.sst,  rep.mat,
                                    rep.c1,   rep.si,  rep.dck,  rep.sid,   rep.pt, 
                                    rep.ds,   rep.vs,  jultime))

        datas_base_qc.append((rep.uid, rep.bad_date, rep.bad_position, 
                             rep.land, rep.bad_track, rep.fewsome_check, 
                             rep.day_check, rep.blacklist))

        datas_sst_qc.append((rep.uid, rep.sst_buddy_fail, rep.no_sst, 
                             rep.sst_below_freezing, rep.no_sst_normal, 
                             rep.sst_climatology_fail))

        datas_mat_qc.append((rep.uid, rep.mat_buddy_fail, rep.no_mat, 
                             rep.no_mat_normal, rep.mat_climatology_fail))

        datas_extra_qc.append((rep.uid, 0, 0, 0, 0, 0, 0, 0, 0, 0))
        
#ship       
        if rep.pt <= 5:
            datas_ob_extras.append((rep.uid, 
                                    0.7, 0.0, 0.8, 0.0, 0.0, rep.sst_norm, 
                                    0.9, 0.0, 0.6, 0.0, 0.0, rep.mat_norm))
#Moored buoy
        if rep.pt == 7:
            datas_ob_extras.append((rep.uid, 
                                    0.5, 0.0, 0.3, 0.0, 0.0, rep.sst_norm, 
                                    0.3, 0.0, 0.5, 0.0, 0.0, rep.mat_norm))
#Drifting buoy
        if rep.pt == 6:
            datas_ob_extras.append((rep.uid, 
                                    0.5, 0.0, 0.3, 0.0, 0.0, rep.sst_norm, 
                                    1.0, 0.0, 0.9, 0.0, 0.0, rep.mat_norm))

    cursor.executemany('INSERT INTO marinereports'+str(year)+
                   ' (uid,id, year,month,day,hour,lat,lon,sst,mat,c1,si,dck,sid,pt,icoads_ds,icoads_vs,jul_day)'+
                   ' VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
                    datas_marinereports)

    cursor.executemany('INSERT INTO base_qc'+str(year)+
                   ' (uid,bad_date,bad_position,land,bad_track,fewsome_check,day_check,blacklist)'+
                   ' VALUES (%s,%s,%s,%s,%s,%s,%s,%s)'
                   ,datas_base_qc)
    
    cursor.executemany('INSERT INTO sst_qc'+str(year)+
                   ' (uid, sst_buddy_fail, no_sst, sst_below_freezing, no_sst_normal, sst_climatology_fail)'+
                   ' VALUES (%s,%s,%s,%s,%s,%s)'
                   ,datas_sst_qc)
    
    cursor.executemany('INSERT INTO mat_qc'+str(year)+
                   ' (uid, mat_buddy_fail, no_mat, no_mat_normal, mat_climatology_fail)'+
                   ' VALUES (%s,%s,%s,%s,%s)'
                   ,datas_mat_qc)

    cursor.executemany('INSERT INTO extra_qc'+str(year)+
                   ' (uid,new_track_check,fewsome_check,new_buddy_check,'+
                   'bayesian_sst_buddy_check,bayesian_mat_buddy_check,'+
                   'bayesian_track_check,spike_check,iquam_spike_check,repeated_value)'+
                   ' VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                   ,datas_extra_qc)
    
    cursor.executemany('INSERT INTO ob_extras'+str(year)+
                   ' (uid,'+
                   ' random_unc_sst, micro_bias_sst, micro_bias_sst_unc, bias_sst,bias_unc_sst,sst_norm,'+
                   ' random_unc_mat, micro_bias_mat, micro_bias_mat_unc, bias_mat,bias_unc_mat,mat_norm)'+
                   ' VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                   ,datas_ob_extras)


    return
def main(argv):
    '''
    The buddy check compares observations to other nearby observations. If the observation differs 
    substantially from the neighbour-average, the observation will be rejected.
    '''

    print '###################'
    print 'Running buddy_check'
    print '###################'
    
    inputfile = 'configuration.txt'

    try:
        opts, args = getopt.getopt(argv, "hi:", 
                                   ["ifile=", 
                                    "year1=", 
                                    "year2="])
    except getopt.GetoptError:
        print 'Usage Make_DB.py -i <configuration_file> '+\
        '--year1 <start year> --year2 <end year>'
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print 'test.py -i <inputfile> -o <outputfile>'
            sys.exit()
        elif opt in ("-i", "--ifile"):
            inputfile = arg
        elif opt in ("-x", "--year1"):
            try:
                year1 = int(arg)
            except:
                sys.exit("Failed: year1 not an integer")
        elif opt in ("-y", "--year2"):
            try:
                year2 = int(arg)
            except:
                sys.exit("Failed: year2 not an integer")

    print 'Input file is ', inputfile
    print 'Running from ', year1, ' to ', year2
    print ''

    config = qc.get_config(inputfile)

    sst_climatology_file  = config['SST_climatology'] 
    nmat_climatology_file = config['MAT_climatology'] 
    icoads_dir            = config['ICOADS_dir'] 
    sst_stdev_climatology_file  = config['Old_SST_stdev_climatology']
    data_base_host        = config['data_base_host']
    data_base_name        = config['data_base_name'] 

    print 'Data base host =', data_base_host
    print 'Data base name =', data_base_name
    print 'SST climatology =', sst_climatology_file
    print 'NMAT climatology =', nmat_climatology_file
    print 'ICOADS directory =', icoads_dir
    print ''

#read in the pentad climatology of standard deviations
    climatology = Dataset(sst_stdev_climatology_file)
    sst_pentad_stdev = climatology.variables['sst'][:]

    connection = MySQLdb.connect(host=data_base_host, 
                                 user='******',
                                 db=data_base_name)
    cursor  = connection.cursor() #read
    cursor2 = connection.cursor() #write
    
    for years, months in qc.year_month_gen(year1, 1, year2, 12):

#want to get a month either side of the 
#target month, which may be in different years
        last_year, last_month = qc.last_month_was(years, months)
        next_year, next_month = qc.next_month_is(years, months)
        
        print years, months
        
        first_year = min([last_year, years, next_year])
        final_year = max([last_year, years, next_year])
        
        if first_year < 1850:
            first_year = 1850
        if final_year > 2014:
            final_year = 2014

#first and last julian days are +- approximately one month
        month_lengths = qc.month_lengths(years)
        jul1 = qc.jul_day(years, months, 1)-25
        jul2 = qc.jul_day(years, months, month_lengths[months-1])+25
        
        for check_variable in ['SST','MAT']:
        
            reps = []
            for yyy in range(first_year, final_year+1):
                
                qcfilter = db.Quality_Control_Filter()
                qcfilter.jul1 = jul1
                qcfilter.jul2 = jul2
                qcfilter.set_multiple_qc_flags_to_pass(['bad_position',
                                                        'bad_date',
                                                        'blacklist'])
                
                if check_variable == 'SST':
                    qcfilter.set_multiple_qc_flags_to_pass(['no_sst',
                                                            'sst_below_freezing',
                                                            'no_sst_normal',
                                                            'sst_climatology_fail'])
                elif check_variable == 'MAT':
                    qcfilter.set_multiple_qc_flags_to_pass(['no_mat',
                                                            'no_mat_normal',
                                                            'mat_climatology_fail'])
                else:
                    print "no such type ", check_variable
                    assert False

                sql_request = db.build_sql_query(yyy, qcfilter)
                
                cursor.execute(sql_request)
                numrows = cursor.rowcount

                for i in range(numrows):
                    rows = cursor.fetchone()
                    rep = qc.MarineReport.report_from_array(rows)
                    reps.append(rep)

            print len(reps)," observations read in"

#Do the buddy check
            if check_variable == 'SST':
                qcs = qc_buddy_check.mds_buddy_check(reps, 
                                                     sst_pentad_stdev, 
                                                     'SST')
            elif check_variable == 'MAT':
                qcs = qc_buddy_check.mds_buddy_check(reps, 
                                                     sst_pentad_stdev, 
                                                     'MAT')
            else:
                print "no such type ", check_variable
                assert False

#put updated QC flags into data base
            for rep in reps:
                if rep.month == months:
                    if check_variable == 'SST':
                        result = db.update_db_qc_single_flag(rep,
                                                             rep.sst_buddy_fail,
                                                             'sst_qc',
                                                             'sst_buddy_fail',
                                                             years,
                                                             cursor2)
                    elif check_variable == 'MAT':
                        result = db.update_db_qc_single_flag(rep,
                                                             rep.mat_buddy_fail,
                                                             'mat_qc',
                                                             'mat_buddy_fail',
                                                             years,
                                                             cursor2)
                    else:
                        print "no such type ", check_variable
                        assert False

            print "Of "+str(len(qcs))+" observations "+\
            str(np.sum(qcs))+" failed "+check_variable+\
            " buddy check"

        connection.commit() #Each month
        #db.report_qc_counts(cursor, years, months)

    connection.close()

    
    print "All Done :)"
def main(argv):
    
    '''
    The new track check program. First the program gets a list of all unique IDs in the month 
    that is to be track checked. It then reads in three months of data at a time: the month 
    you want to track check, a month before and a month after. For each unique ID, the track 
    check is run.
    
    Track check comprises as set of related tests
    
    This program checks positional data for individual ships and buoys for internal consistency; 
    checking reported positions against positions calculated using reported speeds and directions.
    
    The obs are sorted by call-sign then date. Obs can only be checked if they have a valid call-sign 
    that is unique to one ship or buoy, so obs with no call-sign or with the generic call-signs 'SHIP' 
    or 'PLAT' are passed unchecked. The call-sign '0102' was apparently shared by several ships, so obs 
    with this call-sign are also passed unchecked.
    '''
    
    print '###################'
    print 'Running New Track Check'
    print '###################'
    
    inputfile = 'configuration.txt'
    
    try:
        opts, args = getopt.getopt(argv, 
                                   "hi:", 
                                   ["ifile=", 
                                    "year1=", 
                                    "year2="])
    except getopt.GetoptError:
        print 'Usage Make_DB.py -i <configuration_file>'+\
        ' --year1 <start year> --year2 <end year>'
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print 'test.py -i <inputfile> -o <outputfile>'
            sys.exit()
        elif opt in ("-i", "--ifile"):
            inputfile = arg
        elif opt in ("-x", "--year1"):
            try:
                year1 = int(arg)
            except:
                sys.exit("Failed: year1 not an integer")
        elif opt in ("-y", "--year2"):
            try:
                year2 = int(arg)
            except:
                sys.exit("Failed: year2 not an integer")
                
    print 'Input file is ', inputfile
    print 'Running from ', year1, ' to ', year2
    print ''
    
    config = qc.get_config(inputfile)
    
    data_base_host        = config['data_base_host']
    data_base_name        = config['data_base_name'] 

    print 'Data base host =', data_base_host
    print 'Data base name =', data_base_name
 
    print ''

    connection = MySQLdb.connect(host=data_base_host, 
                                 user='******',
                                 db=data_base_name)

    #need two cursors, one for reading and one for making QC changes
    cursor = connection.cursor()
    cursor2 = connection.cursor()
    
    t00 = time.time()
    
    for years, months in qc.year_month_gen(year1, 1, year2, 12):
    
    #want to get a month either side of the target month, 
    #which may be in different years
        last_year, last_month = qc.last_month_was(years, months)
        next_year, next_month = qc.next_month_is(years, months)
        
        print years, months
    
        t0 = time.time()
        
        first_year = min([last_year, years, next_year])
        final_year = max([last_year, years, next_year])
    
        if first_year < 1850:
            first_year = 1850
        if final_year > 1990:
            final_year = 1990
    
    #first and last julian days are +- approximately one month
        month_lengths = qc.month_lengths(years)
        jul1 = qc.jul_day(years, months, 1)-10
        jul2 = qc.jul_day(years, months, month_lengths[months-1])+10
        
        '''Get all unique IDs for this month and fill a dictionary 
        with all the distinct ids that we want to QC as keys and an 
        empty Voyage for each key'''            
        allids = db.get_unique_ids(cursor, years, months)
        reps = {}
        for idrows in allids:
            thisid = idrows[0]
            reps[thisid] = qc.Voyage()
        
        t1 = time.time()
        print "got all IDs ",t1-t0
        
    #extract all data for this month and a month either side
        for yyy in range(first_year, final_year+1):
            
            '''
            Build filter for extracting data from data base and then extract. 
            In this case, we want observations between jul1 and jul2 which pass 
            the base QC checks. 
            '''
            qcfilter = db.Quality_Control_Filter()
            qcfilter.jul1 = jul1
            qcfilter.jul2 = jul2
            qcfilter.set_multiple_qc_flags_to_pass(['bad_position',
                                                    'bad_date',
                                                    'blacklist'])
            
            sql_request = db.build_sql_query(yyy, qcfilter)
            
            cursor.execute(sql_request)
            numrows = cursor.rowcount

    #put each ob into the dictionary if there is a key for it
            for i in range(numrows):
                rows = cursor.fetchone()
                rep = qc.ExtendedMarineReport.report_from_array(rows)
                if rep.id in reps:
                    reps[rep.id].add_report(rep)

        t2 = time.time()
        print "read all obs from DB",t2-t1

    #loop over all the distinct callsigns, extract the obs 
    #where the callsign matches and track check them
        for idrows in allids:
            thisid = idrows[0]
            matches = reps[thisid]
            matches.sort()

#run improved track check with spherical geometry etc.
            mqcs = qc_new_track_check.mds_full_track_check(matches)
            matches.find_repeated_values()

            for rep in matches.reps:
                if rep.month == months:
                    result = db.update_db_qc_single_flag(rep,rep.bad_track,
                                                         'extra_qc',
                                                         'bayesian_track_check',
                                                         years,cursor2)
                    result = db.update_db_qc_single_flag(rep,rep.repeated_value,
                                                         'extra_qc',
                                                         'repeated_value',
                                                         years,cursor2)

            split_matches = qc.split_generic_callsign(matches)

            for split in split_matches:
                qcs = qc_new_track_check.mds_full_track_check(split)

#update QC in the data base but only for the target month
                for i, rep in enumerate(split.reps):
                    if rep.month == months:
                        result = db.update_db_qc_single_flag(rep,
                                                             qcs[i],
                                                             'extra_qc',
                                                             'new_track_check',
                                                             years,
                                                             cursor2)
                        result = db.update_db_qc_single_flag(rep,
                                                             rep.fewsome_check,
                                                             'base_qc',
                                                             'fewsome_check',
                                                             years,
                                                             cursor2)

        connection.commit()

        t3 = time.time()
        print "done ",t3-t2

        #db.report_qc_counts(cursor, years, months)
    
    connection.close()
    
    print "All Done :)"