Ejemplo n.º 1
0
def __run_script__(fns):
    global Plot4,Plot5,Plot6
    from Reduction import reduction,AddCifMetadata
 
    from os.path import basename
    from os.path import join
    import time           #how fast are we going?
    from Formats import output
    
    elapsed = time.clock()
    print 'Started working at %f' % (time.clock()-elapsed)
    df.datasets.clear()
    
    # check input
    if (fns is None or len(fns) == 0) :
        print 'no input datasets'
        return

    # check if input needs to be normalized
    if norm_apply.value:
        # norm_ref is the source of information for normalisation
        # norm_tar is the value norm_ref should become,
        # by multiplication.  If 'auto', the maximum value of norm_ref
        # for the first dataset is used, otherwise any number may be entered.
        norm_ref = str(norm_reference.value)
        norm_tar = str(norm_target.value).lower()

        # check if normalization target needs to be determined
        if len(norm_tar) == 0:
            norm_ref = None
            norm_tar = None
            print 'WARNING: no reference for normalization was specified'
        elif norm_tar == 'auto':
            # set flag
            norm_tar = -1
            # iterate through input datasets
            location = norm_table[norm_ref]     
            print 'utilized reference value for "' + norm_ref + '" is:', norm_tar
            
        # use provided reference value
        else:
            norm_tar = float(norm_tar)
            
    else:
        norm_ref = None
        norm_tar = None

    # check if eff-map needs to be loaded
    if eff_apply.value:
        if not eff_map.value:
            eff = None
            print 'WARNING: no eff-map was specified'
        else:
            eff_map_canonical = str(eff_map.value)
            if eff_map_canonical[0:5] != 'file:':
                eff_map_canonical = 'file:' + eff_map_canonical
            if not eff_map_canonical in eff_map_cache:
                eff_map_cache[eff_map_canonical] = reduction.read_efficiency_cif(eff_map_canonical)
            else:
                print 'Found in cache ' + `eff_map_canonical`
        eff = eff_map_cache[eff_map_canonical]
    else:
        eff = None
    
    # check if vertical tube correction needs to be loaded
    if vtc_apply.value:
        if not vtc_file.value:
            vtc = None
            print 'WARNING: no vtc-file was specified'
        else:
            vtc = str(vtc_file.value)
    else:
        vtc = None
    
    # iterate through input datasets
    # note that the normalisation target (an arbitrary number) is set by
    # the first dataset unless it has already been specified.
    for fn in fns:
        # load dataset
        ds = df[fn]
        # extract basic metadata
        ds = reduction.AddCifMetadata.extract_metadata(ds)
        # remove redundant dimensions
        rs = ds.get_reduced()
        rs.copy_cif_metadata(ds)
        # check if normalized is required 
        if norm_ref:
            ds,norm_tar = reduction.applyNormalization(rs, reference=norm_table[norm_ref], target=norm_tar)
        
        print 'Finished normalisation at %f' % (time.clock()-elapsed)
        # check if vertical tube correction is required
        if vtc:
            ds = reduction.getVerticallyCorrected(ds, vtc)
        print 'Finished vertical offset correction at %f' % (time.clock()-elapsed)
        # check if efficiency correction is required
        if eff:
            ds = reduction.getEfficiencyCorrected(ds, eff)
        
        print 'Finished efficiency correction at %f' % (time.clock()-elapsed)
        # check if we are recalculating gain
        if regain_apply.value:
            b = ds.intg(axis=1).get_reduced()  #reduce dimension
            ignore = regain_ignore.value    #Ignore first two tubes
            # Determine pixels per tube interval
            tube_pos = ds.axes[-1]
            tubesep = abs(tube_pos[0]-tube_pos[-1])/(len(tube_pos)-1)
            tube_steps = ds.axes[0]
            bin_size = abs(tube_steps[0]-tube_steps[-1])/(len(tube_steps)-1)
            pixel_step = int(round(tubesep/bin_size))
            bin_size = tubesep/pixel_step
            print '%f tube separation, %d steps before overlap, ideal binsize %f' % (tubesep,pixel_step,bin_size)
            # Reshape with individual sections summed
            c = b.reshape([b.shape[0]/pixel_step,pixel_step,b.shape[-1]])
            print `b.shape` + "->" + `c.shape`
            # sum the individual unoverlapped sections
            d = c.intg(axis=1)
            e = d.transpose()
            # we skip the first tubes' data as it is all zero
            # Get an initial average to start with
            bottom = vig_lower_boundary.value
            top = vig_upper_boundary.value
            resummed = ds[:,bottom:top,:]
            resummed = resummed.intg(axis=1).get_reduced()
            first_gain = array.ones(len(b.transpose())-ignore)
            first_ave,x,first_var = overlap.apply_gain(resummed.transpose()[ignore:,:],1.0/resummed.transpose().var[ignore:,:],pixel_step,first_gain, calc_var=True)
            if regain_unit_weights.value is True:
                weights = array.ones_like(e[ignore:])
            else:
                weights = 1.0/e[ignore:].var
            q= iterate_data(e[ignore:],weights,pixel_step=1,iter_no=int(regain_iterno.value))
            # Now we actually apply the vertical limits requested
           
            f,x, varf = overlap.apply_gain(resummed.transpose()[ignore:,:],1.0/resummed.transpose().var[ignore:,:],pixel_step,q[0],calc_var=True)
            # Get error for full dataset
            esds = overlap.calc_error_new(b.transpose()[ignore:,:],f,q[0],pixel_step)
            f = Dataset(f)
            f.title = "After scaling"
            f.var = varf
            # construct the ideal axes
            axis = arange(len(f))
            f.axes[0] = axis*bin_size + ds.axes[0][0] + ignore*pixel_step*bin_size
            f.copy_cif_metadata(ds)
            print `f.shape` + ' ' + `x.shape`
            Plot1.set_dataset(f)
            first_ave = Dataset(first_ave)
            first_ave.var = first_var
            first_ave.title = "Before scaling"
            first_ave.axes[0] = f.axes[0]
            Plot1.add_dataset(Dataset(first_ave))
            Plot4.set_dataset(Dataset(q[4]))
            fg = Dataset(q[0])
            fg.var = esds
            Plot5.set_dataset(fg)
            # show old esds
            fgold = Dataset(q[0])
            fgold.var = q[5]
            Plot5.add_dataset(fgold)
            residual_map = Dataset(q[3])
            try:
                Plot6.set_dataset(residual_map)
            except:
                pass
        print 'Finished regain calculation at %f' % (time.clock() - elapsed)
        # Output datasets
        filename_base = join(str(out_folder.value),str(output_stem.value) + basename(str(fn))[:-7])
        if output_cif.value:
            output.write_cif_data(f,filename_base)
        if output_xyd.value:
            output.write_xyd_data(f,filename_base)
        if output_fxye.value:
            output.write_fxye_data(f,filename_base)
        print 'Finished writing data at %f' % (time.clock()-elapsed)
Ejemplo n.º 2
0
def do_overlap(ds,iterno,algo="FordRollett"):
    import time
    from Reduction import overlap
    b = ds.intg(axis=1).get_reduced()  
    # Determine pixels per tube interval
    tube_pos = ds.axes[-1]
    tubesep = abs(tube_pos[0]-tube_pos[-1])/(len(tube_pos)-1)
    tube_steps = ds.axes[0]
    bin_size = abs(tube_steps[0]-tube_steps[-1])/(len(tube_steps)-1)
    pixel_step = int(round(tubesep/bin_size))
    print '%d steps before overlap' % pixel_step
    # Reshape with individual sections summed
    c = b.reshape([b.shape[0]/pixel_step,pixel_step,b.shape[-1]])
    print `b.shape` + "->" + `c.shape`
    # sum the individual unoverlapped sections
    d = c.intg(axis=1)
    e = d.transpose()
    # we skip the first two tubes' data as it is all zero
    gain,dd,interim_result,residual_map,chisquared,oldesds,first_ave = \
        iterate_data(e[3:],pixel_step=1,iter_no=iterno)
    print 'Have gains at %f' % time.clock()
    # calculate errors based on full dataset
    # First get a full model
    model,wd,mv = overlap.apply_gain(b.transpose()[3:],b.var.transpose()[3:],pixel_step,gain)
    esds = overlap.calc_error_new(b.transpose()[3:],model,gain,pixel_step)
    print 'Have full model and errors at %f' % time.clock()
    """ The following lines are intended to improve efficiency
    by a factor of about 10, by using Arrays instead of datasets
    and avoiding the [] operator, which currently involves too
    many lines of Python code per invocation. Note that 
    ArraySectionIter.next() is also code heavy, so calculate the
    sections ourselves."""
    final_gains = array.ones(ds.shape[-1])
    final_gains[2:] = gain
    final_errors = array.zeros(ds.shape[-1])
    final_errors[2:] = esds
    ds_as_array = ds.storage
    rs = array.zeros_like(ds)
    rs_var = array.zeros_like(ds)
    gain_iter = final_gains.__iter__()
    gain_var_iter = final_errors.__iter__()
    print 'RS shape: ' + `rs.shape`
    print 'Gain shape: ' + `final_gains.shape`
    target_shape = [rs.shape[0],rs.shape[1],1]
    for atubeno in range(len(final_gains)):
        rta = rs.get_section([0,0,atubeno],target_shape)
        dta = ds_as_array.get_section([0,0,atubeno],target_shape)
        fgn = gain_iter.next()
        fgvn = gain_var_iter.next()
        rta += dta * fgn
        rtav = rs_var.get_section([0,0,atubeno],target_shape)
        # sigma^2(a*b) = a^2 sigma^2(b) + b^2 sigma^2(a)
        rtav += ds.var.storage.get_section([0,0,atubeno],target_shape)*fgn*fgn + \
                fgvn * dta**2
    # Now build up the important information
    cs = copy(ds)
    cs.storage = rs
    cs.var = rs_var
    cs.copy_cif_metadata(ds)
    # prepare info for CIF file
    import math
    detno = map(lambda a:"%d" % a,range(len(final_gains)))
    gain_as_strings = map(lambda a:"%.4f" % a,final_gains)
    gain_esd = map(lambda a:"%.4f" % math.sqrt(a),final_errors)
    cs.harvest_metadata("CIF").AddCifItem((
        (("_[local]_detector_number","_[local]_refined_gain","_[local]_refined_gain_esd"),),
        ((detno,gain_as_strings,gain_esd),))
        )
    return cs,gain,esds,chisquared