Ejemplo n.º 1
0
def pyobjs_to_h5(g4tracks, pixnoise, filename, v):
    """
    Given the python object lists, write to HDF5 file at filename.
    """

    with h5py.File(filename, "w", libver="latest", driver="core") as f:
        for ind in range(len(g4tracks)):
            pyobj_to_h5 = {}
            indstr = "{:05d}".format(ind)
            g = f.create_group(indstr)

            # write g4track
            if g4tracks[ind]:
                filejob.vprint(v, 3, "Writing " + str(ind) + " g4track at " + time.ctime())
                trackio.write_object_to_hdf5(g4tracks[ind], g, "g4track", pyobj_to_h5=pyobj_to_h5)

            # write each pixnoise
            for key, val in pixnoise.iteritems():
                filejob.vprint(v, 4, "Writing " + str(ind) + " " + key + " at " + time.ctime())
                if isinstance(val[ind], trackdata.Track):
                    trackio.write_object_to_hdf5(val[ind], g, key, pyobj_to_h5=pyobj_to_h5)
                else:
                    # track error code
                    t = g.create_group(key)
                    t.attrs["errorcode"] = val[ind]
Ejemplo n.º 2
0
def pyobjs_to_h5(g4tracks, pixnoise, filename, v):
    """
    Given the python object lists, write to HDF5 file at filename.
    """

    with h5py.File(filename, 'w', libver='latest', driver='core') as f:
        for ind in range(len(g4tracks)):
            pyobj_to_h5 = {}
            indstr = '{:05d}'.format(ind)
            g = f.create_group(indstr)

            # write g4track
            if g4tracks[ind]:
                filejob.vprint(
                    v, 3,
                    'Writing ' + str(ind) + ' g4track at ' + time.ctime())
                trackio.write_object_to_hdf5(g4tracks[ind],
                                             g,
                                             'g4track',
                                             pyobj_to_h5=pyobj_to_h5)

            # write each pixnoise
            for key, val in pixnoise.iteritems():
                filejob.vprint(
                    v, 4,
                    'Writing ' + str(ind) + ' ' + key + ' at ' + time.ctime())
                if isinstance(val[ind], trackdata.Track):
                    trackio.write_object_to_hdf5(val[ind],
                                                 g,
                                                 key,
                                                 pyobj_to_h5=pyobj_to_h5)
                else:
                    # track error code
                    t = g.create_group(key)
                    t.attrs['errorcode'] = val[ind]
Ejemplo n.º 3
0
def pyobj_from_h5(h5filename, v):
    """
    Given a file in h5matlab format, return list of objects
      (trackdata.G4Track and trackdata.Track)

    h5filename: complete filename string
    """

    n_to_run = 10000
    n_run = 0

    with h5py.File(h5filename, 'r', driver='core') as f:
        # get max index number
        indices = []
        for key in f.keys():
            indices.append(int(key))
        max_index = np.max(np.array(indices))
        filejob.vprint(v, 3, '\n Got max_index at ' + time.ctime())

        # initialize array of g4tracks, and dict of diffused tracks
        g4tracks = [[] for i in range(max_index + 1)]
        pixnoise = {}

        # construct tracks
        for track_key in f.keys():
            if n_run > n_to_run:
                continue
            filejob.vprint(v, 3,
                           'Starting ' + track_key + ' at ' + time.ctime())
            n_run += 1

            evt = f[track_key]
            ind = int(track_key)
            g4tracks[ind] = trackdata.G4Track.from_h5matlab(evt)
            for pn_key in evt.keys():
                if not pn_key.startswith('pix'):
                    continue
                if pn_key not in pixnoise:
                    # initialize pixnoise dict entry
                    pixnoise[pn_key] = [[] for i in range(max_index + 1)]
                # add to existing dict
                this_track = trackdata.Track.from_h5matlab(
                    evt[pn_key], g4track=g4tracks[ind])
                # debug
                if this_track is None:
                    pdb.set_trace()
                pixnoise[pn_key][ind] = this_track

    return g4tracks, pixnoise
Ejemplo n.º 4
0
def pyobj_from_h5(h5filename, v):
    """
    Given a file in h5matlab format, return list of objects
      (trackdata.G4Track and trackdata.Track)

    h5filename: complete filename string
    """

    n_to_run = 10000
    n_run = 0

    with h5py.File(h5filename, "r", driver="core") as f:
        # get max index number
        indices = []
        for key in f.keys():
            indices.append(int(key))
        max_index = np.max(np.array(indices))
        filejob.vprint(v, 3, "\n Got max_index at " + time.ctime())

        # initialize array of g4tracks, and dict of diffused tracks
        g4tracks = [[] for i in range(max_index + 1)]
        pixnoise = {}

        # construct tracks
        for track_key in f.keys():
            if n_run > n_to_run:
                continue
            filejob.vprint(v, 3, "Starting " + track_key + " at " + time.ctime())
            n_run += 1

            evt = f[track_key]
            ind = int(track_key)
            g4tracks[ind] = trackdata.G4Track.from_h5matlab(evt)
            for pn_key in evt.keys():
                if not pn_key.startswith("pix"):
                    continue
                if pn_key not in pixnoise:
                    # initialize pixnoise dict entry
                    pixnoise[pn_key] = [[] for i in range(max_index + 1)]
                # add to existing dict
                this_track = trackdata.Track.from_h5matlab(evt[pn_key], g4track=g4tracks[ind])
                # debug
                if this_track is None:
                    pdb.set_trace()
                pixnoise[pn_key][ind] = this_track

    return g4tracks, pixnoise
Ejemplo n.º 5
0
def h5m_to_pyml(loadfile, savefile, v):
    """
    Main work
    """

    filejob.vprint(v, 2, "~ Loading " + loadfile + " at " + time.ctime() + " ~")
    g4tracks, pixnoise = pyobj_from_h5(loadfile, v)
    filejob.vprint(v, 1, "> Saving " + savefile + " at " + time.ctime() + " <")
    pyobjs_to_h5(g4tracks, pixnoise, savefile, v)
    filejob.vprint(v, 2, "= Finished " + savefile + " at " + time.ctime() + " =")
Ejemplo n.º 6
0
def h5m_to_pyml(loadfile, savefile, v):
    """
    Main work
    """

    filejob.vprint(v, 2,
                   '~ Loading ' + loadfile + ' at ' + time.ctime() + ' ~')
    g4tracks, pixnoise = pyobj_from_h5(loadfile, v)
    filejob.vprint(v, 1, '> Saving ' + savefile + ' at ' + time.ctime() + ' <')
    pyobjs_to_h5(g4tracks, pixnoise, savefile, v)
    filejob.vprint(v, 2,
                   '= Finished ' + savefile + ' at ' + time.ctime() + ' =')
Ejemplo n.º 7
0
def get_results(loadfile, savefile, v):
    """
    Get all the important numbers for each event.

    (basic info:)
        Etot
        Edep
        Etrack
        alpha_true
        beta_true
        errorcode
        file
        ind
    (ridge following result:)
        alpha_rf
    (moments result:)
        alpha_m
    (general rejection parameters:)
        min_end_energy
        max_end_energy
        n_ends
    (moments rejection parameters:)
        phi
        edge_pixels
        edge_segments
    (Monte Carlo rejection parameters:)
        overlap flag
        wrong_end flag
        early_scatter flag
        total_scatter_angle
    """

    progressflag = not file_vars()[0]

    pn = 'pix10_5noise15'
    HTname = 'python HT v1.52'
    MTname = 'moments v1.0'

    vprint(
        v, 1, 'Starting {} at {} with {}% mem usage'.format(
            loadfile, time.ctime(),
            psutil.virtual_memory().percent))

    datalen = 1000
    # basic info
    energy_tot_kev = np.nan * np.ones(datalen)
    energy_dep_kev = np.nan * np.ones(datalen)
    energy_track_kev = np.nan * np.ones(datalen)
    alpha_true_deg = np.nan * np.ones(datalen)
    beta_true_deg = np.nan * np.ones(datalen)
    trk_errorcode = np.nan * np.ones(datalen)
    cl_errorcode = np.nan * np.ones(datalen)
    mom_errorcode = np.nan * np.ones(datalen)
    filename = ['' for _ in xrange(datalen)]
    fileind = np.nan * np.ones(datalen)
    # ridge following
    alpha_ridge_deg = np.nan * np.ones(datalen)
    # moments
    alpha_moments_deg = np.nan * np.ones(datalen)
    # rejection
    min_end_energy_kev = np.nan * np.ones(datalen)
    max_end_energy_kev = np.nan * np.ones(datalen)
    n_ends = np.nan * np.ones(datalen)
    phi_deg = np.nan * np.ones(datalen)
    edge_pixels = np.nan * np.ones(datalen)
    edge_segments = np.nan * np.ones(datalen)
    # monte carlo
    overlap_flag = np.nan * np.ones(datalen)
    wrong_end_flag = np.nan * np.ones(datalen)
    early_scatter_flag = np.nan * np.ones(datalen)
    total_scatter_angle_deg = np.nan * np.ones(datalen)

    try:
        with h5py.File(loadfile, 'r',
                       driver='core') as f, h5py.File(savefile,
                                                      'a',
                                                      driver='core') as h5save:
            if progressflag:
                pbar = progressbar.ProgressBar(widgets=[
                    progressbar.Percentage(), ' ',
                    progressbar.Bar(), ' ',
                    progressbar.ETA()
                ],
                                               maxval=datalen)
                pbar.start()

            for ind in xrange(datalen):
                indstr = '{:05d}'.format(ind)
                trkstr = indstr
                clstr = 'cl_' + indstr
                momstr = 'mom_' + indstr

                this_trk_errorcode = 0
                this_cl_errorcode = 0
                this_mom_errorcode = 0

                filename[ind] = os.path.split(loadfile)[-1]
                fileind[ind] = ind

                # track and algorithm alpha's
                try:
                    this_trk = Track.from_hdf5(f[trkstr][pn])
                except KeyError:
                    # could be an MException from MATLAB
                    this_trk_errorcode = 10
                    trk_errorcode[ind] = this_trk_errorcode
                    continue
                except trackio.InterfaceError:
                    read_errorcode = f[trkstr][pn].attrs['errorcode']
                    if read_errorcode > 0:
                        # multiplicity event
                        this_trk_errorcode = read_errorcode
                        trk_errorcode[ind] = this_trk_errorcode
                        continue
                    else:
                        raise
                energy_tot_kev[ind] = this_trk.g4track.energy_tot_kev
                energy_dep_kev[ind] = this_trk.g4track.energy_dep_kev
                energy_track_kev[ind] = this_trk.energy_kev
                alpha_true_deg[ind] = this_trk.g4track.alpha_deg
                beta_true_deg[ind] = this_trk.g4track.beta_deg
                alpha_ridge_deg[ind] = this_trk[HTname].alpha_deg
                alpha_moments_deg[ind] = this_trk[MTname].alpha_deg

                # classifier
                this_cl = cl.Classifier.from_hdf5(f[clstr][pn])
                min_end_energy_kev[ind] = this_cl.min_end_energy
                max_end_energy_kev[ind] = this_cl.max_end_energy
                n_ends[ind] = this_cl.n_ends
                overlap_flag[ind] = this_cl.overlap
                wrong_end_flag[ind] = this_cl.wrong_end
                if this_cl.error == 'TrackTooShortError':
                    this_cl_errorcode = 8
                else:
                    early_scatter_flag[ind] = this_cl.early_scatter
                    total_scatter_angle_deg[ind] = (
                        this_cl.total_scatter_angle / np.pi * 180)

                # moments info
                try:
                    this_mom = tm.MomentsReconstruction.from_hdf5(
                        f[momstr][pn])
                except trackio.InterfaceError:
                    if f[momstr][pn].attrs['errorcode'] == 4:
                        # no ends found
                        this_mom_errorcode = 4
                    else:
                        raise
                else:
                    if this_mom.error == 'CheckSegmentBoxError':
                        this_mom_errorcode = 9
                    elif this_mom.error == 'what the heck happened?':
                        this_mom.errorcode = 11
                    elif this_mom.error == 'Rotation angle conditions not met':
                        this_mom.errorcode = 12
                    else:
                        phi_deg[ind] = this_mom.phi / np.pi * 180
                        edge_pixels[ind] = this_mom.edge_pixel_count
                        edge_segments[ind] = this_mom.edge_pixel_segments

                trk_errorcode[ind] = this_trk_errorcode
                cl_errorcode[ind] = this_cl_errorcode
                mom_errorcode[ind] = this_mom_errorcode

                if progressflag:
                    pbar.update(ind)

            # save to file
            # basic info
            h5save.create_dataset('energy_tot_kev',
                                  shape=(datalen, ),
                                  data=energy_tot_kev)
            h5save.create_dataset('energy_dep_kev',
                                  shape=(datalen, ),
                                  data=energy_dep_kev)
            h5save.create_dataset('energy_track_kev',
                                  shape=(datalen, ),
                                  data=energy_track_kev)
            h5save.create_dataset('alpha_true_deg',
                                  shape=(datalen, ),
                                  data=alpha_true_deg)
            h5save.create_dataset('beta_true_deg',
                                  shape=(datalen, ),
                                  data=beta_true_deg)
            h5save.create_dataset('trk_errorcode',
                                  shape=(datalen, ),
                                  data=trk_errorcode,
                                  dtype='f2')
            h5save.create_dataset('cl_errorcode',
                                  shape=(datalen, ),
                                  data=cl_errorcode,
                                  dtype='f2')
            h5save.create_dataset('mom_errorcode',
                                  shape=(datalen, ),
                                  data=mom_errorcode,
                                  dtype='f2')
            h5save.create_dataset('filename',
                                  shape=(datalen, ),
                                  data=filename,
                                  dtype=h5py.special_dtype(vlen=str))
            h5save.create_dataset('fileind',
                                  shape=(datalen, ),
                                  data=fileind,
                                  dtype='f4')
            # ridge following
            h5save.create_dataset('alpha_ridge_deg',
                                  shape=(datalen, ),
                                  data=alpha_ridge_deg)
            # moments
            h5save.create_dataset('alpha_moments_deg',
                                  shape=(datalen, ),
                                  data=alpha_moments_deg)
            # rejection parameters
            h5save.create_dataset('min_end_energy_kev',
                                  shape=(datalen, ),
                                  data=min_end_energy_kev)
            h5save.create_dataset('max_end_energy_kev',
                                  shape=(datalen, ),
                                  data=max_end_energy_kev)
            h5save.create_dataset('n_ends',
                                  shape=(datalen, ),
                                  data=n_ends,
                                  dtype='f4')
            h5save.create_dataset('phi_deg', shape=(datalen, ), data=phi_deg)
            h5save.create_dataset('edge_pixels',
                                  shape=(datalen, ),
                                  data=edge_pixels,
                                  dtype='f4')
            h5save.create_dataset('edge_segments',
                                  shape=(datalen, ),
                                  data=edge_segments,
                                  dtype='f2')
            # Monte Carlo
            h5save.create_dataset('overlap_flag',
                                  shape=(datalen, ),
                                  data=overlap_flag,
                                  dtype='f2')
            h5save.create_dataset('wrong_end_flag',
                                  shape=(datalen, ),
                                  data=wrong_end_flag,
                                  dtype='f2')
            h5save.create_dataset('early_scatter_flag',
                                  shape=(datalen, ),
                                  data=early_scatter_flag,
                                  dtype='f2')
            h5save.create_dataset('total_scatter_angle_deg',
                                  shape=(datalen, ),
                                  data=total_scatter_angle_deg)

            if progressflag:
                pbar.finish()

    except NotImplementedError:
        pass
Ejemplo n.º 8
0
def classify_etc(loadfile, savefile, v):
    """
    1. Moments algorithm
    2. HybridTrack algorithm
    3. Classify
    """

    progressflag = not file_vars()[0]

    pn = 'pix10_5noise15'
    HTname = 'python HT v1.52'
    MTname = 'moments v1.0'

    vprint(v, 1, 'Starting {} at {} with {}% mem usage'.format(
        loadfile, time.ctime(), psutil.virtual_memory().percent))

    pyobj_to_h5 = {}

    try:
        with h5py.File(loadfile, 'a', driver='core') as f, h5py.File(
                savefile, 'a', driver='core') as h5save:
            #
            n = 0
            if progressflag:
                pbar = progressbar.ProgressBar(
                    widgets=[progressbar.Percentage(), ' ',
                             progressbar.Bar(), ' ',
                             progressbar.ETA()], maxval=len(f))
                pbar.start()

            keylist = f.keys()
            keylist.sort()
            for ind in keylist:
                vprint(v, 3, 'Beginning track {} in {}'.format(ind, loadfile))
                if int(ind) % 50 == 0:
                    vprint(v, 2,
                           'Beginning track {} in {}'.format(ind, loadfile))

                # evt is for the G4Track object
                evt = f[ind]
                # shouldn't get a KeyError because we are looping over ind

                # trk is for the Track object
                try:
                    trk = evt[pn]
                except KeyError:
                    vprint(v, 1,
                           '**Missing key {} in {}{}, skipping'.format(
                               pn, loadfile, evt.name))
                    continue

                trkpath = ind + '/' + pn
                errorcode = 0
                n += 1

                if n > 50:
                    # testing
                    # vprint(v, 1, 'Finished 50 files, exiting')
                    # break
                    pass

                # load G4Track
                vprint(v, 3, 'Loading G4Track {} in {}'.format(ind, loadfile))
                try:
                    this_g4track = G4Track.from_dth5(evt)
                except trackio.InterfaceError:
                    vprint(v, 2, 'InterfaceError in G4Track at {}, {}'.format(
                        loadfile, ind))
                    continue

                # load Track
                vprint(v, 3, 'Loading Track {} in {}'.format(ind, loadfile))
                try:
                    this_track = Track.from_dth5(trk, g4track=this_g4track)
                except trackio.InterfaceError:
                    vprint(v, 2, 'InterfaceError in Track at {}, {}'.format(
                        loadfile, ind))
                    continue

                # check for error codes - Track.from_dth5 may return an int
                if isinstance(this_track, int):
                    this_save = h5save.create_group(trkpath)
                    this_save.attrs.create(
                        'errorcode', this_track, shape=np.shape(this_track))
                    continue

                # run moments algorithm
                vprint(v, 3, 'Running moments on track {} in {}'.format(
                    ind, loadfile))
                try:
                    mom = tm.MomentsReconstruction(this_track.image)
                    mom.reconstruct()
                except ht.NoEndsFound:
                    mom.alpha = None
                    errorcode = 4
                    this_save = h5save.create_group('mom_' + trkpath)
                    this_save.attrs.create(
                        'errorcode', errorcode, shape=np.shape(errorcode))
                else:
                    # write into savefile
                    trackio.write_object_to_hdf5(
                        mom, h5save, 'mom_' + trkpath, pyobj_to_h5=pyobj_to_h5)
                # write into track object
                if mom.alpha:
                    this_track.add_algorithm(
                        MTname,
                        alpha_deg=mom.alpha * 180 / np.pi,
                        beta_deg=np.nan, info=None)
                else:
                    this_track.add_algorithm(
                        MTname,
                        alpha_deg=np.nan,
                        beta_deg=np.nan, info=None)

                # run HT algorithm (v1.52)
                vprint(v, 3, 'Running HT on track {} in {}'.format(
                    ind, loadfile))
                try:
                    _, HTinfo = ht.reconstruct(this_track)
                except (ht.InfiniteLoop, ht.NoEndsFound):
                    # write empty into track object
                    this_track.add_algorithm(
                        HTname,
                        alpha_deg=np.nan,
                        beta_deg=np.nan,
                        info=None)
                else:
                    # trim memory usage
                    if hasattr(HTinfo, 'ridge'):
                        if HTinfo.ridge:
                            for ridgept in HTinfo.ridge:
                                ridgept.cuts = None
                                ridgept.best_cut = None
                    # write into track object
                    this_track.add_algorithm(
                        HTname,
                        alpha_deg=HTinfo.alpha_deg,
                        beta_deg=HTinfo.beta_deg,
                        info=HTinfo)

                # write py HDF5 format into savefile (including alg outputs)
                try:
                    trackio.write_object_to_hdf5(
                        this_track, h5save, trkpath,
                        pyobj_to_h5=pyobj_to_h5)
                except trackio.InterfaceError:
                    vprint(v, 1, 'InterfaceError writing to file at ' +
                           '{}, {}'.format(savefile, ind))
                    continue

                # run classifier
                vprint(v, 3, 'Running MC classifier on track {} in {}'.format(
                    ind, loadfile))
                classifier = cl.Classifier(this_track.g4track)
                try:
                    classifier.mc_classify()
                except cl.TrackTooShortError:
                    classifier.error = 'TrackTooShortError'
                else:
                    vprint(v, 3,
                           'Running ends classifier on track {} in {}'.format(
                               ind, loadfile))
                    try:
                        classifier.end_classify(this_track, mom=mom)
                    except tp.G4TrackTooBigError:
                        classifier.error = 'G4TrackTooBigError'
                    except cl.NoEnds:
                        classifier.error = 'NoEnds'
                # write into savefile
                vprint(v, 3, 'Writing classifier into {} for track {}'.format(
                    savefile, ind))
                trackio.write_object_to_hdf5(
                    classifier, h5save, 'cl_' + trkpath,
                    pyobj_to_h5=pyobj_to_h5)

                if progressflag:
                    pbar.update(n)
            if progressflag:
                pbar.finish()
        # f gets closed here
    except NotImplementedError:
        pass
Ejemplo n.º 9
0
def pyml_run_algs(loadfile, savefile, v):

    progressflag = False     # turn off for parallel

    pnlist = [
        'pix2_5noise0',
        'pix5noise0',
        'pix5noise15',
        'pix10_5noise0',
        'pix10_5noise15',
        'pix10_5noise20',
        'pix10_5noise50',
        'pix10_5noise100',
        'pix10_5noise200',
        'pix10_5noise500',
        'pix10_5noise1000',
        'pix10_5noise2000',
        'pix20noise0',
        'pix40noise0',
    ]
    alglist = {
        'python HT v1.52': ht,
        # 'python HT v1.52a': ht_a,
        # 1.52a causes RuntimeWarning and possibly breaks something more...
        'python HT v1.52b': ht_b,
        'python HT v1.52c': ht_c,
        'python HT v1.52d': ht_d,
    }
    tracklist = {}
    AR = {}
    for pnname in pnlist:
        tracklist[pnname] = []
        AR[pnname] = {}
        for algname in alglist.keys():
            AR[pnname][algname] = []

    vprint(v, 1, 'Starting {} at {} with {}% mem usage'.format(
        loadfile, time.ctime(), psutil.virtual_memory().percent))
    try:
        with h5py.File(loadfile, 'a', driver='core') as h5load:
            filename = h5load.filename
            n = 0
            if progressflag:
                pbar = progressbar.ProgressBar(
                    widgets=[progressbar.Percentage(), ' ',
                             progressbar.Bar(), ' ',
                             progressbar.ETA()], maxval=len(h5load))
                pbar.start()

            keylist = h5load.keys()
            keylist.sort()
            for ind in keylist:
                if int(ind) % 10 == 0:
                    vprint(v, 2, '    Running {} #{} at {}'.format(
                        loadfile, ind, time.ctime()))
                trk = h5load[ind]
                h5_to_pydict = {}
                pydict_to_pyobj = {}
                pyobj_to_h5 = {}

                n += 1
                if n > 50:
                    # pdb.set_trace()
                    # continue  # TODO temp!
                    pass

                for pnname in pnlist:
                    try:
                        pn = trk[pnname]
                    except KeyError:
                        vprint(v, 1,
                               '**Missing key {} in {}{}, skipping'.format(
                                   pnname, loadfile, trk.name))
                        continue
                    vprint(v, 3, 'Running {}{} at {}'.format(
                        loadfile, pn.name, time.ctime()))
                    # load track
                    try:
                        this_track = trackdata.Track.from_hdf5(
                            pn,
                            h5_to_pydict=h5_to_pydict,
                            pydict_to_pyobj=pydict_to_pyobj)
                    except trackio.InterfaceError:
                        vprint(v, 2, 'InterfaceError at {}{}'.format(
                            loadfile, pn.name))
                        continue
                    tracklist[pnname].append(this_track)
                    # each algorithm version
                    for algname, algfunc in alglist.items():
                        vprint(v, 4, 'Running alg {} at {}'.format(
                            algname, time.ctime()))
                        # check for result
                        if algname not in this_track.algorithms:
                            # run algorithm
                            try:
                                _, HTinfo = algfunc.reconstruct(this_track)
                            except algfunc.InfiniteLoop:
                                continue
                            except algfunc.NoEndsFound:
                                continue
                            # trim memory usage!
                            if hasattr(HTinfo, 'ridge'):
                                if HTinfo.ridge:
                                    for ridgept in HTinfo.ridge:
                                        ridgept.cuts = None
                                        ridgept.best_cut = None
                            # write into track object
                            try:
                                this_track.add_algorithm(
                                    algname,
                                    alpha_deg=HTinfo.alpha_deg,
                                    beta_deg=HTinfo.beta_deg,
                                    info=HTinfo)
                                # write into HDF5
                                trackio.write_object_to_hdf5(
                                    this_track.algorithms[algname],
                                    pn['algorithms'], algname,
                                    pyobj_to_h5=pyobj_to_h5)
                            except trackdata.InputError:
                                # already has this algorithm
                                pass

                if progressflag:
                    pbar.update(n)
            if progressflag:
                pbar.finish()
            # h5load gets closed
            vprint(
                v, 1.5,
                '\n  Finished loading {} at {} with {}% mem usage'.format(
                    loadfile, time.ctime(), psutil.virtual_memory().percent))
    except IOError:
        vprint(v, 1, 'IOError: Unable to open file (I think) for {}'.format(
            loadfile))
        return None

    # AlgorithmResults objects
    alglist2 = alglist.keys()  # + ['matlab HT v1.5']
    for pnname in pnlist:
        for algname in alglist2:
            this_AR = evaluation.AlgorithmResults.from_track_list(
                tracklist[pnname], alg_name=algname, filename=filename)
            AR[pnname][algname] = this_AR
    vprint(v, 2, '\n  Created AR objects for {} at {}'.format(
        loadfile, time.ctime()))

    # write to savefile
    try:
        with h5py.File(savefile, 'w', driver='core') as h5save:
            for pnname, AR_pn in AR.items():
                pngroup = h5save.create_group(pnname)
                for algname, this_AR in AR_pn.items():
                    trackio.write_object_to_hdf5(this_AR, pngroup, algname)
        vprint(v, 1.5,
               'Finished saving {} at {}'.format(savefile, time.ctime()))
    except IOError:
        vprint(v, 1, 'IOError: Unable to create file (I think) for {}'.format(
            savefile))

    return None
Ejemplo n.º 10
0
def pyml_run_algs(loadfile, savefile, v):

    progressflag = False  # turn off for parallel

    pnlist = [
        'pix2_5noise0',
        'pix5noise0',
        'pix5noise15',
        'pix10_5noise0',
        'pix10_5noise15',
        'pix10_5noise20',
        'pix10_5noise50',
        'pix10_5noise100',
        'pix10_5noise200',
        'pix10_5noise500',
        'pix10_5noise1000',
        'pix10_5noise2000',
        'pix20noise0',
        'pix40noise0',
    ]
    alglist = {
        'python HT v1.52': ht,
        # 'python HT v1.52a': ht_a,
        # 1.52a causes RuntimeWarning and possibly breaks something more...
        'python HT v1.52b': ht_b,
        'python HT v1.52c': ht_c,
        'python HT v1.52d': ht_d,
    }
    tracklist = {}
    AR = {}
    for pnname in pnlist:
        tracklist[pnname] = []
        AR[pnname] = {}
        for algname in alglist.keys():
            AR[pnname][algname] = []

    vprint(
        v, 1, 'Starting {} at {} with {}% mem usage'.format(
            loadfile, time.ctime(),
            psutil.virtual_memory().percent))
    try:
        with h5py.File(loadfile, 'a', driver='core') as h5load:
            filename = h5load.filename
            n = 0
            if progressflag:
                pbar = progressbar.ProgressBar(widgets=[
                    progressbar.Percentage(), ' ',
                    progressbar.Bar(), ' ',
                    progressbar.ETA()
                ],
                                               maxval=len(h5load))
                pbar.start()

            keylist = h5load.keys()
            keylist.sort()
            for ind in keylist:
                if int(ind) % 10 == 0:
                    vprint(
                        v, 2, '    Running {} #{} at {}'.format(
                            loadfile, ind, time.ctime()))
                trk = h5load[ind]
                h5_to_pydict = {}
                pydict_to_pyobj = {}
                pyobj_to_h5 = {}

                n += 1
                if n > 50:
                    # pdb.set_trace()
                    # continue  # TODO temp!
                    pass

                for pnname in pnlist:
                    try:
                        pn = trk[pnname]
                    except KeyError:
                        vprint(
                            v, 1, '**Missing key {} in {}{}, skipping'.format(
                                pnname, loadfile, trk.name))
                        continue
                    vprint(
                        v, 3,
                        'Running {}{} at {}'.format(loadfile, pn.name,
                                                    time.ctime()))
                    # load track
                    try:
                        this_track = trackdata.Track.from_hdf5(
                            pn,
                            h5_to_pydict=h5_to_pydict,
                            pydict_to_pyobj=pydict_to_pyobj)
                    except trackio.InterfaceError:
                        vprint(
                            v, 2,
                            'InterfaceError at {}{}'.format(loadfile, pn.name))
                        continue
                    tracklist[pnname].append(this_track)
                    # each algorithm version
                    for algname, algfunc in alglist.items():
                        vprint(
                            v, 4, 'Running alg {} at {}'.format(
                                algname, time.ctime()))
                        # check for result
                        if algname not in this_track.algorithms:
                            # run algorithm
                            try:
                                _, HTinfo = algfunc.reconstruct(this_track)
                            except algfunc.InfiniteLoop:
                                continue
                            except algfunc.NoEndsFound:
                                continue
                            # trim memory usage!
                            if hasattr(HTinfo, 'ridge'):
                                if HTinfo.ridge:
                                    for ridgept in HTinfo.ridge:
                                        ridgept.cuts = None
                                        ridgept.best_cut = None
                            # write into track object
                            try:
                                this_track.add_algorithm(
                                    algname,
                                    alpha_deg=HTinfo.alpha_deg,
                                    beta_deg=HTinfo.beta_deg,
                                    info=HTinfo)
                                # write into HDF5
                                trackio.write_object_to_hdf5(
                                    this_track.algorithms[algname],
                                    pn['algorithms'],
                                    algname,
                                    pyobj_to_h5=pyobj_to_h5)
                            except trackdata.InputError:
                                # already has this algorithm
                                pass

                if progressflag:
                    pbar.update(n)
            if progressflag:
                pbar.finish()
            # h5load gets closed
            vprint(
                v, 1.5,
                '\n  Finished loading {} at {} with {}% mem usage'.format(
                    loadfile, time.ctime(),
                    psutil.virtual_memory().percent))
    except IOError:
        vprint(
            v, 1,
            'IOError: Unable to open file (I think) for {}'.format(loadfile))
        return None

    # AlgorithmResults objects
    alglist2 = alglist.keys()  # + ['matlab HT v1.5']
    for pnname in pnlist:
        for algname in alglist2:
            this_AR = evaluation.AlgorithmResults.from_track_list(
                tracklist[pnname], alg_name=algname, filename=filename)
            AR[pnname][algname] = this_AR
    vprint(
        v, 2,
        '\n  Created AR objects for {} at {}'.format(loadfile, time.ctime()))

    # write to savefile
    try:
        with h5py.File(savefile, 'w', driver='core') as h5save:
            for pnname, AR_pn in AR.items():
                pngroup = h5save.create_group(pnname)
                for algname, this_AR in AR_pn.items():
                    trackio.write_object_to_hdf5(this_AR, pngroup, algname)
        vprint(v, 1.5,
               'Finished saving {} at {}'.format(savefile, time.ctime()))
    except IOError:
        vprint(
            v, 1,
            'IOError: Unable to create file (I think) for {}'.format(savefile))

    return None
Ejemplo n.º 11
0
def get_results(loadfile, savefile, v):
    """
    Get all the important numbers for each event.

    (basic info:)
        Etot
        Edep
        Etrack
        alpha_true
        beta_true
        errorcode
        file
        ind
    (ridge following result:)
        alpha_rf
    (moments result:)
        alpha_m
    (general rejection parameters:)
        min_end_energy
        max_end_energy
        n_ends
    (moments rejection parameters:)
        phi
        edge_pixels
        edge_segments
    (Monte Carlo rejection parameters:)
        overlap flag
        wrong_end flag
        early_scatter flag
        total_scatter_angle
    """

    progressflag = not file_vars()[0]

    pn = 'pix10_5noise15'
    HTname = 'python HT v1.52'
    MTname = 'moments v1.0'

    vprint(v, 1, 'Starting {} at {} with {}% mem usage'.format(
        loadfile, time.ctime(), psutil.virtual_memory().percent))

    datalen = 1000
    # basic info
    energy_tot_kev = np.nan * np.ones(datalen)
    energy_dep_kev = np.nan * np.ones(datalen)
    energy_track_kev = np.nan * np.ones(datalen)
    alpha_true_deg = np.nan * np.ones(datalen)
    beta_true_deg = np.nan * np.ones(datalen)
    trk_errorcode = np.nan * np.ones(datalen)
    cl_errorcode = np.nan * np.ones(datalen)
    mom_errorcode = np.nan * np.ones(datalen)
    filename = ['' for _ in xrange(datalen)]
    fileind = np.nan * np.ones(datalen)
    # ridge following
    alpha_ridge_deg = np.nan * np.ones(datalen)
    # moments
    alpha_moments_deg = np.nan * np.ones(datalen)
    # rejection
    min_end_energy_kev = np.nan * np.ones(datalen)
    max_end_energy_kev = np.nan * np.ones(datalen)
    n_ends = np.nan * np.ones(datalen)
    phi_deg = np.nan * np.ones(datalen)
    edge_pixels = np.nan * np.ones(datalen)
    edge_segments = np.nan * np.ones(datalen)
    # monte carlo
    overlap_flag = np.nan * np.ones(datalen)
    wrong_end_flag = np.nan * np.ones(datalen)
    early_scatter_flag = np.nan * np.ones(datalen)
    total_scatter_angle_deg = np.nan * np.ones(datalen)

    try:
        with h5py.File(loadfile, 'r', driver='core') as f, h5py.File(
                savefile, 'a', driver='core') as h5save:
            if progressflag:
                pbar = progressbar.ProgressBar(
                    widgets=[progressbar.Percentage(), ' ',
                             progressbar.Bar(), ' ',
                             progressbar.ETA()], maxval=datalen)
                pbar.start()

            for ind in xrange(datalen):
                indstr = '{:05d}'.format(ind)
                trkstr = indstr
                clstr = 'cl_' + indstr
                momstr = 'mom_' + indstr

                this_trk_errorcode = 0
                this_cl_errorcode = 0
                this_mom_errorcode = 0

                filename[ind] = os.path.split(loadfile)[-1]
                fileind[ind] = ind

                # track and algorithm alpha's
                try:
                    this_trk = Track.from_hdf5(f[trkstr][pn])
                except KeyError:
                    # could be an MException from MATLAB
                    this_trk_errorcode = 10
                    trk_errorcode[ind] = this_trk_errorcode
                    continue
                except trackio.InterfaceError:
                    read_errorcode = f[trkstr][pn].attrs['errorcode']
                    if read_errorcode > 0:
                        # multiplicity event
                        this_trk_errorcode = read_errorcode
                        trk_errorcode[ind] = this_trk_errorcode
                        continue
                    else:
                        raise
                energy_tot_kev[ind] = this_trk.g4track.energy_tot_kev
                energy_dep_kev[ind] = this_trk.g4track.energy_dep_kev
                energy_track_kev[ind] = this_trk.energy_kev
                alpha_true_deg[ind] = this_trk.g4track.alpha_deg
                beta_true_deg[ind] = this_trk.g4track.beta_deg
                alpha_ridge_deg[ind] = this_trk[HTname].alpha_deg
                alpha_moments_deg[ind] = this_trk[MTname].alpha_deg

                # classifier
                this_cl = cl.Classifier.from_hdf5(f[clstr][pn])
                min_end_energy_kev[ind] = this_cl.min_end_energy
                max_end_energy_kev[ind] = this_cl.max_end_energy
                n_ends[ind] = this_cl.n_ends
                overlap_flag[ind] = this_cl.overlap
                wrong_end_flag[ind] = this_cl.wrong_end
                if this_cl.error == 'TrackTooShortError':
                    this_cl_errorcode = 8
                else:
                    early_scatter_flag[ind] = this_cl.early_scatter
                    total_scatter_angle_deg[ind] = (
                        this_cl.total_scatter_angle / np.pi * 180)

                # moments info
                try:
                    this_mom = tm.MomentsReconstruction.from_hdf5(
                        f[momstr][pn])
                except trackio.InterfaceError:
                    if f[momstr][pn].attrs['errorcode'] == 4:
                        # no ends found
                        this_mom_errorcode = 4
                    else:
                        raise
                else:
                    if this_mom.error == 'CheckSegmentBoxError':
                        this_mom_errorcode = 9
                    elif this_mom.error == 'what the heck happened?':
                        this_mom.errorcode = 11
                    elif this_mom.error == 'Rotation angle conditions not met':
                        this_mom.errorcode = 12
                    else:
                        phi_deg[ind] = this_mom.phi / np.pi * 180
                        edge_pixels[ind] = this_mom.edge_pixel_count
                        edge_segments[ind] = this_mom.edge_pixel_segments

                trk_errorcode[ind] = this_trk_errorcode
                cl_errorcode[ind] = this_cl_errorcode
                mom_errorcode[ind] = this_mom_errorcode

                if progressflag:
                    pbar.update(ind)

            # save to file
            # basic info
            h5save.create_dataset(
                'energy_tot_kev', shape=(datalen,), data=energy_tot_kev)
            h5save.create_dataset(
                'energy_dep_kev', shape=(datalen,), data=energy_dep_kev)
            h5save.create_dataset(
                'energy_track_kev', shape=(datalen,), data=energy_track_kev)
            h5save.create_dataset(
                'alpha_true_deg', shape=(datalen,), data=alpha_true_deg)
            h5save.create_dataset(
                'beta_true_deg', shape=(datalen,), data=beta_true_deg)
            h5save.create_dataset(
                'trk_errorcode', shape=(datalen,), data=trk_errorcode,
                dtype='f2')
            h5save.create_dataset(
                'cl_errorcode', shape=(datalen,), data=cl_errorcode,
                dtype='f2')
            h5save.create_dataset(
                'mom_errorcode', shape=(datalen,), data=mom_errorcode,
                dtype='f2')
            h5save.create_dataset(
                'filename', shape=(datalen,), data=filename,
                dtype=h5py.special_dtype(vlen=str))
            h5save.create_dataset(
                'fileind', shape=(datalen,), data=fileind,
                dtype='f4')
            # ridge following
            h5save.create_dataset(
                'alpha_ridge_deg', shape=(datalen,), data=alpha_ridge_deg)
            # moments
            h5save.create_dataset(
                'alpha_moments_deg', shape=(datalen,), data=alpha_moments_deg)
            # rejection parameters
            h5save.create_dataset(
                'min_end_energy_kev', shape=(datalen,),
                data=min_end_energy_kev)
            h5save.create_dataset(
                'max_end_energy_kev', shape=(datalen,),
                data=max_end_energy_kev)
            h5save.create_dataset(
                'n_ends', shape=(datalen,), data=n_ends,
                dtype='f4')
            h5save.create_dataset(
                'phi_deg', shape=(datalen,), data=phi_deg)
            h5save.create_dataset(
                'edge_pixels', shape=(datalen,), data=edge_pixels,
                dtype='f4')
            h5save.create_dataset(
                'edge_segments', shape=(datalen,), data=edge_segments,
                dtype='f2')
            # Monte Carlo
            h5save.create_dataset(
                'overlap_flag', shape=(datalen,), data=overlap_flag,
                dtype='f2')
            h5save.create_dataset(
                'wrong_end_flag', shape=(datalen,), data=wrong_end_flag,
                dtype='f2')
            h5save.create_dataset(
                'early_scatter_flag', shape=(datalen,),
                data=early_scatter_flag, dtype='f2')
            h5save.create_dataset(
                'total_scatter_angle_deg', shape=(datalen,),
                data=total_scatter_angle_deg)

            if progressflag:
                pbar.finish()

    except NotImplementedError:
        pass