Ejemplo n.º 1
0
def read_data(data_files):
    x_files, y_files = data_files
    XX, Y = None, None
    for x_file in x_files:
        print 'Reading..', x_file
        sys.stdout.flush()
        X_subject = np.recfromtxt(x_file, delimiter=',')
        if XX is None:
            XX = X_subject
        else:
            XX = np.concatenate((XX, X_subject))

    XX = XX.reshape((XX.shape[0], 3, config.width, config.height, config.time_slice))
    #print XX.shape

    for y_file in y_files:
        print 'Reading..', y_file
        sys.stdout.flush()
        Y_subject = np.recfromtxt(y_file)
        if Y is None:
            Y = Y_subject
        else:
            Y = np.concatenate((Y, Y_subject))

    return XX, Y
Ejemplo n.º 2
0
def check_n_in_aper(radius_factor=1, k=100):

	for catfile in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):

		print
		print catfile
		names = open(catfile).readline().split()[1:]
		cat = np.recfromtxt(catfile, names=names)

		xscfile = catfile.replace('combined_hdr_catalog.txt','2mass_xsc.tbl')
		print xscfile
		names = open(xscfile).read().split('\n')[76].split('|')[1:-1]
		xsc = np.recfromtxt(xscfile, skip_header=80, names=names)

		n_in_aper = []
		coords = radec_to_coords(cat.ra, cat.dec)
		kdt = KDT(coords)
		for i in range(xsc.size):
			r_deg = xsc.r_ext[i]/3600.

			idx, ds = spherematch2(xsc.ra[i], xsc.dec[i], cat.ra, cat.dec,
				kdt, tolerance=radius_factor*r_deg, k=k)
			n_in_aper.append(ds.size)
		for i in [(i,n_in_aper.count(i)) for i in set(n_in_aper)]:
			print i
Ejemplo n.º 3
0
    def initialize_database(self, configuration_dir=None):
        """Read in GMOS filter/grating information, for matching to headers."""
        if configuration_dir is None:
            configuration_dir = default_configuration_dir

        logger.info('Reading Filter information')

        gmos_filters = np.recfromtxt(
            os.path.join(configuration_dir, 'GMOSfilters.dat'),
            names=['name', 'wave_start', 'wave_end', 'fname'])
        for line in gmos_filters:
            new_filter = GMOSFilter(name=line['name'], 
                                    wavelength_start_value=line['wave_start'],
                                    wavelength_start_unit='nm', 
                                    wavelength_end_value=line['wave_end'],
                                    wavelength_end_unit='nm', 
                                    fname=line['fname'],
                                    path=os.path.join(configuration_dir, 
                                                      'filter_data'))
            self.session.add(new_filter)

        open_filter = GMOSFilter(name='open', wavelength_start_value=0,
                                 wavelength_start_unit='nm', 
                                 wavelength_end_value=np.inf,
                                 wavelength_end_unit='nm', 
                                 fname=None, path=None)
        self.session.add(open_filter)

        gmos_gratings = np.recfromtxt(
            os.path.join(configuration_dir, 'GMOSgratings.dat'),
            names = ['name', 'ruling_density', 'blaze_wave', 'R', 'coverage',
                     'wave_start', 'wave_end', 'wave_offset', 'y_offset'])
        logger.info('Reading grating information')
        for line in gmos_gratings:
            new_grating = GMOSGrating(
                name=line['name'], 
                ruling_density_value=line['ruling_density'],
                blaze_wavelength_value=line['blaze_wave'], 
                R=line['R'],
                coverage_value=line['coverage'], 
                wavelength_start_value=line['wave_start'],
                wavelength_end_value=line['wave_end'],
                wavelength_offset_value=line['wave_offset'], 
                y_offset_value=line['y_offset'])
            self.session.add(new_grating)

        mirror = GMOSGrating(name='mirror', 
                             ruling_density_value=0.0,
                             blaze_wavelength_value=0.0, R=0.0,
                             coverage_value=np.inf, 
                             wavelength_start_value=0.0,
                             wavelength_end_value=np.inf,
                             wavelength_offset_value=0.0, 
                             y_offset_value=0.0)
        self.session.add(mirror)
        self.session.commit()
Ejemplo n.º 4
0
def _get_data():
    filepath = dirname(abspath(__file__))
    ##### EDIT THE FOLLOWING TO POINT TO DatasetName.csv #####
    with open(filepath + '/statecrime.csv', 'rb') as f:
        try:
            data = np.recfromtxt(f, delimiter=",", names=True,
                                 dtype=None, encoding='utf-8')
        except TypeError:
            data = np.recfromtxt(f, delimiter=",", names=True, dtype=None)
    return data
Ejemplo n.º 5
0
	def handle_special(self, q, p):
		if q[0] == '\\':
			NULL = p.stdout.readline()
			if len(q.split()) > 1:
				names = p.stdout.readline().split(',')
				r = np.recfromtxt(p.stdout, skip_footer=2, delimiter=',', names=names)
			else:
				names = p.stdout.readline().split(',')
				r = np.recfromtxt(p.stdout, skip_footer=1, delimiter=',', names=names)

		else:
			names = p.stdout.readline().split(',')
			r = np.recfromtxt(p.stdout, skip_footer=1, delimiter=',', names=names)
		return r
Ejemplo n.º 6
0
    def layoutFromTxt(filename):
        """Read plate layout from text file and return a structured array."""

        if not os.path.isfile(filename):
            msg = "No Plate Layout provided. File not found {}".format(filename)
            raise IOError(msg)

        try:
            rec = np.recfromtxt(filename, dtype=LayoutDtype, skip_header=True)
        except ValueError:
            rec = np.recfromtxt(filename, dtype=LayoutDtype, delimiter="\t",
                                skip_header=True)

        return rec
Ejemplo n.º 7
0
def load():
    """
    Loads the Grunfeld data and returns a Dataset class.

    Returns
    -------
    Dataset instance:
        See DATASET_PROPOSAL.txt for more information.

    Notes
    -----
    raw_data has the firm variable expanded to dummy variables for each
    firm (ie., there is no reference dummy)
    """
    filepath = dirname(abspath(__file__))
    data = recfromtxt(open(filepath + '/grunfeld.csv','rb'), delimiter=",",
            names=True, dtype="f8,f8,f8,a17,f8")
    names = list(data.dtype.names)
    endog = array(data[names[0]], dtype=float)
    endog_name = names[0]
    exog = data[list(names[1:])]
    exog_name = list(names[1:])
    dataset = Dataset(data=data, names=names, endog=endog, exog=exog,
            endog_name=endog_name, exog_name=exog_name)
    raw_data = categorical(data, col='firm', drop=True)
    dataset.raw_data = raw_data
    return dataset
Ejemplo n.º 8
0
    def get_model_conditions(self, model):
        """Returns a description of all conditions for a given model

        Parameters
        ----------
        model : int
          Model identifier.

        Returns
        -------
        list(dict)
          A list of a model conditions is returned, where each item is a
          dictionary with keys ``id`` (numerical condition ID), ``task``
          (numerical task ID for the task containing this condition), and
          ``name`` (the literal condition name). This information is
          returned in a list (instead of a dictionary), because the openfmri
          specification of model conditions contains no unique condition
          identifier. Conditions are only uniquely described by the combination
          of task and condition ID.
        """
        def_fname = _opj(self._basedir, 'models', _model2id(model),
                         'condition_key.txt')
        def_data = np.recfromtxt(def_fname)
        conds = []
        # load model meta data
        for dd in def_data:
            cond = {}
            cond['task'] = _id2int(dd[0])
            cond['id'] = _id2int(dd[1])
            cond['name'] = dd[2]
            conds.append(cond)
        return conds
Ejemplo n.º 9
0
def _get_data():
    filepath = dirname(abspath(__file__))
    with open(filepath + "/anes96.csv", "rb") as f:
        data = recfromtxt(f, delimiter="\t", names=True, dtype=float)
        logpopul = log(data["popul"] + 0.1)
        data = nprf.append_fields(data, "logpopul", logpopul, usemask=False, asrecarray=True)
    return data
Ejemplo n.º 10
0
    def __init__(self, fnames=None, params=None):
        '''params: a list of numpy arrays'''
        super(MovementParamPlot, self).__init__()

        if fnames is None:
            # XXX Hardcode hack to get envisage working
            fnames = ['data/movement_params.txt']
            params = np.recfromtxt(fnames[0], dtype=PARAM_DTYPE)

        self.file_list = fnames
        self.file_name = fnames[0]
        self.params_num = 0
        self.max_params = len(params) - 1

        self.params = params

        # XXX - the following info should go (soon) into 
        # nipype.interfaces.fsl.McFLIRT
        self.trans_plot = self.create_line_plot('trans', 'mm')
        self.rot_plot = self.create_line_plot('rot', 'radians (clockwise)',
                                              self.trans_plot.index_range)

        # Note - it doesn't matter which plot you use to init the tools
        self.zoom = tools.ZoomTool(self.trans_plot, tool_mode='range', 
                                   axis='index')
        self.pan = tools.PanTool(self.rot_plot, constrain=True,
                                 constrain_direction='x')
        self.trans_plot.tools.extend((self.zoom, self.pan))
        self.rot_plot.tools.extend((self.pan, self.zoom))
Ejemplo n.º 11
0
def read(filename):
    """
    Read a table (.tbl) file.

    Parameters:

    * filename  Name of table file to read 

    Returns: (comments,rec)

    * comments  List of comments (strings terminated with newline)
    * rec       Records array with named fields.

    """
    # pull out the comment lines from the file (start with #)
    f = open(filename,'r')
    comments = [l for l in f if l[0]=='#']
    f.close()
    
    # find the line beginning with # NAMES and parse out the column names
    nl = [i for i,l in enumerate(comments) if l[:7]=="# NAMES"]
    if len(nl)!=1:
        raise IOError("%s does not have a # NAMES line"%(filename))
    dtd = {'names':comments.pop(nl[0])[7:].split()}
    
    # find the line beginning with # DTYPE and parse out the column names
    dl = [i for i,l in enumerate(comments) if l[:9]=="# FORMATS"]
    if len(dl)!=1:
        raise IOError("%s does not have a # FORMATS line"%(filename))
    dtd['formats'] = comments.pop(dl[0])[9:].split()
    
    # return the data as a records array
    return comments,np.atleast_1d(np.recfromtxt(filename,dtype=dtd))
Ejemplo n.º 12
0
 def __init__(self, pathlike):
     if isinstance(pathlike, str):
         paths = glob(pathlike)
     else:
         paths = pathlike
     paths.sort()
     pfile = self
     pfile._vars = dict()
 
     files = [open(path) for path in paths]
     datas = [np.recfromtxt(f, names = True, case_sensitive = True) for f in files]
     data = np.ma.concatenate(datas)
     desired_unit = dict(O3 = 'ppb', GMAO_TEMP = 'K', PRESS = 'hPa', TEMP = 'K')
     unit_factor = {'ppt': 1e12, 'ppb': 1e9}
     pfile.createDimension('time', data.shape[0])
     for ki, key in enumerate(data.dtype.names):
         typecode = data[key].dtype.char
         if typecode not in ('c', 'S'):
             unit = desired_unit.get(key, 'ppt')
             factor = unit_factor.get(unit, 1)
             values = np.ma.masked_values(data[key], -1000) * factor
         else:
             unit = 'unknown'
             values = data[key]
         pfile.createVariable(key, typecode, dimensions = ('time',), units = unit, values = values)
Ejemplo n.º 13
0
def SNrest():
    path = "../data/restframe/"
    objnames, band, mjd, mag, magerr, stype = [],[],[],[],[], []
    formatcode = ('|S16,'.rstrip('#') +'f8,'*6 + '|S16,' + 4 * 'f8,' + '|S16,' * 3 + 'f8,' * 2 + '|S16,' + 'f8,' * 2)
    filenames = os.listdir(path)
    for filename in filenames:
        data = np.recfromtxt(os.path.join(path, filename),usecols = (0,1,2,3,4), dtype = formatcode, names = True, skip_header = 13, case_sensitive = 'lower', invalid_raise = False)
        name = np.empty(len(data.band), dtype = 'S20')
        name.fill(filename)
        objnames.append(name)
        data.band = [x.lower() for x in data.band]
        band.append(data.band)
        mjd.append(data.phase)
        mag.append(data.mag)
        magerr.append(data.err)
        
        
    objnames = np.fromiter(itertools.chain.from_iterable(objnames), dtype = 'S20')
    band = np.fromiter(itertools.chain.from_iterable(band), dtype = 'S16')
    mjd = np.fromiter(itertools.chain.from_iterable(mjd), dtype = 'float')
    mag = np.fromiter(itertools.chain.from_iterable(mag), dtype = 'float')
    magerr = np.fromiter(itertools.chain.from_iterable(magerr), dtype = 'float')
    stype = np.full(len(objnames), 1)
    LC = Lightcurve(objnames, band, mjd, mag, magerr, stype)
    return LC
Ejemplo n.º 14
0
def readin_aavso(filename):
    formatcode = ('f8,'*4 + '|S16,'*20).rstrip(',')
    data = np.recfromtxt(filename, delimiter='\t', names=True, dtype=formatcode,autostrip=True,case_sensitive='lower', invalid_raise=False)

    ind = np.where((data.band == 'V') & (data.uncertainty > 0) & (np.isnan(data.uncertainty) == 0) & (data.uncertainty < 0.02))
    banddata = data[ind]
    return banddata
Ejemplo n.º 15
0
def get_region_corners(catalog):
	cat = np.recfromtxt(catalog, names=open(catalog).readline().split()[1:])
	c1 = (cat.ra.min(), cat.dec[cat.ra==cat.ra.min()][0])
	c2 = (cat.ra[cat.dec==cat.dec.max()][0], cat.dec.max())
	c3 = (cat.ra.max(), cat.dec[cat.ra==cat.ra.max()][0])
	c4 = (cat.ra[cat.dec==cat.dec.min()][0], cat.dec.min())
	return reduce(lambda x,y: x+y, [c1, c2, c3, c4])
Ejemplo n.º 16
0
def query_region_db(ra_center, dec_center, radius, region_dir):
    """
    Returns a list of files relative to regions which are within the provided cone

    :param ra_center: R.A. of the center of the cone
    :param dec_center: Dec. of the center of the cone
    :param radius: Radius of the cone (in arcmin)
    :param region_dir: Path of the directory containing the database file and the region files
    :return: list of region files
    """

    database_file = os.path.join(region_dir, 'region_database.txt')

    data = np.recfromtxt(database_file, names=True)

    # Compute the angular distance between all regions and the center of the cone

    distances = angular_distance(float(ra_center), float(dec_center), data["RA"], data["DEC"], unit='arcmin')

    # Select all regions within the cone

    idx = (distances <= float(radius))

    # Return the corresponding region files

    data_list = []

    for i in data['REGION_FILE'][idx]:
        data_list.append(os.path.join(region_dir, i))

    return data_list
Ejemplo n.º 17
0
    def _initialize_gmos_filters(self, configuration_dir):
        """
        Read in GMOS filter/grating information, for matching to headers.
        """
        logger.info('Reading Filter information')

        gmos_filters = np.recfromtxt(
            os.path.join(configuration_dir, 'GMOSfilters.dat'),
            names=['name', 'wave_start', 'wave_end', 'fname'])
        for line in gmos_filters:
            new_filter = GMOSFilter(name=line['name'],
                                    wavelength_start_value=line['wave_start'],
                                    wavelength_start_unit='nm',
                                    wavelength_end_value=line['wave_end'],
                                    wavelength_end_unit='nm',
                                    fname=line['fname'],
                                    path=os.path.join('gmos', 'filter_data'))
            self.session.add(new_filter)

        open_filter = GMOSFilter(name='open', wavelength_start_value=0,
                                 wavelength_start_unit='nm',
                                 wavelength_end_value=np.inf,
                                 wavelength_end_unit='nm',
                                 fname=None, path=None)
        self.session.add(open_filter)
Ejemplo n.º 18
0
 def __init__(self, path, coordkeys = "time time_bounds TFLAG ETFLAG latitude latitude_bounds longitude longitude_bounds lat lat_bnds lon lon_bnds etam_pressure etai_pressure layer_bounds layer47 layer".split(), delimiter = ',', names = True, **kwds):
     """
     path - place to find csv file
     coordkeys - use these keys as dimensions and coordinate variables
     delimiter - use this as delimiter (default = ',')
     names - see help in recfromtxt (Default = True)
     kwds - np.recfromtxt keywords
     
     * Note: currently only works when all coordinate variables are 1-d
     """
     kwds['names'] = names
     kwds['delimiter'] =  delimiter
     data = np.recfromtxt(path, **kwds)
     dimkeys = [dk for dk in data.dtype.names if dk in coordkeys]
     varkeys = [vk for vk in data.dtype.names if not vk in coordkeys]
     for dk in dimkeys:
         dv = np.unique(data[dk])
         dv.sort()
         self.createDimension(dk, len(dv))
         dvar = self.createVariable(dk, dv.dtype.char, (dk,))
         dvar[:] = dv
     
     for vk in varkeys:
         vv = data[vk]
         var = self.createVariable(vk, vv.dtype.char, tuple(dimkeys))
         for idx in np.ndindex(var.shape):
             thisidx = np.sum([data[dk] == self.variables[dk][di] for di, dk in zip(idx, dimkeys)], axis = 0) == len(dimkeys)
             if thisidx.any():
                 var[idx] = vv[thisidx]
Ejemplo n.º 19
0
def getList(file):
    posList= np.recfromtxt(file)
    l = [posList['f0'],posList['f2'],posList['f3']]
    l = np.array(l)
    l = l.T
    names = posList['f4']
    return l,names
Ejemplo n.º 20
0
def sigma_clip_non_hdr(filepath):

    """
	Eliminates sources with SNR less than the 'sigma_clip' parameter from setup
	file. Also checks for negative flux sources which may remain after this
	step (can happen when uncertainty values are also negative).
	"""

    work_dir = "/".join(filepath.split("/")[:-1])
    meta = json.load(open(work_dir + "/metadata.json"))

    names = open(filepath).readline().split()[1:]
    data = np.recfromtxt(filepath, names=names)

    # get rid of low SNR sources
    snr = data["flux"] / data["unc"]
    good = snr >= meta["sigma_clip"]
    data = data[good]

    # get rid of any remaining negative flux sources
    good = data["flux"] > 0
    data = data[good]

    # get rid of id column
    data = data[["ra", "dec", "flux", "unc", "n_obs"]]

    # write to disk
    fmt = ["%0.8f"] * 2 + ["%.4e"] * 2 + ["%i"]
    out_path = filepath.replace(".txt", "_sigclip.txt")
    header = " ".join(names[1:])
    np.savetxt(out_path, data, fmt=fmt, header=header, comments="")
    print("created file: " + out_path)
Ejemplo n.º 21
0
def load():
    """
    Load the star98 data and returns a Dataset class instance.

    Returns
    -------
    Load instance:
        a class of the data with array attrbutes 'endog' and 'exog'
    """
    filepath = dirname(abspath(__file__))
##### EDIT THE FOLLOWING TO POINT TO DatasetName.csv #####
    names = ["NABOVE","NBELOW","LOWINC","PERASIAN","PERBLACK","PERHISP",
            "PERMINTE","AVYRSEXP","AVSALK","PERSPENK","PTRATIO","PCTAF",
            "PCTCHRT","PCTYRRND","PERMINTE_AVYRSEXP","PERMINTE_AVSAL",
            "AVYRSEXP_AVSAL","PERSPEN_PTRATIO","PERSPEN_PCTAF","PTRATIO_PCTAF",
            "PERMINTE_AVYRSEXP_AVSAL","PERSPEN_PTRATIO_PCTAF"]
    data = recfromtxt(open(filepath + '/star98.csv',"rb"), delimiter=",",
            names=names, skip_header=1, dtype=float)
    names = list(data.dtype.names)
    # endog = (successes, failures)
    NABOVE = array(data[names[1]]).astype(float) # successes
    NBELOW = array(data[names[0]]).astype(float) \
                - array(data[names[1]]).astype(float) # now its failures
    endog = column_stack((NABOVE,NBELOW))
    endog_name = names[:2]
    exog = column_stack(data[i] for i in names[2:]).astype(float)
    exog_name = names[2:]
    dataset = Dataset(data=data, names=names, endog=endog, exog=exog,
            endog_name = endog_name, exog_name=exog_name)
    return dataset
def movie_dataset(subj, task, label, **kwargs):
    # ds = movie_dataset(
    #       2,
    #       'avmovie',
    #       'bold3Tp2',
    #       mask='src/tnt/sub-02/bold3Tp2/brain_mask.nii.gz')
    cur_max_time = 0
    segments = []
    if not 'add_sa' in kwargs:
        add_sa = {}
    for seg in range(1, 9):
        print 'Seg', seg
        mc = np.recfromtxt(
            'sub-%.2i/in_%s/sub-%.2i_task-%s_run-%i_bold_mcparams.txt'
            % (subj, label, subj, task, seg),
            names=('mc_xtrans', 'mc_ytrans', 'mc_ztrans', 'mc_xrot',
                   'mc_yrot', 'mc_zrot'))
        for i in mc.dtype.fields:
            add_sa[i] = mc[i]
        ds = preprocessed_fmri_dataset(
            'sub-%.2i/in_%s/sub-%.2i_task-%s_run-%i_bold.nii.gz'
            % (subj, label, subj, task, seg),
            add_sa=add_sa,
            **kwargs)
        ds.sa['movie_segment'] = [seg] * len(ds)
        TR = np.diff(ds.sa.time_coords).mean()
        ## truncate segment time series to remove overlap
        if seg > 1:
            ds = ds[4:]
        if seg < 8:
            ds = ds[:-4]
        ds.sa['movie_time'] = np.arange(len(ds)) * TR + cur_max_time
        cur_max_time = ds.sa.movie_time[-1] + TR
        segments.append(ds)
    return vstack(segments)
Ejemplo n.º 23
0
    def _initialize_gmos_gratings(self, configuration_dir):
        gmos_gratings = np.recfromtxt(
            os.path.join(configuration_dir, 'GMOSgratings.dat'),
            names = ['name', 'ruling_density', 'blaze_wave', 'R', 'coverage',
                     'wave_start', 'wave_end', 'wave_offset', 'y_offset'])
        logger.info('Reading grating information')
        for line in gmos_gratings:
            new_grating = GMOSGrating(
                name=line['name'],
                ruling_density_value=line['ruling_density'],
                blaze_wavelength_value=line['blaze_wave'],
                R=line['R'],
                coverage_value=line['coverage'],
                wavelength_start_value=line['wave_start'],
                wavelength_end_value=line['wave_end'],
                wavelength_offset_value=line['wave_offset'],
                y_offset_value=line['y_offset'])
            self.session.add(new_grating)

        mirror = GMOSGrating(name='mirror',
                             ruling_density_value=0.0,
                             blaze_wavelength_value=0.0, R=0.0,
                             coverage_value=np.inf,
                             wavelength_start_value=0.0,
                             wavelength_end_value=np.inf,
                             wavelength_offset_value=0.0,
                             y_offset_value=0.0)
        self.session.add(mirror)
        self.session.commit()
Ejemplo n.º 24
0
def _get_data():
    filepath = dirname(abspath(__file__))
    ##### EDIT THE FOLLOWING TO POINT TO DatasetName.csv #####
    with open(filepath + '/spector.csv',"rb") as f:
        data = np.recfromtxt(f, delimiter=" ",
                             names=True, dtype=float, usecols=(1,2,3,4))
        return data
Ejemplo n.º 25
0
def _get_data():
    filepath = dirname(abspath(__file__))
    ##### EDIT THE FOLLOWING TO POINT TO DatasetName.csv #####
    with open(filepath + '/engel.csv', 'rb') as f:
        data = np.recfromtxt(f,
                             delimiter=",", names = True, dtype=float)
    return data
Ejemplo n.º 26
0
 def readFrameList(self, parFidx, parWDir):
     self.pos=0
     self.fcsvIdx=None
     self.wdir=None
     self.listFImg=[]
     if os.path.isfile(parFidx):
         tmpList=np.recfromtxt(parFidx)
         isAllOk=True
         if len(tmpList)>0:
             for ii in tmpList:
                 tmpFimg=ii
                 if parWDir!=None:
                     tmpFimg='%s/%s' % (parWDir, ii)
                 if os.path.isfile(tmpFimg):
                     self.listFImg.append(tmpFimg)
                 else:
                     isAllOk=False
                     if self.isDebug:
                         print "Can't find file [%s]" % tmpFimg
                     break
         else:
             isAllOk=False
         if isAllOk:
             self.fcsvIdx=parFidx
             self.wdir=parWDir
         else:
             self.listFImg=[]
Ejemplo n.º 27
0
def create_histogram(parameter_name, nbins=100, writeFile=True, skipfirst=0, truncate=False, smooth=False):
	"""
	Returns a histogram and some statistics about this parameter.
		
	@param writeFile: if true, write the histogram to paramname.histogram
	"""
	f = "%s-chain-0.prob.dump" % parameter_name
	values = numpy.recfromtxt(f)[skipfirst::nevery]

	statistics = {
		'min':   float(values.min()),
		'max':   float(values.max()),
		'stdev': float(values.std()),
		'mean':  float(values.mean()),
		'median':float(numpy.median(values)),
		'q1':    float(scipy.stats.scoreatpercentile(values, 25)),
		'q3':    float(scipy.stats.scoreatpercentile(values, 75)),
		'p5':    float(scipy.stats.scoreatpercentile(values, 5)),
		'p95':    float(scipy.stats.scoreatpercentile(values, 95)),
	}
	
	hist = scipy.histogram(values, bins = nbins if not smooth else nbins*10, normed=True)
	histwithborders = numpy.dstack([hist[1][0:nbins], hist[1][1:nbins+1], hist[0]])
	if writeFile:
		scipy.savetxt('%s.histogram' % parameter_name, histwithborders[0], delimiter="\t")
	return histwithborders[0], statistics
Ejemplo n.º 28
0
def read_csv(filename, column_headers, column_types):
    # use csv module to parse csv (understands quotes etc...) and use \t as
    # seperator to help numpy parsing the csv
    reader = csv.reader(open(filename))
    if column_headers:
        names = reader.next()
    return np.recfromtxt(("\t".join(row) for row in reader),
                         delimiter="\t", names=names, dtype=column_types)
Ejemplo n.º 29
0
 def test_recfromtxt(self):
     #
     data = StringIO("A,B\n0,1\n2,3")
     kwargs = dict(delimiter=",", missing_values="N/A", names=True)
     test = np.recfromtxt(data, **kwargs)
     control = np.array([(0, 1), (2, 3)], dtype=[("A", np.int), ("B", np.int)])
     self.assertTrue(isinstance(test, np.recarray))
     assert_equal(test, control)
     #
     data = StringIO("A,B\n0,1\n2,N/A")
     test = np.recfromtxt(data, dtype=None, usemask=True, **kwargs)
     control = ma.array(
         [(0, 1), (2, -1)], mask=[(False, False), (False, True)], dtype=[("A", np.int), ("B", np.int)]
     )
     assert_equal(test, control)
     assert_equal(test.mask, control.mask)
     assert_equal(test.A, [0, 2])
Ejemplo n.º 30
0
def _get_data():
    filepath = dirname(abspath(__file__))
    data = recfromtxt(open(filepath + '/anes96.csv',"rb"), delimiter="\t",
            names = True, dtype=float)
    logpopul = log(data['popul'] + .1)
    data = nprf.append_fields(data, 'logpopul', logpopul, usemask=False,
                                                          asrecarray=True)
    return data
Ejemplo n.º 31
0

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.plot(x,y,z,color = 'purple')
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
plt.legend([r'$x_0 = {0}$ $y_0 = {1}$ $z_0 = {2}$'.format(x[0],y[0],z[0])],bbox_to_anchor=(0.78, 1.13))
ax.set_title(r'$\sigma = 10$ $\beta = 8/3$ $\rho = 28$')
plt.savefig('2_a.png')
plt.show()

'''

lorenz_b = np.recfromtxt('lorenz_b.txt', names=True)

x_b = lorenz_b['x']
y_b = lorenz_b['y']
z_b = lorenz_b['z']

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.plot(x_b, y_b, z_b, color='green')
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
plt.legend(
    [r'$x_0 = {0}$ $y_0 = {1}$ $z_0 = {2}$'.format(x_b[0], y_b[0], z_b[0])],
    bbox_to_anchor=(0.75, 1.13))
ax.set_title(r'$\sigma = 10$ $\beta = 8/3$ $\rho = 28$')
Ejemplo n.º 32
0
def runmain(argv=1):
    if argv == 1:
        argv = sys.argv

    usage = 'usage: %prog [options]\n'
    cmdlineParser = OptionParser(usage=usage)
    cmdlineParser.add_option(
        "--dataFilePattern",
        dest="pattern",
        default='../data/*.csv',
        help='Name of the datafile containing returns data (default: %default)'
    )
    cmdlineParser.add_option(
        "--randomData",
        dest="random",
        default=False,
        help='Use randomly genrated data (default: %default)')

    (cmdoptions, args) = cmdlineParser.parse_args(argv)
    random = cmdoptions.random
    outfile = open('returnsPA.txt', 'w')
    files = glob.glob(cmdoptions.pattern)
    sectorNames = []
    cumPReturns = 1
    cumBReturns = 1

    for f in files:
        linkedAttr
        if len(sectorNames) == 0:
            data = np.recfromtxt(f, delimiter=',', names=True)
            names = data.dtype.names
            sectors = {}
            sectorNames = set(data['Sector'])
            sectorNames = list(sectorNames)

            linkedAttr = np.zeros((len(sectorNames), 4))

        outfile.write(f + '\n')

        effectNames, attr, overallPortfolioReturns, overallBenchmarkReturns = runPA(
            f, random, sectorNames, outfile)
        periodCoeff = (np.log(1 + overallPortfolioReturns) -
                       np.log(1 + overallBenchmarkReturns)) / (
                           overallPortfolioReturns - overallBenchmarkReturns)
        linkedAttr += attr * periodCoeff
        cumPReturns = cumPReturns * (1 + overallPortfolioReturns)
        cumBReturns = cumBReturns * (1 + overallBenchmarkReturns)

    overallCoeff = (np.log(cumPReturns) - np.log(cumBReturns)) / (cumPReturns -
                                                                  cumBReturns)
    linkedAttr = linkedAttr / overallCoeff

    header = 'Sector'

    for effect in effectNames:
        header = header + ',' + effect

    outfile.write(header + '\n')
    for sector in sectorNames:
        dataStr = sector
        index = sectorNames.index(sector)
        data = linkedAttr[sectorNames.index(sector)]
        count = 0
        while count < len(data):
            dataStr = str(dataStr) + ',' + str(data[count])
            count = count + 1
        outfile.write(dataStr + '\n')

    cumActiveReturns = cumPReturns - cumBReturns
    linkedSumofEffects = linkedAttr.sum()
    if (abs(cumActiveReturns - linkedSumofEffects) < 0.0001):
        print(True)
    else:
        print(False)

    outfile.write('overall cumulative active returns,' +
                  str(cumActiveReturns) + '\n')
    outfile.write('sum of linked effects,' + str(linkedSumofEffects) + '\n')
    outfile.close()
Ejemplo n.º 33
0
def runPA(filename, random, sectorNames, outfile):
    data = np.recfromtxt(filename, delimiter=',', names=True)  #数据
    names = data.dtype.names  #
    sectors = {}
    for name in sectorNames:
        sectors[name] = {}
    numRecord = len(data)
    count = 0
    portfolioWeights = {}
    benchmarkWeights = {}
    portfolioReturns = {}
    benchmarkReturns = {}
    interactionEffects = {}
    selectionEffects = {}
    styleAllocationEffects = {}
    sectorAllocationEffects = {}
    overallPortfolioReturns = 0
    overallBenchmarkReturns = 0
    sumPW = 0
    sumBW = 0
    while count < numRecord:
        record = data[count]
        count = count + 1
        sectorName = record[names.index('Sector')]
        styleName = sectorName + b'.' + record[names.index('Style')]
        assetName = record[names.index('Asset')]
        if random == True:
            assetReturns = (2 * np.random.rand() - 1) / 10
            if count == numRecord:
                assetPW = 1 - sumPW
                assetBW = 1 - sumBW
            else:
                assetPW = np.random.rand() / 15
                assetBW = np.random.rand() / 15
                sumPW += assetPW
                sumBW += assetBW
        else:
            assetReturns = record[names.index('Asset_Returns')]
            assetPW = record[names.index('Portfolio_Weight')]
            assetBW = record[names.index('Benchmark_Weight')]

        assetPReturns = assetPW * assetReturns
        assetBReturns = assetBW * assetReturns
        overallPortfolioReturns += assetPReturns
        overallBenchmarkReturns += assetBReturns
        sectorDict = sectors[sectorName]
        try:
            styleDict = sectorDict[styleName]
        except KeyError:
            styleDict = {}
            sectorDict[styleName] = styleDict
        styleDict[assetName] = assetReturns

        addToDict(portfolioWeights, styleName, assetPW)
        addToDict(portfolioWeights, sectorName, assetPW)
        addToDict(portfolioReturns, styleName, assetPReturns)
        addToDict(portfolioReturns, sectorName, assetPReturns)

        addToDict(benchmarkWeights, styleName, assetBW)
        addToDict(benchmarkWeights, sectorName, assetBW)
        addToDict(benchmarkReturns, styleName, assetBReturns)
        addToDict(benchmarkReturns, sectorName, assetBReturns)

        portfolioWeights[assetName] = assetPW
        benchmarkWeights[assetName] = assetBW
        portfolioReturns[assetName] = assetReturns
        benchmarkReturns[assetName] = assetReturns

    for sector, styleDict in sectors.items():
        sectorPW = portfolioWeights.get(sector)
        sectorBW = benchmarkWeights.get(sector)
        sectorPR = portfolioReturns.get(sector)
        sectorBR = benchmarkReturns.get(sector)
        sectorIntact = 0
        sectorSelec = 0
        sectorStyleAlloc = 0
        sectorAlloc = (sectorPW - sectorBW) * (sectorBR / sectorBW -
                                               overallBenchmarkReturns)
        for style, styDict in styleDict.items():
            stylePW = portfolioWeights.get(style)
            styleBW = benchmarkWeights.get(style)
            stylePR = portfolioReturns.get(style)
            styleBR = benchmarkReturns.get(style)
            styleIntact = 0
            styleSelec = 0
            styleAlloc = sectorPW * (
                stylePW / sectorPW -
                styleBW / sectorBW) * (styleBR / styleBW - sectorBR / sectorBW)
            styleAllocationEffects[style] = styleAlloc
            sectorStyleAlloc += styleAlloc
            for asset in styDict.keys():
                assetPW = portfolioWeights.get(asset)
                assetBW = benchmarkWeights.get(asset)
                assetPR = portfolioReturns.get(asset)
                assetBR = benchmarkReturns.get(asset)
                selec = styleBW * (assetPW / stylePW - assetBW / styleBW) * (
                    assetPR - styleBR / styleBW)
                intact = (assetPW - assetBW) * (stylePR / stylePW -
                                                styleBR / styleBW)
                interactionEffects[asset] = selec
                selectionEffects[asset] = intact
                styleIntact += intact
                styleSelec += selec
            interactionEffects[style] = styleIntact
            selectionEffects[style] = styleSelec
            sectorIntact += styleIntact
            sectorSelec += styleSelec
        interactionEffects[sector] = sectorIntact
        selectionEffects[sector] = sectorSelec
        styleAllocationEffects[sector] = sectorStyleAlloc
        sectorAllocationEffects[sector] = sectorAlloc

    overallActiveReturns = overallPortfolioReturns - overallBenchmarkReturns
    checkSum(sectors, sectorAllocationEffects, styleAllocationEffects,
             selectionEffects, interactionEffects, overallActiveReturns,
             outfile)

    effectNames = [
        'sectorAllocationEffects', 'styleAllocationEffects',
        'selectionEffects', 'interactionEffects'
    ]
    attr = np.zeros((len(sectorNames), 4))
    for sector in sectorNames:
        j = 0
        for data in (sectorAllocationEffects, styleAllocationEffects,
                     selectionEffects, interactionEffects):
            attr[sectorNames.index(sector), j] = data[sector]
            j = j + 1

    outfile.write('overallPortfolioReturns' + ',' +
                  str(overallPortfolioReturns) + '\n')
    outfile.write('overallBenchmarkReturns' + ',' +
                  str(overallBenchmarkReturns) + '\n')
    return effectNames, attr, overallPortfolioReturns, overallBenchmarkReturns
Ejemplo n.º 34
0
Archivo: npyio.py Proyecto: ekand/numpy
import pathlib
from typing import IO

import numpy.typing as npt
import numpy as np

str_path: str
bytes_path: bytes
pathlib_path: pathlib.Path
str_file: IO[str]
AR_i8: npt.NDArray[np.int64]

np.load(str_file)  # E: incompatible type

np.save(bytes_path, AR_i8)  # E: incompatible type
np.save(str_file, AR_i8)  # E: incompatible type

np.savez(bytes_path, AR_i8)  # E: incompatible type
np.savez(str_file, AR_i8)  # E: incompatible type

np.savez_compressed(bytes_path, AR_i8)  # E: incompatible type
np.savez_compressed(str_file, AR_i8)  # E: incompatible type

np.loadtxt(bytes_path)  # E: incompatible type

np.fromregex(bytes_path, ".", np.int64)  # E: No overload variant

np.recfromtxt(bytes_path)  # E: incompatible type

np.recfromcsv(bytes_path)  # E: incompatible type
Ejemplo n.º 35
0
def _get_data():
    filepath = dirname(abspath(__file__))
    ##### EDIT THE FOLLOWING TO POINT TO DatasetName.csv #####
    with open(filepath + '/cancer.csv', 'rb') as f:
        data = np.recfromtxt(f, delimiter=",", names=True, dtype=float)
        return data
Ejemplo n.º 36
0
def _get_data():
    filepath = dirname(abspath(__file__))
    with open(filepath + '/stackloss.csv', "rb") as f:
        data = recfromtxt(f, delimiter=",", names=True, dtype=float)
        return data
Ejemplo n.º 37
0

logger = logging.getLogger(__name__)

tardis_dir = os.path.realpath(tardis.__path__[0])


def get_data_path(fname):
    return os.path.join(tardis_dir, 'data', fname)


def get_tests_data_path(fname):
    return os.path.join(tardis_dir, 'tests', 'data', fname)


atomic_symbols_data = np.recfromtxt(get_data_path('atomic_symbols.dat'),
                                    names=['atomic_number', 'symbol'])
symbol2atomic_number = OrderedDict(
    zip(atomic_symbols_data['symbol'], atomic_symbols_data['atomic_number']))
atomic_number2symbol = OrderedDict(atomic_symbols_data)

synpp_default_yaml_fname = get_data_path('synpp_default.yaml')


def int_to_roman(int_input):
    """
    from http://code.activestate.com/recipes/81611-roman-numerals/
    Convert an integer to Roman numerals.

    :param int_input: an integer between 1 and 3999
    :returns result: roman equivalent string of passed :param{int_input}
Ejemplo n.º 38
0
def _get_data():
    filepath = dirname(abspath(__file__))
    data = np.recfromtxt(open(filepath + '/modechoice.csv', 'rb'),
            delimiter=";", names = True, dtype=float)
    return data
Ejemplo n.º 39
0
import numpy as np
import pickle,gzip
import numpy as np, pickle, gzip, sys, glob, time, os, scipy as sc
sys.path.append('modules')
import modules
import sky_local
from pylab import *
sims = modules.scl_cmb.simulations()
ip_folder = sys.argv[1]

fnamearr = glob.glob('%s/stacked*_03500_clusters*'%(ip_folder))
paramfile = glob.glob('%s/*used.txt'%(ip_folder))[0]
kappa_file = glob.glob('%s/kappa_COV*100cluters.pkl.gz'%(ip_folder))[0]
kappa_COV = pickle.load(gzip.open(kappa_file))
params = np.recfromtxt(paramfile,usecols=[0],delimiter = '=')
paramvals = np.recfromtxt(paramfile,usecols=[1],delimiter = '=')

param_dict = {}
cc = 3.0
for p,pval in zip(params,paramvals):
        tmp = pval.strip()
        try:
                float(tmp)
                if tmp.find('.')>-1:
                        param_dict[p.strip()] = float(tmp)
                else:
                        param_dict[p.strip()] = int(tmp)
        except:
                if tmp == 'None':
                        param_dict[p.strip()] = None
Ejemplo n.º 40
0
def csv2rec(filename):
    return np.recfromtxt(filename,
                         dtype=None,
                         delimiter=',',
                         names=True,
                         encoding='utf-8')
Ejemplo n.º 41
0
def _get_data():
    filepath = dirname(abspath(__file__))
    data = recfromtxt(open(PATH, "rb"), delimiter=",", names=True, dtype=float)
    return data
Ejemplo n.º 42
0
import numpy as np
import matplotlib.pyplot as plt
from scipy.fftpack import fft, fftfreq
from scipy import interpolate
import matplotlib.gridspec as gridspec

data = "rev26_27.dat"
recdat = np.recfromtxt(data, names=["time", "r", "theta", "phi"])
r = recdat.r
t = recdat.time

not_nan = np.logical_not(np.isnan(r))
i = np.arange(len(r))
rinterp = np.interp(i, i[not_nan], r[not_nan])

period = 1 / fftfreq(34504, (t[-1] - t[0]) / 34504)
fftamp = fft(rinterp)

sort = np.argsort(fftamp)
fftamp = fftamp[sort]
period = period[sort]

gs = gridspec.GridSpec(8, 1)

plt.figure(figsize=[12, 12])

ax1 = plt.subplot(gs[:2, :])
ax1.plot(t, rinterp, "r")
ax1.plot(t, r, "b")
ax1.set_xlim([934.9, 959])
ax1.set_xlabel("time since 2004(days)")
Ejemplo n.º 43
0
def p(m):
    fig = ml.figure()
    brain = surfer.Brain('mni', 'rh', 'cortex', curv=False, figure=fig)
    brain.add_data(m[:, v])


fname1 = env.data + '/structural/thalamusR_SA_NV_10to21_2closestTo16.csv'
fname2 = env.data + '/structural/thalamusR_SA_NV_10to21.csv'

cortex = np.genfromtxt(fname1, delimiter=',')
# removing first column and first row, because they're headers
cortex = scipy.delete(cortex, 0, 1)
cortex = scipy.delete(cortex, 0, 0)
# format it to be subjects x variables
cortex = cortex.T
c_names = np.recfromtxt(fname1, delimiter=',')[0][1:]

cortex2 = np.genfromtxt(fname2, delimiter=',')
# removing first column and first row, because they're headers
cortex2 = scipy.delete(cortex2, 0, 1)
cortex2 = scipy.delete(cortex2, 0, 0)
# format it to be subjects x variables
cortex2 = cortex2.T
c2_names = np.recfromtxt(fname2, delimiter=',')[0][1:]

# we have a different number of subjects in the two files, so let's crop the extra subjects in the bigger file to fit the smaller one
nvoxels = cortex.shape[1]

# we'll assume that cortex has more subjects than cortex2
if cortex.shape[0] < cortex2.shape[0]:
    tmp = cortex.copy()
Ejemplo n.º 44
0
#! /usr/bin/env python
# fit a collection to T1 trajectories to a decaying exponential

import scipy.optimize
import numpy as np
import pickle

# read in the trajectories and times
trajs = np.load("traj.npy")
t1 = np.recfromtxt("time.dat")


# fitting function and residual calculation
def fit_func(p, x):
    A, R2 = p
    # bound A between 0.98 and 1.02 (although fits do not reflect this)
    if A > 1.02:
        A = 1.02
    if A < 0.98:
        A = 0.98

    return A * np.exp(-1.0 * np.array(x) * R2 / 1.0e6)


def residuals(p, y, x):
    err = y - fit_func(p, x)
    return err


p0 = [1.0, 0.05]  # initial guess
Ejemplo n.º 45
0
#############################################################################
# Retrieve the files of the Haxby dataset
from nilearn import datasets

haxby_dataset = datasets.fetch_haxby_simple()

# print basic information on the dataset
print('Mask nifti image (3D) is located at: %s' % haxby_dataset.mask)
print('Functional nifti image (4D) is located at: %s' %
      haxby_dataset.func[0])

#############################################################################
# Load the behavioral data
import numpy as np
y, session = np.loadtxt(haxby_dataset.session_target[0]).astype("int").T
conditions = np.recfromtxt(haxby_dataset.conditions_target[0])['f0']

# Restrict to faces and houses
condition_mask = np.logical_or(conditions == b'face', conditions == b'house')
y = y[condition_mask]
conditions = conditions[condition_mask]

# We have 2 conditions
n_conditions = np.size(np.unique(y))

#############################################################################
# Prepare the fMRI data
from nilearn.input_data import NiftiMasker

mask_filename = haxby_dataset.mask
# For decoding, standardizing is often very important
Ejemplo n.º 46
0
def _get_data():
    filepath = dirname(abspath(__file__))
    with open(filepath+'/longley.csv',"rb") as f:
        data = recfromtxt(f, delimiter=",",
                          names=True, dtype=float, usecols=(1,2,3,4,5,6,7))
    return data
Ejemplo n.º 47
0
 def _read_from_text(self, infile):
     data = np.recfromtxt(infile)
     data = data.transpose()
     self.flux = data[0]
     self.Ne = dict([(amp, ne)
                     for amp, ne in zip(imutils.allAmps(), data[1:])])
Ejemplo n.º 48
0
#! /usr/bin/env python
# Example scipt to show integration of a 1D spectrum

import nmrglue as ng
import numpy as np
import matplotlib.pyplot as plt

# read in the data from a NMRPipe file
dic, data = ng.pipe.read("1d_data.ft")
length = data.shape[0]

# read in the integration limits
peak_list = np.recfromtxt("limits.in")

# determind the ppm scale
uc = ng.pipe.make_uc(dic, data)
ppm_scale = uc.ppm_scale()

# plot the spectrum
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(ppm_scale, data, 'k-')

# prepare the output file
f = open("area.out", 'w')
f.write("#Name\tStart\tStop\tArea\n")

# loop over the integration limits
for name, start, end in peak_list:
    min = uc(start, "ppm")
    max = uc(end, "ppm")
Ejemplo n.º 49
0
#from transit import Central, System, Body

if not on_rtd:
    import astropy.constants as const
    AU = const.au.cgs.value
    RSUN = const.R_sun.cgs.value
    MSUN = const.M_sun.cgs.value
    REARTH = const.R_earth.cgs.value
    MEARTH = const.M_earth.cgs.value
    G = const.G.cgs.value
    DAY = 86400.

    DATAFOLDER = os.path.abspath(
        os.path.join(os.path.dirname(__file__), 'data'))

    LDDATA = np.recfromtxt('{}/keplerld.dat'.format(DATAFOLDER), names=True)
    LDOK = ((LDDATA.teff < 10000) & (LDDATA.logg > 2.0) & (LDDATA.feh > -2))
    LDPOINTS = np.array([LDDATA.teff[LDOK], LDDATA.logg[LDOK]]).T
    U1FN = interpnd(LDPOINTS, LDDATA.u1[LDOK])
    U2FN = interpnd(LDPOINTS, LDDATA.u2[LDOK])
else:
    const, AU, RSUN, MSUN = (None, None, None, None)
    REARTH, MEARTH, DAY = (None, None, None)
    DATAFOLDER = None
    LDDATA, LDOK, LDPOINTS, U1FN, U2FN = (None, None, None, None, None)

MAXSLOPE = 30


def ldcoeffs(teff, logg=4.5, feh=0):
    """
from scipy import stats

bands = [[1, 4], [4, 8], [8, 13], [13, 30], [30, 50]]
sx_fname = '/Users/sudregp/data/meg/inatt.txt'
subjs_fname = '/Users/sudregp/data/meg/usable_subjects_5segs13p654.txt'
data_dir = '/Users/sudregp/data/meg/connectivity/'
lmethod = 'pca_flip'
cmethod = 0

selected_labels = []
# selected_labels = ['isthmuscingulate-rh', 'superiorfrontal-rh', 'inferiorparietal-rh', 'isthmuscingulate-lh', 'superiorfrontal-lh', 'inferiorparietal-lh']
# selected_labels = ['isthmuscingulate-rh', 'superiorfrontal-rh', 'inferiorparietal-rh']

fid = open(subjs_fname, 'r')
subjs = [line.rstrip() for line in fid]
res = np.recfromtxt(sx_fname, delimiter='\t')
sx = {}
for rec in res:
    sx[rec[0]] = rec[1]

print 'sx =',sx_fname
m = ['pli','imcoh','plv','wpli','pli2_unbiased','wpli2_debiased']
print lmethod, '-', m[cmethod]

labels, label_colors = mne.labels_from_parc(subjs[0], parc='aparc')
nlabels=len(labels)
il = np.tril_indices(nlabels, k=-1)
if len(selected_labels)>0:
    label_names = [l.name for l in labels]
    idx = [l for s in selected_labels for l, label in enumerate(label_names) if label == s]
    keep = [False]*len(il[0])
Ejemplo n.º 51
0
    parser = argparse.ArgumentParser(description="Take output from Bayesian block algorithm and cross match with "
                                                 "previously flagged variable sources")
    parser.add_argument("--bbfile", help="Path of input text file", required=True)
    parser.add_argument("--tsvfile", help="Path of tsv file", required=True)
    parser.add_argument("--outfile", help="Path of out file", required=True)

    args = parser.parse_args()

    # get directory path and file name from input file arguments

    bb_file_path = os.path.abspath(os.path.expandvars(os.path.expanduser(args.bbfile)))
    tsv_file_path = os.path.abspath(os.path.expandvars(os.path.expanduser(args.tsvfile)))

    # read BB data into array
    bb_data = np.array(np.recfromtxt(bb_file_path, names=True), ndmin=1)

    # number of rows of data
    bb_n = len(bb_data)

    # read tsv data into array
    tsv_data = np.recfromtxt(tsv_file_path, delimiter='\t', skip_header=11, names=True)

    # Filter out all non-variable sources

    variability = np.array(map(lambda x: x.replace(" ", "") == "TRUE", tsv_data['var_flag']))

    idx = (variability == True)

    variable_sources = tsv_data[idx]
Ejemplo n.º 52
0
def _get_data():
    filepath = dirname(abspath(__file__))
    with open(filepath + '/macrodata.csv', 'rb') as f:
        data = recfromtxt(f, delimiter=",",
                          names=True, dtype=float)
    return data
Ejemplo n.º 53
0
                    mydate = dt.datetime.strptime(data[i][date_idx],
                                                  '%m/%d/%y')
                    data[i][date_idx] = dt.datetime.strftime(
                        mydate, '%m/%d/%Y')
                except:
                    try:
                        dt.datetime.strptime(data[i][date_idx], '%m/%d/%Y')
                    except:
                        print data[i][date_idx], 'is not a valid date!',\
                              ' (row %d)'%(i+2)
                        remove_rows.append(i)
    data = np.delete(data, remove_rows, axis=0)
    return data


data1 = np.recfromtxt(fname1, delimiter='\t')
data2 = np.recfromtxt(fname2, delimiter='\t')
hdr1 = data1[0]
hdr2 = data2[0]

# let's do a series of quick checks to clean up the data
data1 = clean_up(data1[1:], mrn1, date1)
data2 = clean_up(data2[1:], mrn2, date2)

merged_data = []
# do this for rows in data1
for row in range(len(data1)):
    # store row indexes in data2 that match the current mrn
    matching_rows = [
        i for i in range(len(data2))
        if int(data1[row][mrn1]) == int(data2[i][mrn2])
Ejemplo n.º 54
0
    'USA', 'France', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA',
    'USA', 'USA', 'Germany', 'Japan', 'USA', 'USA', 'USA', 'USA', 'Germany',
    'Japan', 'Japan', 'USA', 'Sweden', 'USA', 'France', 'Japan', 'Germany',
    'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA',
    'USA', 'USA', 'Germany', 'Japan', 'Japan', 'USA', 'USA', 'Japan', 'Japan',
    'Japan', 'Japan', 'Japan', 'Japan', 'USA', 'USA', 'USA', 'USA', 'Japan',
    'USA', 'USA', 'USA', 'Germany', 'USA', 'USA', 'USA'
])

#accommodate recfromtxt for python 3.2, requires bytes
ss = asbytes(ss)
ss2 = asbytes(ss2)
ss3 = asbytes(ss3)
ss5 = asbytes(ss5)

dta = np.recfromtxt(BytesIO(ss), names=("Rust", "Brand", "Replication"))
dta2 = np.recfromtxt(BytesIO(ss2),
                     names=("idx", "Treatment", "StressReduction"))
dta3 = np.recfromtxt(BytesIO(ss3), names=("Brand", "Relief"))
dta5 = np.recfromtxt(BytesIO(ss5),
                     names=('pair', 'mean', 'lower', 'upper', 'sig'),
                     delimiter='\t')
sas_ = dta5[[1, 3, 2]]

from statsmodels.stats.multicomp import (tukeyhsd, pairwise_tukeyhsd,
                                         MultiComparison)
#import statsmodels.sandbox.stats.multicomp as multi
#print tukeyhsd(dta['Brand'], dta['Rust'])


def get_thsd(mci, alpha=0.05):
    data_dir = home + '/data/results/meg_Yeo/seeds/%s/' % sys.argv[1]
    my_test = sys.argv[2]
# data_dir = home + '/data/results/meg_Yeo/seeds/net1_lMidOccipital/'
# mytest = 'inatt'
thresh = .95
nperms = 5000
dist = 5  # distance between voxels to be considered connected (mm)
alphas = [.05, .01]

fid1 = open(g1_fname, 'r')
fid2 = open(g2_fname, 'r')
fid3 = open(g3_fname, 'r')
g1 = [line.rstrip() for line in fid1]
g2 = [line.rstrip() for line in fid2]
g3 = [line.rstrip() for line in fid3]
res = np.recfromtxt(inatt_fname, delimiter='\t')
inatt = {}
for rec in res:
    inatt[rec[0]] = rec[1]
res = np.recfromtxt(hi_fname, delimiter='\t')
hi = {}
for rec in res:
    hi[rec[0]] = rec[1]

print 'g1 =', g1_fname
print 'g2 =', g2_fname
print 'g3 =', g3_fname

vox_pos = np.genfromtxt(home +
                        '/data/meg/sam_narrow_5mm/brainTargetsInTLR.txt',
                        skip_header=1)
Ejemplo n.º 56
0
#read in the flat data
print('Reading in flat data')
flatData = np.zeros([2, 4, 256, 256], dtype=np.double)  #delay data
#just doing ch1/2 for warm mission right now, leaving in option for others if/when those corrections become available
for cryo in range(0, 1):
    for Ch in range(1, 3):
        flatHDU = fits.open(flatFiles[cryo, Ch - 1])
        flatData[cryo, Ch - 1] = flatHDU[0].data

#read in the delay files
print('Reading in Frame Delay Data')

#Read the delay files for the first frame correction
delayInfo = np.recfromtxt(
    FrameDelayFile,
    dtype=[
        ('frame', np.int),  #Number
        ('delay', np.float32)  #delay since last frame
    ])

#create a lookup index for the frame delay correction
delay_to_index = interp1d(delayInfo.delay,
                          delayInfo.frame,
                          bounds_error=False,
                          fill_value='extrapolate')

#Read the fits files
delayData = np.zeros([2, len(delayInfo), 256, 256],
                     dtype=np.double)  #delay data
for i in range(0, len(delayInfo)):
    for Ch in range(1, 3):
        delayFile = './cal/labdark.' + str(i) + '.' + str(Ch) + '.fits'
Ejemplo n.º 57
0
data_dir = expanduser('~') + '/downloads/pymvpa-exampledata/'

# create sklearn's Bunch of data
dataset_files = Bunch(
    func=data_dir+'bold.nii.gz',
    session_target=data_dir+'attributes.txt',
    mask=data_dir+'mask.nii.gz',
    conditions_target=data_dir+'attributes_literal.txt'
    )

# fmri_data and mask are copied to break any reference to the original object
bold_img = nibabel.load(dataset_files.func)
fmri_data = bold_img.get_data().astype(float)
affine = bold_img.get_affine()
y, session = np.loadtxt(dataset_files.session_target).astype("int").T
conditions = np.recfromtxt(dataset_files.conditions_target)['f0']
mask = dataset_files.mask

# fmri_data.shape is (40, 64, 64, 1452)
# and mask.shape is (40, 64, 64)

# ### Preprocess data
# Build the mean image because we have no anatomic data
mean_img = fmri_data.mean(axis=-1)


# ### Restrict to faces and houses
condition_mask = np.logical_or(conditions == 'face', conditions == 'house')
X = fmri_data[..., condition_mask]
y = y[condition_mask]
# session = session[condition_mask]
Ejemplo n.º 58
0
    def prepare_fields(self,
                       infile=None,
                       outfile=None,
                       mode='smash_dither',
                       plot=True,
                       smcnod=False):
        """ Create the list of fields to be targeted by this survey.

        Parameters:
        -----------
        infile : File containing all possible field locations.
        outfile: Output file of selected fields
        mode   : Mode for dithering: 'smash_dither', 'smash_rotate', 'decam_dither', 'none'
        plot   : Create an output plot of selected fields.

        Returns:
        --------
        fields : A FieldArray of the selected fields.
        """
        # Import the dither function here...
        #def dither(ra,dec,dx,dy):
        #    return ra,dec

        if mode is None or mode.lower() == 'none':

            def dither(ra, dec, dx, dy):
                return ra, dec

            TILINGS = [(0, 0), (0, 0), (0, 0), (0, 0)]
        elif mode.lower() == 'smash_dither':
            TILINGS = [(0, 0), (1.0, 0.0), (-1.0, 0.0), (0.0, -0.75)]
            dither = self.smash_dither
        elif mode.lower() == 'smash_rotate':
            TILINGS = [(0, 0), (0.75, 0.75), (-0.75, 0.75), (0.0, -0.75)]
            dither = self.smash_rotate
        elif mode.lower() == 'decam_dither':
            TILINGS = [(0., 0.), (8 / 3. * CCD_X, -11 / 3. * CCD_Y),
                       (8 / 3. * CCD_X, 8 / 3. * CCD_Y), (-8 / 3. * CCD_X, 0.)]
            dither = self.decam_dither

        if infile is None:
            infile = os.path.join(fileio.get_datadir(),
                                  'smash_fields_alltiles.txt')
        data = np.recfromtxt(infile, names=True)

        # Apply footprint selection after tiling/dither
        #sel = obztak.utils.projector.footprint(data['RA'],data['DEC'])

        # This is currently a non-op
        smash_id = data['ID']
        ra = data['RA']
        dec = data['DEC']

        nhexes = len(data)
        #ntilings = len(DECAM_DITHERS)
        ntilings = len(TILINGS)
        nbands = len(BANDS)
        nfields = nhexes * nbands * ntilings

        logging.info("Number of hexes: %d" % nhexes)
        logging.info("Number of tilings: %d" % ntilings)
        logging.info("Number of filters: %d" % nbands)

        fields = FieldArray(nfields)
        fields['HEX'] = np.tile(np.repeat(smash_id, nbands), ntilings)
        fields['PRIORITY'].fill(1)
        fields['TILING'] = np.repeat(np.arange(1, ntilings + 1),
                                     nhexes * nbands)
        fields['FILTER'] = np.tile(BANDS, nhexes * ntilings)

        #for i in range(ntilings):
        for i, tiling in enumerate(TILINGS):
            idx0 = i * nhexes * nbands
            idx1 = idx0 + nhexes * nbands
            ra_dither, dec_dither = dither(ra, dec, tiling[0], tiling[1])
            fields['RA'][idx0:idx1] = np.repeat(ra_dither, nbands)
            fields['DEC'][idx0:idx1] = np.repeat(dec_dither, nbands)

        # Apply footprint selection after tiling/dither
        sel = self.footprint(fields['RA'], fields['DEC'])  # NORMAL OPERATION
        if smcnod:
            # Include SMC northern overdensity fields
            sel_smcnod = self.footprintSMCNOD(fields)  # SMCNOD OPERATION
            sel = sel | sel_smcnod
            #sel = sel_smcnod
            fields['PRIORITY'][sel_smcnod] = 99
        #if True:
        #    # Include 'bridge' region between Magellanic Clouds
        #    sel_bridge = self.footprintBridge(fields['RA'],fields['DEC'])
        #    sel = sel | sel_bridge
        sel = sel & (fields['DEC'] > constants.SOUTHERN_REACH)
        fields = fields[sel]

        logging.info("Number of target fields: %d" % len(fields))

        if plot:
            import pylab as plt
            import obztak.utils.ortho

            plt.ion()

            fig, basemap = obztak.utils.ortho.makePlot('2016/2/11 03:00',
                                                       center=(0, -90),
                                                       airmass=False,
                                                       moon=False)

            proj = obztak.utils.ortho.safeProj(basemap, fields['RA'],
                                               fields['DEC'])
            basemap.scatter(*proj,
                            c=fields['TILING'],
                            edgecolor='none',
                            s=50,
                            cmap='Spectral',
                            vmin=0,
                            vmax=len(TILINGS))
            colorbar = plt.colorbar(label='Tiling')

            if outfile:
                outfig = os.path.splitext(outfile)[0] + '.png'
                fig.savefig(outfig, bbox_inches='tight')
            if not sys.flags.interactive:
                plt.show(block=True)

        if outfile: fields.write(outfile)

        return fields
Ejemplo n.º 59
0
        Option(
            "--objid",
            help="Event ID to be stored in FITS header [default: %default]"),
    ])
opts, args = parser.parse_args()
infilename = command.get_input_filename(parser, args)

if opts.output is None:
    parser.error('--output: missing required argument')

# Late imports.
import numpy as np
import lalinference.fits
import lalinference.bayestar.postprocess

samples = np.recfromtxt(infilename, names=True)
theta = 0.5 * np.pi - samples['dec']
phi = samples['ra']

p = lalinference.bayestar.postprocess.adaptive_healpix_histogram(
    theta,
    phi,
    opts.samples_per_bin,
    nside=opts.nside,
    max_nside=opts.max_nside)

# Write output to FITS file.
lalinference.fits.write_sky_map(opts.output,
                                p,
                                creator=parser.get_prog_name(),
                                objid=opts.objid,
Ejemplo n.º 60
0
def create_all():
    db.create_all(bind=None)

    telescopes = ["ZTF", "Gattini", "DECam", "KPED", "GROWTH-India"]
    available_filters = {
        "ZTF": ["g", "r", "i"],
        "Gattini": ["J"],
        "DECam": ["g", "r", "i", "z"],
        "KPED": ["U", "g", "r", "i"],
        "GROWTH-India": ["g", "r", "i", "z"]
    }

    plan_args = {
        'ZTF': {
            'filt': ['g', 'r', 'g'],
            'exposuretimes': [300.0, 300.0, 300.0],
            'doReferences': True,
            'doUsePrimary': True,
            'doBalanceExposure': False,
            'doDither': False,
            'usePrevious': False,
            'doCompletedObservations': False,
            'doPlannedObservations': False,
            'cobs': [None, None],
            'schedule_type': 'greedy',
            'filterScheduleType': 'block',
            'airmass': 2.5,
            'schedule_strategy': 'tiling',
            'mindiff': 30. * 60.,
            'doMaxTiles': False,
            'max_nb_tiles': 1000
        },
        'DECam': {
            'filt': ['g', 'z'],
            'exposuretimes': [25.0, 25.0],
            'doReferences': True,
            'doUsePrimary': False,
            'doBalanceExposure': False,
            'doDither': True,
            'usePrevious': False,
            'doCompletedObservations': False,
            'doPlannedObservations': False,
            'cobs': [None, None],
            'schedule_type': 'greedy_slew',
            'filterScheduleType': 'integrated',
            'airmass': 2.5,
            'schedule_strategy': 'tiling',
            'mindiff': 30. * 60.,
            'doMaxTiles': False,
            'max_nb_tiles': 1000
        },
        'Gattini': {
            'filt': ['J'],
            'exposuretimes': [300.0],
            'doReferences': False,
            'doUsePrimary': False,
            'doBalanceExposure': False,
            'doDither': False,
            'usePrevious': False,
            'doCompletedObservations': False,
            'doPlannedObservations': False,
            'cobs': [None, None],
            'schedule_type': 'greedy',
            'filterScheduleType': 'block',
            'airmass': 2.5,
            'schedule_strategy': 'tiling',
            'mindiff': 30. * 60.,
            'doMaxTiles': False,
            'max_nb_tiles': 1000
        },
        'KPED': {
            'filt': ['r'],
            'exposuretimes': [300.0],
            'doReferences': False,
            'doUsePrimary': False,
            'doBalanceExposure': False,
            'doDither': False,
            'usePrevious': False,
            'doCompletedObservations': False,
            'doPlannedObservations': False,
            'cobs': [None, None],
            'schedule_type': 'greedy',
            'filterScheduleType': 'integrated',
            'airmass': 2.5,
            'schedule_strategy': 'catalog',
            'mindiff': 30. * 60.,
            'doMaxTiles': False,
            'max_nb_tiles': 1000
        },
        'GROWTH-India': {
            'filt': ['r'],
            'exposuretimes': [300.0],
            'doReferences': False,
            'doUsePrimary': False,
            'doBalanceExposure': False,
            'doDither': False,
            'usePrevious': False,
            'doCompletedObservations': False,
            'doPlannedObservations': False,
            'cobs': [None, None],
            'schedule_type': 'greedy',
            'filterScheduleType': 'integrated',
            'airmass': 2.5,
            'schedule_strategy': 'catalog',
            'mindiff': 30. * 60.,
            'doMaxTiles': False,
            'max_nb_tiles': 1000
        }
    }

    with tqdm(telescopes) as telescope_progress:
        for tele in telescope_progress:
            telescope_progress.set_description('populating {}'.format(tele))

            filename = pkg_resources.resource_filename(__name__,
                                                       'input/%s.ref' % tele)
            if os.path.isfile(filename):
                refstable = table.Table.read(filename,
                                             format='ascii',
                                             data_start=2,
                                             data_end=-1)
                refs = table.unique(refstable, keys=['field', 'fid'])
                if "maglimcat" not in refs.columns:
                    refs["maglimcat"] = np.nan

                reference_images = {
                    group[0]['field']: group['fid'].astype(int).tolist()
                    for group in refs.group_by('field').groups
                }
                reference_mags = {
                    group[0]['field']: group['maglimcat'].tolist()
                    for group in refs.group_by('field').groups
                }

            else:
                reference_images = {}
                reference_mags = {}

            tesspath = 'input/%s.tess' % tele
            try:
                tessfile = app.open_instance_resource(tesspath)
            except IOError:
                tessfile = pkg_resources.resource_stream(__name__, tesspath)
            tessfilename = tessfile.name
            tessfile.close()
            fields = np.recfromtxt(tessfilename,
                                   usecols=range(3),
                                   names=['field_id', 'ra', 'dec'])

            with pkg_resources.resource_stream(__name__,
                                               'config/%s.config' % tele) as g:
                config_struct = {}
                for line in g.readlines():
                    line_without_return = line.decode().split("\n")
                    line_split = line_without_return[0].split(" ")
                    line_split = list(filter(None, line_split))
                    if line_split:
                        try:
                            config_struct[line_split[0]] = float(line_split[1])
                        except ValueError:
                            config_struct[line_split[0]] = line_split[1]

            db.session.merge(
                Telescope(telescope=tele,
                          lat=config_struct["latitude"],
                          lon=config_struct["longitude"],
                          elevation=config_struct["elevation"],
                          timezone=config_struct["timezone"],
                          filters=available_filters[tele],
                          default_plan_args=plan_args[tele]))

            for field_id, ra, dec in tqdm(fields, 'populating fields'):
                ref_filter_ids = reference_images.get(field_id, [])
                ref_filter_mags = []
                for val in reference_mags.get(field_id, []):
                    ref_filter_mags.append(val)
                bands = {1: 'g', 2: 'r', 3: 'i', 4: 'z', 5: 'J'}
                ref_filter_bands = [bands.get(n, n) for n in ref_filter_ids]

                if config_struct["FOV_type"] == "square":
                    ipix, radecs, patch, area = gwemopt.utils.getSquarePixels(
                        ra, dec, config_struct["FOV"], Localization.nside)
                elif config_struct["FOV_type"] == "circle":
                    ipix, radecs, patch, area = gwemopt.utils.getCirclePixels(
                        ra, dec, config_struct["FOV"], Localization.nside)
                if len(radecs) == 0:
                    continue
                corners = np.vstack((radecs, radecs[0, :]))
                if corners.size == 10:
                    corners_copy = copy.deepcopy(corners)
                    corners[2] = corners_copy[3]
                    corners[3] = corners_copy[2]
                contour = {
                    'type': 'Feature',
                    'geometry': {
                        'type': 'MultiLineString',
                        'coordinates': [corners.tolist()]
                    },
                    'properties': {
                        'telescope': tele,
                        'field_id': int(field_id),
                        'ra': ra,
                        'dec': dec,
                        'depth': dict(zip(ref_filter_bands, ref_filter_mags))
                    }
                }
                db.session.merge(
                    Field(telescope=tele,
                          field_id=int(field_id),
                          ra=ra,
                          dec=dec,
                          contour=contour,
                          reference_filter_ids=ref_filter_ids,
                          reference_filter_mags=ref_filter_mags,
                          ipix=ipix.tolist()))

            if tele == "ZTF":
                quadrant_coords = get_ztf_quadrants()

                skyoffset_frames = coordinates.SkyCoord(
                    fields['ra'], fields['dec'], unit=u.deg).skyoffset_frame()

                quadrant_coords_icrs = coordinates.SkyCoord(
                    *np.tile(quadrant_coords[:, np.newaxis, ...],
                             (len(fields), 1, 1)),
                    unit=u.deg,
                    frame=skyoffset_frames[:, np.newaxis,
                                           np.newaxis]).transform_to(
                                               coordinates.ICRS)

                quadrant_xyz = np.moveaxis(
                    quadrant_coords_icrs.cartesian.xyz.value, 0, -1)

                for field_id, xyz in zip(
                        tqdm(fields['field_id'], 'populating subfields'),
                        quadrant_xyz):
                    for ii, xyz in enumerate(xyz):
                        ipix = hp.query_polygon(Localization.nside, xyz)
                        db.session.merge(
                            SubField(telescope=tele,
                                     field_id=int(field_id),
                                     subfield_id=int(ii),
                                     ipix=ipix.tolist()))