def read_data(data_files): x_files, y_files = data_files XX, Y = None, None for x_file in x_files: print 'Reading..', x_file sys.stdout.flush() X_subject = np.recfromtxt(x_file, delimiter=',') if XX is None: XX = X_subject else: XX = np.concatenate((XX, X_subject)) XX = XX.reshape((XX.shape[0], 3, config.width, config.height, config.time_slice)) #print XX.shape for y_file in y_files: print 'Reading..', y_file sys.stdout.flush() Y_subject = np.recfromtxt(y_file) if Y is None: Y = Y_subject else: Y = np.concatenate((Y, Y_subject)) return XX, Y
def check_n_in_aper(radius_factor=1, k=100): for catfile in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"): print print catfile names = open(catfile).readline().split()[1:] cat = np.recfromtxt(catfile, names=names) xscfile = catfile.replace('combined_hdr_catalog.txt','2mass_xsc.tbl') print xscfile names = open(xscfile).read().split('\n')[76].split('|')[1:-1] xsc = np.recfromtxt(xscfile, skip_header=80, names=names) n_in_aper = [] coords = radec_to_coords(cat.ra, cat.dec) kdt = KDT(coords) for i in range(xsc.size): r_deg = xsc.r_ext[i]/3600. idx, ds = spherematch2(xsc.ra[i], xsc.dec[i], cat.ra, cat.dec, kdt, tolerance=radius_factor*r_deg, k=k) n_in_aper.append(ds.size) for i in [(i,n_in_aper.count(i)) for i in set(n_in_aper)]: print i
def initialize_database(self, configuration_dir=None): """Read in GMOS filter/grating information, for matching to headers.""" if configuration_dir is None: configuration_dir = default_configuration_dir logger.info('Reading Filter information') gmos_filters = np.recfromtxt( os.path.join(configuration_dir, 'GMOSfilters.dat'), names=['name', 'wave_start', 'wave_end', 'fname']) for line in gmos_filters: new_filter = GMOSFilter(name=line['name'], wavelength_start_value=line['wave_start'], wavelength_start_unit='nm', wavelength_end_value=line['wave_end'], wavelength_end_unit='nm', fname=line['fname'], path=os.path.join(configuration_dir, 'filter_data')) self.session.add(new_filter) open_filter = GMOSFilter(name='open', wavelength_start_value=0, wavelength_start_unit='nm', wavelength_end_value=np.inf, wavelength_end_unit='nm', fname=None, path=None) self.session.add(open_filter) gmos_gratings = np.recfromtxt( os.path.join(configuration_dir, 'GMOSgratings.dat'), names = ['name', 'ruling_density', 'blaze_wave', 'R', 'coverage', 'wave_start', 'wave_end', 'wave_offset', 'y_offset']) logger.info('Reading grating information') for line in gmos_gratings: new_grating = GMOSGrating( name=line['name'], ruling_density_value=line['ruling_density'], blaze_wavelength_value=line['blaze_wave'], R=line['R'], coverage_value=line['coverage'], wavelength_start_value=line['wave_start'], wavelength_end_value=line['wave_end'], wavelength_offset_value=line['wave_offset'], y_offset_value=line['y_offset']) self.session.add(new_grating) mirror = GMOSGrating(name='mirror', ruling_density_value=0.0, blaze_wavelength_value=0.0, R=0.0, coverage_value=np.inf, wavelength_start_value=0.0, wavelength_end_value=np.inf, wavelength_offset_value=0.0, y_offset_value=0.0) self.session.add(mirror) self.session.commit()
def _get_data(): filepath = dirname(abspath(__file__)) ##### EDIT THE FOLLOWING TO POINT TO DatasetName.csv ##### with open(filepath + '/statecrime.csv', 'rb') as f: try: data = np.recfromtxt(f, delimiter=",", names=True, dtype=None, encoding='utf-8') except TypeError: data = np.recfromtxt(f, delimiter=",", names=True, dtype=None) return data
def handle_special(self, q, p): if q[0] == '\\': NULL = p.stdout.readline() if len(q.split()) > 1: names = p.stdout.readline().split(',') r = np.recfromtxt(p.stdout, skip_footer=2, delimiter=',', names=names) else: names = p.stdout.readline().split(',') r = np.recfromtxt(p.stdout, skip_footer=1, delimiter=',', names=names) else: names = p.stdout.readline().split(',') r = np.recfromtxt(p.stdout, skip_footer=1, delimiter=',', names=names) return r
def layoutFromTxt(filename): """Read plate layout from text file and return a structured array.""" if not os.path.isfile(filename): msg = "No Plate Layout provided. File not found {}".format(filename) raise IOError(msg) try: rec = np.recfromtxt(filename, dtype=LayoutDtype, skip_header=True) except ValueError: rec = np.recfromtxt(filename, dtype=LayoutDtype, delimiter="\t", skip_header=True) return rec
def load(): """ Loads the Grunfeld data and returns a Dataset class. Returns ------- Dataset instance: See DATASET_PROPOSAL.txt for more information. Notes ----- raw_data has the firm variable expanded to dummy variables for each firm (ie., there is no reference dummy) """ filepath = dirname(abspath(__file__)) data = recfromtxt(open(filepath + '/grunfeld.csv','rb'), delimiter=",", names=True, dtype="f8,f8,f8,a17,f8") names = list(data.dtype.names) endog = array(data[names[0]], dtype=float) endog_name = names[0] exog = data[list(names[1:])] exog_name = list(names[1:]) dataset = Dataset(data=data, names=names, endog=endog, exog=exog, endog_name=endog_name, exog_name=exog_name) raw_data = categorical(data, col='firm', drop=True) dataset.raw_data = raw_data return dataset
def get_model_conditions(self, model): """Returns a description of all conditions for a given model Parameters ---------- model : int Model identifier. Returns ------- list(dict) A list of a model conditions is returned, where each item is a dictionary with keys ``id`` (numerical condition ID), ``task`` (numerical task ID for the task containing this condition), and ``name`` (the literal condition name). This information is returned in a list (instead of a dictionary), because the openfmri specification of model conditions contains no unique condition identifier. Conditions are only uniquely described by the combination of task and condition ID. """ def_fname = _opj(self._basedir, 'models', _model2id(model), 'condition_key.txt') def_data = np.recfromtxt(def_fname) conds = [] # load model meta data for dd in def_data: cond = {} cond['task'] = _id2int(dd[0]) cond['id'] = _id2int(dd[1]) cond['name'] = dd[2] conds.append(cond) return conds
def _get_data(): filepath = dirname(abspath(__file__)) with open(filepath + "/anes96.csv", "rb") as f: data = recfromtxt(f, delimiter="\t", names=True, dtype=float) logpopul = log(data["popul"] + 0.1) data = nprf.append_fields(data, "logpopul", logpopul, usemask=False, asrecarray=True) return data
def __init__(self, fnames=None, params=None): '''params: a list of numpy arrays''' super(MovementParamPlot, self).__init__() if fnames is None: # XXX Hardcode hack to get envisage working fnames = ['data/movement_params.txt'] params = np.recfromtxt(fnames[0], dtype=PARAM_DTYPE) self.file_list = fnames self.file_name = fnames[0] self.params_num = 0 self.max_params = len(params) - 1 self.params = params # XXX - the following info should go (soon) into # nipype.interfaces.fsl.McFLIRT self.trans_plot = self.create_line_plot('trans', 'mm') self.rot_plot = self.create_line_plot('rot', 'radians (clockwise)', self.trans_plot.index_range) # Note - it doesn't matter which plot you use to init the tools self.zoom = tools.ZoomTool(self.trans_plot, tool_mode='range', axis='index') self.pan = tools.PanTool(self.rot_plot, constrain=True, constrain_direction='x') self.trans_plot.tools.extend((self.zoom, self.pan)) self.rot_plot.tools.extend((self.pan, self.zoom))
def read(filename): """ Read a table (.tbl) file. Parameters: * filename Name of table file to read Returns: (comments,rec) * comments List of comments (strings terminated with newline) * rec Records array with named fields. """ # pull out the comment lines from the file (start with #) f = open(filename,'r') comments = [l for l in f if l[0]=='#'] f.close() # find the line beginning with # NAMES and parse out the column names nl = [i for i,l in enumerate(comments) if l[:7]=="# NAMES"] if len(nl)!=1: raise IOError("%s does not have a # NAMES line"%(filename)) dtd = {'names':comments.pop(nl[0])[7:].split()} # find the line beginning with # DTYPE and parse out the column names dl = [i for i,l in enumerate(comments) if l[:9]=="# FORMATS"] if len(dl)!=1: raise IOError("%s does not have a # FORMATS line"%(filename)) dtd['formats'] = comments.pop(dl[0])[9:].split() # return the data as a records array return comments,np.atleast_1d(np.recfromtxt(filename,dtype=dtd))
def __init__(self, pathlike): if isinstance(pathlike, str): paths = glob(pathlike) else: paths = pathlike paths.sort() pfile = self pfile._vars = dict() files = [open(path) for path in paths] datas = [np.recfromtxt(f, names = True, case_sensitive = True) for f in files] data = np.ma.concatenate(datas) desired_unit = dict(O3 = 'ppb', GMAO_TEMP = 'K', PRESS = 'hPa', TEMP = 'K') unit_factor = {'ppt': 1e12, 'ppb': 1e9} pfile.createDimension('time', data.shape[0]) for ki, key in enumerate(data.dtype.names): typecode = data[key].dtype.char if typecode not in ('c', 'S'): unit = desired_unit.get(key, 'ppt') factor = unit_factor.get(unit, 1) values = np.ma.masked_values(data[key], -1000) * factor else: unit = 'unknown' values = data[key] pfile.createVariable(key, typecode, dimensions = ('time',), units = unit, values = values)
def SNrest(): path = "../data/restframe/" objnames, band, mjd, mag, magerr, stype = [],[],[],[],[], [] formatcode = ('|S16,'.rstrip('#') +'f8,'*6 + '|S16,' + 4 * 'f8,' + '|S16,' * 3 + 'f8,' * 2 + '|S16,' + 'f8,' * 2) filenames = os.listdir(path) for filename in filenames: data = np.recfromtxt(os.path.join(path, filename),usecols = (0,1,2,3,4), dtype = formatcode, names = True, skip_header = 13, case_sensitive = 'lower', invalid_raise = False) name = np.empty(len(data.band), dtype = 'S20') name.fill(filename) objnames.append(name) data.band = [x.lower() for x in data.band] band.append(data.band) mjd.append(data.phase) mag.append(data.mag) magerr.append(data.err) objnames = np.fromiter(itertools.chain.from_iterable(objnames), dtype = 'S20') band = np.fromiter(itertools.chain.from_iterable(band), dtype = 'S16') mjd = np.fromiter(itertools.chain.from_iterable(mjd), dtype = 'float') mag = np.fromiter(itertools.chain.from_iterable(mag), dtype = 'float') magerr = np.fromiter(itertools.chain.from_iterable(magerr), dtype = 'float') stype = np.full(len(objnames), 1) LC = Lightcurve(objnames, band, mjd, mag, magerr, stype) return LC
def readin_aavso(filename): formatcode = ('f8,'*4 + '|S16,'*20).rstrip(',') data = np.recfromtxt(filename, delimiter='\t', names=True, dtype=formatcode,autostrip=True,case_sensitive='lower', invalid_raise=False) ind = np.where((data.band == 'V') & (data.uncertainty > 0) & (np.isnan(data.uncertainty) == 0) & (data.uncertainty < 0.02)) banddata = data[ind] return banddata
def get_region_corners(catalog): cat = np.recfromtxt(catalog, names=open(catalog).readline().split()[1:]) c1 = (cat.ra.min(), cat.dec[cat.ra==cat.ra.min()][0]) c2 = (cat.ra[cat.dec==cat.dec.max()][0], cat.dec.max()) c3 = (cat.ra.max(), cat.dec[cat.ra==cat.ra.max()][0]) c4 = (cat.ra[cat.dec==cat.dec.min()][0], cat.dec.min()) return reduce(lambda x,y: x+y, [c1, c2, c3, c4])
def query_region_db(ra_center, dec_center, radius, region_dir): """ Returns a list of files relative to regions which are within the provided cone :param ra_center: R.A. of the center of the cone :param dec_center: Dec. of the center of the cone :param radius: Radius of the cone (in arcmin) :param region_dir: Path of the directory containing the database file and the region files :return: list of region files """ database_file = os.path.join(region_dir, 'region_database.txt') data = np.recfromtxt(database_file, names=True) # Compute the angular distance between all regions and the center of the cone distances = angular_distance(float(ra_center), float(dec_center), data["RA"], data["DEC"], unit='arcmin') # Select all regions within the cone idx = (distances <= float(radius)) # Return the corresponding region files data_list = [] for i in data['REGION_FILE'][idx]: data_list.append(os.path.join(region_dir, i)) return data_list
def _initialize_gmos_filters(self, configuration_dir): """ Read in GMOS filter/grating information, for matching to headers. """ logger.info('Reading Filter information') gmos_filters = np.recfromtxt( os.path.join(configuration_dir, 'GMOSfilters.dat'), names=['name', 'wave_start', 'wave_end', 'fname']) for line in gmos_filters: new_filter = GMOSFilter(name=line['name'], wavelength_start_value=line['wave_start'], wavelength_start_unit='nm', wavelength_end_value=line['wave_end'], wavelength_end_unit='nm', fname=line['fname'], path=os.path.join('gmos', 'filter_data')) self.session.add(new_filter) open_filter = GMOSFilter(name='open', wavelength_start_value=0, wavelength_start_unit='nm', wavelength_end_value=np.inf, wavelength_end_unit='nm', fname=None, path=None) self.session.add(open_filter)
def __init__(self, path, coordkeys = "time time_bounds TFLAG ETFLAG latitude latitude_bounds longitude longitude_bounds lat lat_bnds lon lon_bnds etam_pressure etai_pressure layer_bounds layer47 layer".split(), delimiter = ',', names = True, **kwds): """ path - place to find csv file coordkeys - use these keys as dimensions and coordinate variables delimiter - use this as delimiter (default = ',') names - see help in recfromtxt (Default = True) kwds - np.recfromtxt keywords * Note: currently only works when all coordinate variables are 1-d """ kwds['names'] = names kwds['delimiter'] = delimiter data = np.recfromtxt(path, **kwds) dimkeys = [dk for dk in data.dtype.names if dk in coordkeys] varkeys = [vk for vk in data.dtype.names if not vk in coordkeys] for dk in dimkeys: dv = np.unique(data[dk]) dv.sort() self.createDimension(dk, len(dv)) dvar = self.createVariable(dk, dv.dtype.char, (dk,)) dvar[:] = dv for vk in varkeys: vv = data[vk] var = self.createVariable(vk, vv.dtype.char, tuple(dimkeys)) for idx in np.ndindex(var.shape): thisidx = np.sum([data[dk] == self.variables[dk][di] for di, dk in zip(idx, dimkeys)], axis = 0) == len(dimkeys) if thisidx.any(): var[idx] = vv[thisidx]
def getList(file): posList= np.recfromtxt(file) l = [posList['f0'],posList['f2'],posList['f3']] l = np.array(l) l = l.T names = posList['f4'] return l,names
def sigma_clip_non_hdr(filepath): """ Eliminates sources with SNR less than the 'sigma_clip' parameter from setup file. Also checks for negative flux sources which may remain after this step (can happen when uncertainty values are also negative). """ work_dir = "/".join(filepath.split("/")[:-1]) meta = json.load(open(work_dir + "/metadata.json")) names = open(filepath).readline().split()[1:] data = np.recfromtxt(filepath, names=names) # get rid of low SNR sources snr = data["flux"] / data["unc"] good = snr >= meta["sigma_clip"] data = data[good] # get rid of any remaining negative flux sources good = data["flux"] > 0 data = data[good] # get rid of id column data = data[["ra", "dec", "flux", "unc", "n_obs"]] # write to disk fmt = ["%0.8f"] * 2 + ["%.4e"] * 2 + ["%i"] out_path = filepath.replace(".txt", "_sigclip.txt") header = " ".join(names[1:]) np.savetxt(out_path, data, fmt=fmt, header=header, comments="") print("created file: " + out_path)
def load(): """ Load the star98 data and returns a Dataset class instance. Returns ------- Load instance: a class of the data with array attrbutes 'endog' and 'exog' """ filepath = dirname(abspath(__file__)) ##### EDIT THE FOLLOWING TO POINT TO DatasetName.csv ##### names = ["NABOVE","NBELOW","LOWINC","PERASIAN","PERBLACK","PERHISP", "PERMINTE","AVYRSEXP","AVSALK","PERSPENK","PTRATIO","PCTAF", "PCTCHRT","PCTYRRND","PERMINTE_AVYRSEXP","PERMINTE_AVSAL", "AVYRSEXP_AVSAL","PERSPEN_PTRATIO","PERSPEN_PCTAF","PTRATIO_PCTAF", "PERMINTE_AVYRSEXP_AVSAL","PERSPEN_PTRATIO_PCTAF"] data = recfromtxt(open(filepath + '/star98.csv',"rb"), delimiter=",", names=names, skip_header=1, dtype=float) names = list(data.dtype.names) # endog = (successes, failures) NABOVE = array(data[names[1]]).astype(float) # successes NBELOW = array(data[names[0]]).astype(float) \ - array(data[names[1]]).astype(float) # now its failures endog = column_stack((NABOVE,NBELOW)) endog_name = names[:2] exog = column_stack(data[i] for i in names[2:]).astype(float) exog_name = names[2:] dataset = Dataset(data=data, names=names, endog=endog, exog=exog, endog_name = endog_name, exog_name=exog_name) return dataset
def movie_dataset(subj, task, label, **kwargs): # ds = movie_dataset( # 2, # 'avmovie', # 'bold3Tp2', # mask='src/tnt/sub-02/bold3Tp2/brain_mask.nii.gz') cur_max_time = 0 segments = [] if not 'add_sa' in kwargs: add_sa = {} for seg in range(1, 9): print 'Seg', seg mc = np.recfromtxt( 'sub-%.2i/in_%s/sub-%.2i_task-%s_run-%i_bold_mcparams.txt' % (subj, label, subj, task, seg), names=('mc_xtrans', 'mc_ytrans', 'mc_ztrans', 'mc_xrot', 'mc_yrot', 'mc_zrot')) for i in mc.dtype.fields: add_sa[i] = mc[i] ds = preprocessed_fmri_dataset( 'sub-%.2i/in_%s/sub-%.2i_task-%s_run-%i_bold.nii.gz' % (subj, label, subj, task, seg), add_sa=add_sa, **kwargs) ds.sa['movie_segment'] = [seg] * len(ds) TR = np.diff(ds.sa.time_coords).mean() ## truncate segment time series to remove overlap if seg > 1: ds = ds[4:] if seg < 8: ds = ds[:-4] ds.sa['movie_time'] = np.arange(len(ds)) * TR + cur_max_time cur_max_time = ds.sa.movie_time[-1] + TR segments.append(ds) return vstack(segments)
def _initialize_gmos_gratings(self, configuration_dir): gmos_gratings = np.recfromtxt( os.path.join(configuration_dir, 'GMOSgratings.dat'), names = ['name', 'ruling_density', 'blaze_wave', 'R', 'coverage', 'wave_start', 'wave_end', 'wave_offset', 'y_offset']) logger.info('Reading grating information') for line in gmos_gratings: new_grating = GMOSGrating( name=line['name'], ruling_density_value=line['ruling_density'], blaze_wavelength_value=line['blaze_wave'], R=line['R'], coverage_value=line['coverage'], wavelength_start_value=line['wave_start'], wavelength_end_value=line['wave_end'], wavelength_offset_value=line['wave_offset'], y_offset_value=line['y_offset']) self.session.add(new_grating) mirror = GMOSGrating(name='mirror', ruling_density_value=0.0, blaze_wavelength_value=0.0, R=0.0, coverage_value=np.inf, wavelength_start_value=0.0, wavelength_end_value=np.inf, wavelength_offset_value=0.0, y_offset_value=0.0) self.session.add(mirror) self.session.commit()
def _get_data(): filepath = dirname(abspath(__file__)) ##### EDIT THE FOLLOWING TO POINT TO DatasetName.csv ##### with open(filepath + '/spector.csv',"rb") as f: data = np.recfromtxt(f, delimiter=" ", names=True, dtype=float, usecols=(1,2,3,4)) return data
def _get_data(): filepath = dirname(abspath(__file__)) ##### EDIT THE FOLLOWING TO POINT TO DatasetName.csv ##### with open(filepath + '/engel.csv', 'rb') as f: data = np.recfromtxt(f, delimiter=",", names = True, dtype=float) return data
def readFrameList(self, parFidx, parWDir): self.pos=0 self.fcsvIdx=None self.wdir=None self.listFImg=[] if os.path.isfile(parFidx): tmpList=np.recfromtxt(parFidx) isAllOk=True if len(tmpList)>0: for ii in tmpList: tmpFimg=ii if parWDir!=None: tmpFimg='%s/%s' % (parWDir, ii) if os.path.isfile(tmpFimg): self.listFImg.append(tmpFimg) else: isAllOk=False if self.isDebug: print "Can't find file [%s]" % tmpFimg break else: isAllOk=False if isAllOk: self.fcsvIdx=parFidx self.wdir=parWDir else: self.listFImg=[]
def create_histogram(parameter_name, nbins=100, writeFile=True, skipfirst=0, truncate=False, smooth=False): """ Returns a histogram and some statistics about this parameter. @param writeFile: if true, write the histogram to paramname.histogram """ f = "%s-chain-0.prob.dump" % parameter_name values = numpy.recfromtxt(f)[skipfirst::nevery] statistics = { 'min': float(values.min()), 'max': float(values.max()), 'stdev': float(values.std()), 'mean': float(values.mean()), 'median':float(numpy.median(values)), 'q1': float(scipy.stats.scoreatpercentile(values, 25)), 'q3': float(scipy.stats.scoreatpercentile(values, 75)), 'p5': float(scipy.stats.scoreatpercentile(values, 5)), 'p95': float(scipy.stats.scoreatpercentile(values, 95)), } hist = scipy.histogram(values, bins = nbins if not smooth else nbins*10, normed=True) histwithborders = numpy.dstack([hist[1][0:nbins], hist[1][1:nbins+1], hist[0]]) if writeFile: scipy.savetxt('%s.histogram' % parameter_name, histwithborders[0], delimiter="\t") return histwithborders[0], statistics
def read_csv(filename, column_headers, column_types): # use csv module to parse csv (understands quotes etc...) and use \t as # seperator to help numpy parsing the csv reader = csv.reader(open(filename)) if column_headers: names = reader.next() return np.recfromtxt(("\t".join(row) for row in reader), delimiter="\t", names=names, dtype=column_types)
def test_recfromtxt(self): # data = StringIO("A,B\n0,1\n2,3") kwargs = dict(delimiter=",", missing_values="N/A", names=True) test = np.recfromtxt(data, **kwargs) control = np.array([(0, 1), (2, 3)], dtype=[("A", np.int), ("B", np.int)]) self.assertTrue(isinstance(test, np.recarray)) assert_equal(test, control) # data = StringIO("A,B\n0,1\n2,N/A") test = np.recfromtxt(data, dtype=None, usemask=True, **kwargs) control = ma.array( [(0, 1), (2, -1)], mask=[(False, False), (False, True)], dtype=[("A", np.int), ("B", np.int)] ) assert_equal(test, control) assert_equal(test.mask, control.mask) assert_equal(test.A, [0, 2])
def _get_data(): filepath = dirname(abspath(__file__)) data = recfromtxt(open(filepath + '/anes96.csv',"rb"), delimiter="\t", names = True, dtype=float) logpopul = log(data['popul'] + .1) data = nprf.append_fields(data, 'logpopul', logpopul, usemask=False, asrecarray=True) return data
fig = plt.figure() ax = fig.add_subplot(111, projection='3d') ax.plot(x,y,z,color = 'purple') ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Z') plt.legend([r'$x_0 = {0}$ $y_0 = {1}$ $z_0 = {2}$'.format(x[0],y[0],z[0])],bbox_to_anchor=(0.78, 1.13)) ax.set_title(r'$\sigma = 10$ $\beta = 8/3$ $\rho = 28$') plt.savefig('2_a.png') plt.show() ''' lorenz_b = np.recfromtxt('lorenz_b.txt', names=True) x_b = lorenz_b['x'] y_b = lorenz_b['y'] z_b = lorenz_b['z'] fig = plt.figure() ax = fig.add_subplot(111, projection='3d') ax.plot(x_b, y_b, z_b, color='green') ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Z') plt.legend( [r'$x_0 = {0}$ $y_0 = {1}$ $z_0 = {2}$'.format(x_b[0], y_b[0], z_b[0])], bbox_to_anchor=(0.75, 1.13)) ax.set_title(r'$\sigma = 10$ $\beta = 8/3$ $\rho = 28$')
def runmain(argv=1): if argv == 1: argv = sys.argv usage = 'usage: %prog [options]\n' cmdlineParser = OptionParser(usage=usage) cmdlineParser.add_option( "--dataFilePattern", dest="pattern", default='../data/*.csv', help='Name of the datafile containing returns data (default: %default)' ) cmdlineParser.add_option( "--randomData", dest="random", default=False, help='Use randomly genrated data (default: %default)') (cmdoptions, args) = cmdlineParser.parse_args(argv) random = cmdoptions.random outfile = open('returnsPA.txt', 'w') files = glob.glob(cmdoptions.pattern) sectorNames = [] cumPReturns = 1 cumBReturns = 1 for f in files: linkedAttr if len(sectorNames) == 0: data = np.recfromtxt(f, delimiter=',', names=True) names = data.dtype.names sectors = {} sectorNames = set(data['Sector']) sectorNames = list(sectorNames) linkedAttr = np.zeros((len(sectorNames), 4)) outfile.write(f + '\n') effectNames, attr, overallPortfolioReturns, overallBenchmarkReturns = runPA( f, random, sectorNames, outfile) periodCoeff = (np.log(1 + overallPortfolioReturns) - np.log(1 + overallBenchmarkReturns)) / ( overallPortfolioReturns - overallBenchmarkReturns) linkedAttr += attr * periodCoeff cumPReturns = cumPReturns * (1 + overallPortfolioReturns) cumBReturns = cumBReturns * (1 + overallBenchmarkReturns) overallCoeff = (np.log(cumPReturns) - np.log(cumBReturns)) / (cumPReturns - cumBReturns) linkedAttr = linkedAttr / overallCoeff header = 'Sector' for effect in effectNames: header = header + ',' + effect outfile.write(header + '\n') for sector in sectorNames: dataStr = sector index = sectorNames.index(sector) data = linkedAttr[sectorNames.index(sector)] count = 0 while count < len(data): dataStr = str(dataStr) + ',' + str(data[count]) count = count + 1 outfile.write(dataStr + '\n') cumActiveReturns = cumPReturns - cumBReturns linkedSumofEffects = linkedAttr.sum() if (abs(cumActiveReturns - linkedSumofEffects) < 0.0001): print(True) else: print(False) outfile.write('overall cumulative active returns,' + str(cumActiveReturns) + '\n') outfile.write('sum of linked effects,' + str(linkedSumofEffects) + '\n') outfile.close()
def runPA(filename, random, sectorNames, outfile): data = np.recfromtxt(filename, delimiter=',', names=True) #数据 names = data.dtype.names # sectors = {} for name in sectorNames: sectors[name] = {} numRecord = len(data) count = 0 portfolioWeights = {} benchmarkWeights = {} portfolioReturns = {} benchmarkReturns = {} interactionEffects = {} selectionEffects = {} styleAllocationEffects = {} sectorAllocationEffects = {} overallPortfolioReturns = 0 overallBenchmarkReturns = 0 sumPW = 0 sumBW = 0 while count < numRecord: record = data[count] count = count + 1 sectorName = record[names.index('Sector')] styleName = sectorName + b'.' + record[names.index('Style')] assetName = record[names.index('Asset')] if random == True: assetReturns = (2 * np.random.rand() - 1) / 10 if count == numRecord: assetPW = 1 - sumPW assetBW = 1 - sumBW else: assetPW = np.random.rand() / 15 assetBW = np.random.rand() / 15 sumPW += assetPW sumBW += assetBW else: assetReturns = record[names.index('Asset_Returns')] assetPW = record[names.index('Portfolio_Weight')] assetBW = record[names.index('Benchmark_Weight')] assetPReturns = assetPW * assetReturns assetBReturns = assetBW * assetReturns overallPortfolioReturns += assetPReturns overallBenchmarkReturns += assetBReturns sectorDict = sectors[sectorName] try: styleDict = sectorDict[styleName] except KeyError: styleDict = {} sectorDict[styleName] = styleDict styleDict[assetName] = assetReturns addToDict(portfolioWeights, styleName, assetPW) addToDict(portfolioWeights, sectorName, assetPW) addToDict(portfolioReturns, styleName, assetPReturns) addToDict(portfolioReturns, sectorName, assetPReturns) addToDict(benchmarkWeights, styleName, assetBW) addToDict(benchmarkWeights, sectorName, assetBW) addToDict(benchmarkReturns, styleName, assetBReturns) addToDict(benchmarkReturns, sectorName, assetBReturns) portfolioWeights[assetName] = assetPW benchmarkWeights[assetName] = assetBW portfolioReturns[assetName] = assetReturns benchmarkReturns[assetName] = assetReturns for sector, styleDict in sectors.items(): sectorPW = portfolioWeights.get(sector) sectorBW = benchmarkWeights.get(sector) sectorPR = portfolioReturns.get(sector) sectorBR = benchmarkReturns.get(sector) sectorIntact = 0 sectorSelec = 0 sectorStyleAlloc = 0 sectorAlloc = (sectorPW - sectorBW) * (sectorBR / sectorBW - overallBenchmarkReturns) for style, styDict in styleDict.items(): stylePW = portfolioWeights.get(style) styleBW = benchmarkWeights.get(style) stylePR = portfolioReturns.get(style) styleBR = benchmarkReturns.get(style) styleIntact = 0 styleSelec = 0 styleAlloc = sectorPW * ( stylePW / sectorPW - styleBW / sectorBW) * (styleBR / styleBW - sectorBR / sectorBW) styleAllocationEffects[style] = styleAlloc sectorStyleAlloc += styleAlloc for asset in styDict.keys(): assetPW = portfolioWeights.get(asset) assetBW = benchmarkWeights.get(asset) assetPR = portfolioReturns.get(asset) assetBR = benchmarkReturns.get(asset) selec = styleBW * (assetPW / stylePW - assetBW / styleBW) * ( assetPR - styleBR / styleBW) intact = (assetPW - assetBW) * (stylePR / stylePW - styleBR / styleBW) interactionEffects[asset] = selec selectionEffects[asset] = intact styleIntact += intact styleSelec += selec interactionEffects[style] = styleIntact selectionEffects[style] = styleSelec sectorIntact += styleIntact sectorSelec += styleSelec interactionEffects[sector] = sectorIntact selectionEffects[sector] = sectorSelec styleAllocationEffects[sector] = sectorStyleAlloc sectorAllocationEffects[sector] = sectorAlloc overallActiveReturns = overallPortfolioReturns - overallBenchmarkReturns checkSum(sectors, sectorAllocationEffects, styleAllocationEffects, selectionEffects, interactionEffects, overallActiveReturns, outfile) effectNames = [ 'sectorAllocationEffects', 'styleAllocationEffects', 'selectionEffects', 'interactionEffects' ] attr = np.zeros((len(sectorNames), 4)) for sector in sectorNames: j = 0 for data in (sectorAllocationEffects, styleAllocationEffects, selectionEffects, interactionEffects): attr[sectorNames.index(sector), j] = data[sector] j = j + 1 outfile.write('overallPortfolioReturns' + ',' + str(overallPortfolioReturns) + '\n') outfile.write('overallBenchmarkReturns' + ',' + str(overallBenchmarkReturns) + '\n') return effectNames, attr, overallPortfolioReturns, overallBenchmarkReturns
import pathlib from typing import IO import numpy.typing as npt import numpy as np str_path: str bytes_path: bytes pathlib_path: pathlib.Path str_file: IO[str] AR_i8: npt.NDArray[np.int64] np.load(str_file) # E: incompatible type np.save(bytes_path, AR_i8) # E: incompatible type np.save(str_file, AR_i8) # E: incompatible type np.savez(bytes_path, AR_i8) # E: incompatible type np.savez(str_file, AR_i8) # E: incompatible type np.savez_compressed(bytes_path, AR_i8) # E: incompatible type np.savez_compressed(str_file, AR_i8) # E: incompatible type np.loadtxt(bytes_path) # E: incompatible type np.fromregex(bytes_path, ".", np.int64) # E: No overload variant np.recfromtxt(bytes_path) # E: incompatible type np.recfromcsv(bytes_path) # E: incompatible type
def _get_data(): filepath = dirname(abspath(__file__)) ##### EDIT THE FOLLOWING TO POINT TO DatasetName.csv ##### with open(filepath + '/cancer.csv', 'rb') as f: data = np.recfromtxt(f, delimiter=",", names=True, dtype=float) return data
def _get_data(): filepath = dirname(abspath(__file__)) with open(filepath + '/stackloss.csv', "rb") as f: data = recfromtxt(f, delimiter=",", names=True, dtype=float) return data
logger = logging.getLogger(__name__) tardis_dir = os.path.realpath(tardis.__path__[0]) def get_data_path(fname): return os.path.join(tardis_dir, 'data', fname) def get_tests_data_path(fname): return os.path.join(tardis_dir, 'tests', 'data', fname) atomic_symbols_data = np.recfromtxt(get_data_path('atomic_symbols.dat'), names=['atomic_number', 'symbol']) symbol2atomic_number = OrderedDict( zip(atomic_symbols_data['symbol'], atomic_symbols_data['atomic_number'])) atomic_number2symbol = OrderedDict(atomic_symbols_data) synpp_default_yaml_fname = get_data_path('synpp_default.yaml') def int_to_roman(int_input): """ from http://code.activestate.com/recipes/81611-roman-numerals/ Convert an integer to Roman numerals. :param int_input: an integer between 1 and 3999 :returns result: roman equivalent string of passed :param{int_input}
def _get_data(): filepath = dirname(abspath(__file__)) data = np.recfromtxt(open(filepath + '/modechoice.csv', 'rb'), delimiter=";", names = True, dtype=float) return data
import numpy as np import pickle,gzip import numpy as np, pickle, gzip, sys, glob, time, os, scipy as sc sys.path.append('modules') import modules import sky_local from pylab import * sims = modules.scl_cmb.simulations() ip_folder = sys.argv[1] fnamearr = glob.glob('%s/stacked*_03500_clusters*'%(ip_folder)) paramfile = glob.glob('%s/*used.txt'%(ip_folder))[0] kappa_file = glob.glob('%s/kappa_COV*100cluters.pkl.gz'%(ip_folder))[0] kappa_COV = pickle.load(gzip.open(kappa_file)) params = np.recfromtxt(paramfile,usecols=[0],delimiter = '=') paramvals = np.recfromtxt(paramfile,usecols=[1],delimiter = '=') param_dict = {} cc = 3.0 for p,pval in zip(params,paramvals): tmp = pval.strip() try: float(tmp) if tmp.find('.')>-1: param_dict[p.strip()] = float(tmp) else: param_dict[p.strip()] = int(tmp) except: if tmp == 'None': param_dict[p.strip()] = None
def csv2rec(filename): return np.recfromtxt(filename, dtype=None, delimiter=',', names=True, encoding='utf-8')
def _get_data(): filepath = dirname(abspath(__file__)) data = recfromtxt(open(PATH, "rb"), delimiter=",", names=True, dtype=float) return data
import numpy as np import matplotlib.pyplot as plt from scipy.fftpack import fft, fftfreq from scipy import interpolate import matplotlib.gridspec as gridspec data = "rev26_27.dat" recdat = np.recfromtxt(data, names=["time", "r", "theta", "phi"]) r = recdat.r t = recdat.time not_nan = np.logical_not(np.isnan(r)) i = np.arange(len(r)) rinterp = np.interp(i, i[not_nan], r[not_nan]) period = 1 / fftfreq(34504, (t[-1] - t[0]) / 34504) fftamp = fft(rinterp) sort = np.argsort(fftamp) fftamp = fftamp[sort] period = period[sort] gs = gridspec.GridSpec(8, 1) plt.figure(figsize=[12, 12]) ax1 = plt.subplot(gs[:2, :]) ax1.plot(t, rinterp, "r") ax1.plot(t, r, "b") ax1.set_xlim([934.9, 959]) ax1.set_xlabel("time since 2004(days)")
def p(m): fig = ml.figure() brain = surfer.Brain('mni', 'rh', 'cortex', curv=False, figure=fig) brain.add_data(m[:, v]) fname1 = env.data + '/structural/thalamusR_SA_NV_10to21_2closestTo16.csv' fname2 = env.data + '/structural/thalamusR_SA_NV_10to21.csv' cortex = np.genfromtxt(fname1, delimiter=',') # removing first column and first row, because they're headers cortex = scipy.delete(cortex, 0, 1) cortex = scipy.delete(cortex, 0, 0) # format it to be subjects x variables cortex = cortex.T c_names = np.recfromtxt(fname1, delimiter=',')[0][1:] cortex2 = np.genfromtxt(fname2, delimiter=',') # removing first column and first row, because they're headers cortex2 = scipy.delete(cortex2, 0, 1) cortex2 = scipy.delete(cortex2, 0, 0) # format it to be subjects x variables cortex2 = cortex2.T c2_names = np.recfromtxt(fname2, delimiter=',')[0][1:] # we have a different number of subjects in the two files, so let's crop the extra subjects in the bigger file to fit the smaller one nvoxels = cortex.shape[1] # we'll assume that cortex has more subjects than cortex2 if cortex.shape[0] < cortex2.shape[0]: tmp = cortex.copy()
#! /usr/bin/env python # fit a collection to T1 trajectories to a decaying exponential import scipy.optimize import numpy as np import pickle # read in the trajectories and times trajs = np.load("traj.npy") t1 = np.recfromtxt("time.dat") # fitting function and residual calculation def fit_func(p, x): A, R2 = p # bound A between 0.98 and 1.02 (although fits do not reflect this) if A > 1.02: A = 1.02 if A < 0.98: A = 0.98 return A * np.exp(-1.0 * np.array(x) * R2 / 1.0e6) def residuals(p, y, x): err = y - fit_func(p, x) return err p0 = [1.0, 0.05] # initial guess
############################################################################# # Retrieve the files of the Haxby dataset from nilearn import datasets haxby_dataset = datasets.fetch_haxby_simple() # print basic information on the dataset print('Mask nifti image (3D) is located at: %s' % haxby_dataset.mask) print('Functional nifti image (4D) is located at: %s' % haxby_dataset.func[0]) ############################################################################# # Load the behavioral data import numpy as np y, session = np.loadtxt(haxby_dataset.session_target[0]).astype("int").T conditions = np.recfromtxt(haxby_dataset.conditions_target[0])['f0'] # Restrict to faces and houses condition_mask = np.logical_or(conditions == b'face', conditions == b'house') y = y[condition_mask] conditions = conditions[condition_mask] # We have 2 conditions n_conditions = np.size(np.unique(y)) ############################################################################# # Prepare the fMRI data from nilearn.input_data import NiftiMasker mask_filename = haxby_dataset.mask # For decoding, standardizing is often very important
def _get_data(): filepath = dirname(abspath(__file__)) with open(filepath+'/longley.csv',"rb") as f: data = recfromtxt(f, delimiter=",", names=True, dtype=float, usecols=(1,2,3,4,5,6,7)) return data
def _read_from_text(self, infile): data = np.recfromtxt(infile) data = data.transpose() self.flux = data[0] self.Ne = dict([(amp, ne) for amp, ne in zip(imutils.allAmps(), data[1:])])
#! /usr/bin/env python # Example scipt to show integration of a 1D spectrum import nmrglue as ng import numpy as np import matplotlib.pyplot as plt # read in the data from a NMRPipe file dic, data = ng.pipe.read("1d_data.ft") length = data.shape[0] # read in the integration limits peak_list = np.recfromtxt("limits.in") # determind the ppm scale uc = ng.pipe.make_uc(dic, data) ppm_scale = uc.ppm_scale() # plot the spectrum fig = plt.figure() ax = fig.add_subplot(111) ax.plot(ppm_scale, data, 'k-') # prepare the output file f = open("area.out", 'w') f.write("#Name\tStart\tStop\tArea\n") # loop over the integration limits for name, start, end in peak_list: min = uc(start, "ppm") max = uc(end, "ppm")
#from transit import Central, System, Body if not on_rtd: import astropy.constants as const AU = const.au.cgs.value RSUN = const.R_sun.cgs.value MSUN = const.M_sun.cgs.value REARTH = const.R_earth.cgs.value MEARTH = const.M_earth.cgs.value G = const.G.cgs.value DAY = 86400. DATAFOLDER = os.path.abspath( os.path.join(os.path.dirname(__file__), 'data')) LDDATA = np.recfromtxt('{}/keplerld.dat'.format(DATAFOLDER), names=True) LDOK = ((LDDATA.teff < 10000) & (LDDATA.logg > 2.0) & (LDDATA.feh > -2)) LDPOINTS = np.array([LDDATA.teff[LDOK], LDDATA.logg[LDOK]]).T U1FN = interpnd(LDPOINTS, LDDATA.u1[LDOK]) U2FN = interpnd(LDPOINTS, LDDATA.u2[LDOK]) else: const, AU, RSUN, MSUN = (None, None, None, None) REARTH, MEARTH, DAY = (None, None, None) DATAFOLDER = None LDDATA, LDOK, LDPOINTS, U1FN, U2FN = (None, None, None, None, None) MAXSLOPE = 30 def ldcoeffs(teff, logg=4.5, feh=0): """
from scipy import stats bands = [[1, 4], [4, 8], [8, 13], [13, 30], [30, 50]] sx_fname = '/Users/sudregp/data/meg/inatt.txt' subjs_fname = '/Users/sudregp/data/meg/usable_subjects_5segs13p654.txt' data_dir = '/Users/sudregp/data/meg/connectivity/' lmethod = 'pca_flip' cmethod = 0 selected_labels = [] # selected_labels = ['isthmuscingulate-rh', 'superiorfrontal-rh', 'inferiorparietal-rh', 'isthmuscingulate-lh', 'superiorfrontal-lh', 'inferiorparietal-lh'] # selected_labels = ['isthmuscingulate-rh', 'superiorfrontal-rh', 'inferiorparietal-rh'] fid = open(subjs_fname, 'r') subjs = [line.rstrip() for line in fid] res = np.recfromtxt(sx_fname, delimiter='\t') sx = {} for rec in res: sx[rec[0]] = rec[1] print 'sx =',sx_fname m = ['pli','imcoh','plv','wpli','pli2_unbiased','wpli2_debiased'] print lmethod, '-', m[cmethod] labels, label_colors = mne.labels_from_parc(subjs[0], parc='aparc') nlabels=len(labels) il = np.tril_indices(nlabels, k=-1) if len(selected_labels)>0: label_names = [l.name for l in labels] idx = [l for s in selected_labels for l, label in enumerate(label_names) if label == s] keep = [False]*len(il[0])
parser = argparse.ArgumentParser(description="Take output from Bayesian block algorithm and cross match with " "previously flagged variable sources") parser.add_argument("--bbfile", help="Path of input text file", required=True) parser.add_argument("--tsvfile", help="Path of tsv file", required=True) parser.add_argument("--outfile", help="Path of out file", required=True) args = parser.parse_args() # get directory path and file name from input file arguments bb_file_path = os.path.abspath(os.path.expandvars(os.path.expanduser(args.bbfile))) tsv_file_path = os.path.abspath(os.path.expandvars(os.path.expanduser(args.tsvfile))) # read BB data into array bb_data = np.array(np.recfromtxt(bb_file_path, names=True), ndmin=1) # number of rows of data bb_n = len(bb_data) # read tsv data into array tsv_data = np.recfromtxt(tsv_file_path, delimiter='\t', skip_header=11, names=True) # Filter out all non-variable sources variability = np.array(map(lambda x: x.replace(" ", "") == "TRUE", tsv_data['var_flag'])) idx = (variability == True) variable_sources = tsv_data[idx]
def _get_data(): filepath = dirname(abspath(__file__)) with open(filepath + '/macrodata.csv', 'rb') as f: data = recfromtxt(f, delimiter=",", names=True, dtype=float) return data
mydate = dt.datetime.strptime(data[i][date_idx], '%m/%d/%y') data[i][date_idx] = dt.datetime.strftime( mydate, '%m/%d/%Y') except: try: dt.datetime.strptime(data[i][date_idx], '%m/%d/%Y') except: print data[i][date_idx], 'is not a valid date!',\ ' (row %d)'%(i+2) remove_rows.append(i) data = np.delete(data, remove_rows, axis=0) return data data1 = np.recfromtxt(fname1, delimiter='\t') data2 = np.recfromtxt(fname2, delimiter='\t') hdr1 = data1[0] hdr2 = data2[0] # let's do a series of quick checks to clean up the data data1 = clean_up(data1[1:], mrn1, date1) data2 = clean_up(data2[1:], mrn2, date2) merged_data = [] # do this for rows in data1 for row in range(len(data1)): # store row indexes in data2 that match the current mrn matching_rows = [ i for i in range(len(data2)) if int(data1[row][mrn1]) == int(data2[i][mrn2])
'USA', 'France', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'Germany', 'Japan', 'USA', 'USA', 'USA', 'USA', 'Germany', 'Japan', 'Japan', 'USA', 'Sweden', 'USA', 'France', 'Japan', 'Germany', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'USA', 'Germany', 'Japan', 'Japan', 'USA', 'USA', 'Japan', 'Japan', 'Japan', 'Japan', 'Japan', 'Japan', 'USA', 'USA', 'USA', 'USA', 'Japan', 'USA', 'USA', 'USA', 'Germany', 'USA', 'USA', 'USA' ]) #accommodate recfromtxt for python 3.2, requires bytes ss = asbytes(ss) ss2 = asbytes(ss2) ss3 = asbytes(ss3) ss5 = asbytes(ss5) dta = np.recfromtxt(BytesIO(ss), names=("Rust", "Brand", "Replication")) dta2 = np.recfromtxt(BytesIO(ss2), names=("idx", "Treatment", "StressReduction")) dta3 = np.recfromtxt(BytesIO(ss3), names=("Brand", "Relief")) dta5 = np.recfromtxt(BytesIO(ss5), names=('pair', 'mean', 'lower', 'upper', 'sig'), delimiter='\t') sas_ = dta5[[1, 3, 2]] from statsmodels.stats.multicomp import (tukeyhsd, pairwise_tukeyhsd, MultiComparison) #import statsmodels.sandbox.stats.multicomp as multi #print tukeyhsd(dta['Brand'], dta['Rust']) def get_thsd(mci, alpha=0.05):
data_dir = home + '/data/results/meg_Yeo/seeds/%s/' % sys.argv[1] my_test = sys.argv[2] # data_dir = home + '/data/results/meg_Yeo/seeds/net1_lMidOccipital/' # mytest = 'inatt' thresh = .95 nperms = 5000 dist = 5 # distance between voxels to be considered connected (mm) alphas = [.05, .01] fid1 = open(g1_fname, 'r') fid2 = open(g2_fname, 'r') fid3 = open(g3_fname, 'r') g1 = [line.rstrip() for line in fid1] g2 = [line.rstrip() for line in fid2] g3 = [line.rstrip() for line in fid3] res = np.recfromtxt(inatt_fname, delimiter='\t') inatt = {} for rec in res: inatt[rec[0]] = rec[1] res = np.recfromtxt(hi_fname, delimiter='\t') hi = {} for rec in res: hi[rec[0]] = rec[1] print 'g1 =', g1_fname print 'g2 =', g2_fname print 'g3 =', g3_fname vox_pos = np.genfromtxt(home + '/data/meg/sam_narrow_5mm/brainTargetsInTLR.txt', skip_header=1)
#read in the flat data print('Reading in flat data') flatData = np.zeros([2, 4, 256, 256], dtype=np.double) #delay data #just doing ch1/2 for warm mission right now, leaving in option for others if/when those corrections become available for cryo in range(0, 1): for Ch in range(1, 3): flatHDU = fits.open(flatFiles[cryo, Ch - 1]) flatData[cryo, Ch - 1] = flatHDU[0].data #read in the delay files print('Reading in Frame Delay Data') #Read the delay files for the first frame correction delayInfo = np.recfromtxt( FrameDelayFile, dtype=[ ('frame', np.int), #Number ('delay', np.float32) #delay since last frame ]) #create a lookup index for the frame delay correction delay_to_index = interp1d(delayInfo.delay, delayInfo.frame, bounds_error=False, fill_value='extrapolate') #Read the fits files delayData = np.zeros([2, len(delayInfo), 256, 256], dtype=np.double) #delay data for i in range(0, len(delayInfo)): for Ch in range(1, 3): delayFile = './cal/labdark.' + str(i) + '.' + str(Ch) + '.fits'
data_dir = expanduser('~') + '/downloads/pymvpa-exampledata/' # create sklearn's Bunch of data dataset_files = Bunch( func=data_dir+'bold.nii.gz', session_target=data_dir+'attributes.txt', mask=data_dir+'mask.nii.gz', conditions_target=data_dir+'attributes_literal.txt' ) # fmri_data and mask are copied to break any reference to the original object bold_img = nibabel.load(dataset_files.func) fmri_data = bold_img.get_data().astype(float) affine = bold_img.get_affine() y, session = np.loadtxt(dataset_files.session_target).astype("int").T conditions = np.recfromtxt(dataset_files.conditions_target)['f0'] mask = dataset_files.mask # fmri_data.shape is (40, 64, 64, 1452) # and mask.shape is (40, 64, 64) # ### Preprocess data # Build the mean image because we have no anatomic data mean_img = fmri_data.mean(axis=-1) # ### Restrict to faces and houses condition_mask = np.logical_or(conditions == 'face', conditions == 'house') X = fmri_data[..., condition_mask] y = y[condition_mask] # session = session[condition_mask]
def prepare_fields(self, infile=None, outfile=None, mode='smash_dither', plot=True, smcnod=False): """ Create the list of fields to be targeted by this survey. Parameters: ----------- infile : File containing all possible field locations. outfile: Output file of selected fields mode : Mode for dithering: 'smash_dither', 'smash_rotate', 'decam_dither', 'none' plot : Create an output plot of selected fields. Returns: -------- fields : A FieldArray of the selected fields. """ # Import the dither function here... #def dither(ra,dec,dx,dy): # return ra,dec if mode is None or mode.lower() == 'none': def dither(ra, dec, dx, dy): return ra, dec TILINGS = [(0, 0), (0, 0), (0, 0), (0, 0)] elif mode.lower() == 'smash_dither': TILINGS = [(0, 0), (1.0, 0.0), (-1.0, 0.0), (0.0, -0.75)] dither = self.smash_dither elif mode.lower() == 'smash_rotate': TILINGS = [(0, 0), (0.75, 0.75), (-0.75, 0.75), (0.0, -0.75)] dither = self.smash_rotate elif mode.lower() == 'decam_dither': TILINGS = [(0., 0.), (8 / 3. * CCD_X, -11 / 3. * CCD_Y), (8 / 3. * CCD_X, 8 / 3. * CCD_Y), (-8 / 3. * CCD_X, 0.)] dither = self.decam_dither if infile is None: infile = os.path.join(fileio.get_datadir(), 'smash_fields_alltiles.txt') data = np.recfromtxt(infile, names=True) # Apply footprint selection after tiling/dither #sel = obztak.utils.projector.footprint(data['RA'],data['DEC']) # This is currently a non-op smash_id = data['ID'] ra = data['RA'] dec = data['DEC'] nhexes = len(data) #ntilings = len(DECAM_DITHERS) ntilings = len(TILINGS) nbands = len(BANDS) nfields = nhexes * nbands * ntilings logging.info("Number of hexes: %d" % nhexes) logging.info("Number of tilings: %d" % ntilings) logging.info("Number of filters: %d" % nbands) fields = FieldArray(nfields) fields['HEX'] = np.tile(np.repeat(smash_id, nbands), ntilings) fields['PRIORITY'].fill(1) fields['TILING'] = np.repeat(np.arange(1, ntilings + 1), nhexes * nbands) fields['FILTER'] = np.tile(BANDS, nhexes * ntilings) #for i in range(ntilings): for i, tiling in enumerate(TILINGS): idx0 = i * nhexes * nbands idx1 = idx0 + nhexes * nbands ra_dither, dec_dither = dither(ra, dec, tiling[0], tiling[1]) fields['RA'][idx0:idx1] = np.repeat(ra_dither, nbands) fields['DEC'][idx0:idx1] = np.repeat(dec_dither, nbands) # Apply footprint selection after tiling/dither sel = self.footprint(fields['RA'], fields['DEC']) # NORMAL OPERATION if smcnod: # Include SMC northern overdensity fields sel_smcnod = self.footprintSMCNOD(fields) # SMCNOD OPERATION sel = sel | sel_smcnod #sel = sel_smcnod fields['PRIORITY'][sel_smcnod] = 99 #if True: # # Include 'bridge' region between Magellanic Clouds # sel_bridge = self.footprintBridge(fields['RA'],fields['DEC']) # sel = sel | sel_bridge sel = sel & (fields['DEC'] > constants.SOUTHERN_REACH) fields = fields[sel] logging.info("Number of target fields: %d" % len(fields)) if plot: import pylab as plt import obztak.utils.ortho plt.ion() fig, basemap = obztak.utils.ortho.makePlot('2016/2/11 03:00', center=(0, -90), airmass=False, moon=False) proj = obztak.utils.ortho.safeProj(basemap, fields['RA'], fields['DEC']) basemap.scatter(*proj, c=fields['TILING'], edgecolor='none', s=50, cmap='Spectral', vmin=0, vmax=len(TILINGS)) colorbar = plt.colorbar(label='Tiling') if outfile: outfig = os.path.splitext(outfile)[0] + '.png' fig.savefig(outfig, bbox_inches='tight') if not sys.flags.interactive: plt.show(block=True) if outfile: fields.write(outfile) return fields
Option( "--objid", help="Event ID to be stored in FITS header [default: %default]"), ]) opts, args = parser.parse_args() infilename = command.get_input_filename(parser, args) if opts.output is None: parser.error('--output: missing required argument') # Late imports. import numpy as np import lalinference.fits import lalinference.bayestar.postprocess samples = np.recfromtxt(infilename, names=True) theta = 0.5 * np.pi - samples['dec'] phi = samples['ra'] p = lalinference.bayestar.postprocess.adaptive_healpix_histogram( theta, phi, opts.samples_per_bin, nside=opts.nside, max_nside=opts.max_nside) # Write output to FITS file. lalinference.fits.write_sky_map(opts.output, p, creator=parser.get_prog_name(), objid=opts.objid,
def create_all(): db.create_all(bind=None) telescopes = ["ZTF", "Gattini", "DECam", "KPED", "GROWTH-India"] available_filters = { "ZTF": ["g", "r", "i"], "Gattini": ["J"], "DECam": ["g", "r", "i", "z"], "KPED": ["U", "g", "r", "i"], "GROWTH-India": ["g", "r", "i", "z"] } plan_args = { 'ZTF': { 'filt': ['g', 'r', 'g'], 'exposuretimes': [300.0, 300.0, 300.0], 'doReferences': True, 'doUsePrimary': True, 'doBalanceExposure': False, 'doDither': False, 'usePrevious': False, 'doCompletedObservations': False, 'doPlannedObservations': False, 'cobs': [None, None], 'schedule_type': 'greedy', 'filterScheduleType': 'block', 'airmass': 2.5, 'schedule_strategy': 'tiling', 'mindiff': 30. * 60., 'doMaxTiles': False, 'max_nb_tiles': 1000 }, 'DECam': { 'filt': ['g', 'z'], 'exposuretimes': [25.0, 25.0], 'doReferences': True, 'doUsePrimary': False, 'doBalanceExposure': False, 'doDither': True, 'usePrevious': False, 'doCompletedObservations': False, 'doPlannedObservations': False, 'cobs': [None, None], 'schedule_type': 'greedy_slew', 'filterScheduleType': 'integrated', 'airmass': 2.5, 'schedule_strategy': 'tiling', 'mindiff': 30. * 60., 'doMaxTiles': False, 'max_nb_tiles': 1000 }, 'Gattini': { 'filt': ['J'], 'exposuretimes': [300.0], 'doReferences': False, 'doUsePrimary': False, 'doBalanceExposure': False, 'doDither': False, 'usePrevious': False, 'doCompletedObservations': False, 'doPlannedObservations': False, 'cobs': [None, None], 'schedule_type': 'greedy', 'filterScheduleType': 'block', 'airmass': 2.5, 'schedule_strategy': 'tiling', 'mindiff': 30. * 60., 'doMaxTiles': False, 'max_nb_tiles': 1000 }, 'KPED': { 'filt': ['r'], 'exposuretimes': [300.0], 'doReferences': False, 'doUsePrimary': False, 'doBalanceExposure': False, 'doDither': False, 'usePrevious': False, 'doCompletedObservations': False, 'doPlannedObservations': False, 'cobs': [None, None], 'schedule_type': 'greedy', 'filterScheduleType': 'integrated', 'airmass': 2.5, 'schedule_strategy': 'catalog', 'mindiff': 30. * 60., 'doMaxTiles': False, 'max_nb_tiles': 1000 }, 'GROWTH-India': { 'filt': ['r'], 'exposuretimes': [300.0], 'doReferences': False, 'doUsePrimary': False, 'doBalanceExposure': False, 'doDither': False, 'usePrevious': False, 'doCompletedObservations': False, 'doPlannedObservations': False, 'cobs': [None, None], 'schedule_type': 'greedy', 'filterScheduleType': 'integrated', 'airmass': 2.5, 'schedule_strategy': 'catalog', 'mindiff': 30. * 60., 'doMaxTiles': False, 'max_nb_tiles': 1000 } } with tqdm(telescopes) as telescope_progress: for tele in telescope_progress: telescope_progress.set_description('populating {}'.format(tele)) filename = pkg_resources.resource_filename(__name__, 'input/%s.ref' % tele) if os.path.isfile(filename): refstable = table.Table.read(filename, format='ascii', data_start=2, data_end=-1) refs = table.unique(refstable, keys=['field', 'fid']) if "maglimcat" not in refs.columns: refs["maglimcat"] = np.nan reference_images = { group[0]['field']: group['fid'].astype(int).tolist() for group in refs.group_by('field').groups } reference_mags = { group[0]['field']: group['maglimcat'].tolist() for group in refs.group_by('field').groups } else: reference_images = {} reference_mags = {} tesspath = 'input/%s.tess' % tele try: tessfile = app.open_instance_resource(tesspath) except IOError: tessfile = pkg_resources.resource_stream(__name__, tesspath) tessfilename = tessfile.name tessfile.close() fields = np.recfromtxt(tessfilename, usecols=range(3), names=['field_id', 'ra', 'dec']) with pkg_resources.resource_stream(__name__, 'config/%s.config' % tele) as g: config_struct = {} for line in g.readlines(): line_without_return = line.decode().split("\n") line_split = line_without_return[0].split(" ") line_split = list(filter(None, line_split)) if line_split: try: config_struct[line_split[0]] = float(line_split[1]) except ValueError: config_struct[line_split[0]] = line_split[1] db.session.merge( Telescope(telescope=tele, lat=config_struct["latitude"], lon=config_struct["longitude"], elevation=config_struct["elevation"], timezone=config_struct["timezone"], filters=available_filters[tele], default_plan_args=plan_args[tele])) for field_id, ra, dec in tqdm(fields, 'populating fields'): ref_filter_ids = reference_images.get(field_id, []) ref_filter_mags = [] for val in reference_mags.get(field_id, []): ref_filter_mags.append(val) bands = {1: 'g', 2: 'r', 3: 'i', 4: 'z', 5: 'J'} ref_filter_bands = [bands.get(n, n) for n in ref_filter_ids] if config_struct["FOV_type"] == "square": ipix, radecs, patch, area = gwemopt.utils.getSquarePixels( ra, dec, config_struct["FOV"], Localization.nside) elif config_struct["FOV_type"] == "circle": ipix, radecs, patch, area = gwemopt.utils.getCirclePixels( ra, dec, config_struct["FOV"], Localization.nside) if len(radecs) == 0: continue corners = np.vstack((radecs, radecs[0, :])) if corners.size == 10: corners_copy = copy.deepcopy(corners) corners[2] = corners_copy[3] corners[3] = corners_copy[2] contour = { 'type': 'Feature', 'geometry': { 'type': 'MultiLineString', 'coordinates': [corners.tolist()] }, 'properties': { 'telescope': tele, 'field_id': int(field_id), 'ra': ra, 'dec': dec, 'depth': dict(zip(ref_filter_bands, ref_filter_mags)) } } db.session.merge( Field(telescope=tele, field_id=int(field_id), ra=ra, dec=dec, contour=contour, reference_filter_ids=ref_filter_ids, reference_filter_mags=ref_filter_mags, ipix=ipix.tolist())) if tele == "ZTF": quadrant_coords = get_ztf_quadrants() skyoffset_frames = coordinates.SkyCoord( fields['ra'], fields['dec'], unit=u.deg).skyoffset_frame() quadrant_coords_icrs = coordinates.SkyCoord( *np.tile(quadrant_coords[:, np.newaxis, ...], (len(fields), 1, 1)), unit=u.deg, frame=skyoffset_frames[:, np.newaxis, np.newaxis]).transform_to( coordinates.ICRS) quadrant_xyz = np.moveaxis( quadrant_coords_icrs.cartesian.xyz.value, 0, -1) for field_id, xyz in zip( tqdm(fields['field_id'], 'populating subfields'), quadrant_xyz): for ii, xyz in enumerate(xyz): ipix = hp.query_polygon(Localization.nside, xyz) db.session.merge( SubField(telescope=tele, field_id=int(field_id), subfield_id=int(ii), ipix=ipix.tolist()))