def read(input_fname, gold_fname, output_fname=None, with_confidence=False, max_sent_size=_max_sent_size): inp = read_system_input(input_fname, max_sent_size=max_sent_size) gold = read_gold_standard(gold_fname) if output_fname: out = read_system_output(output_fname, with_confidence) return merge_arrays((inp, gold, out), flatten=True) else: return merge_arrays((inp, gold), flatten=True)
def test_solo_w_flatten(self): # Test merge_arrays on a single array w & w/o flattening w = self.data[0] test = merge_arrays(w, flatten=False) assert_equal(test, w) test = merge_arrays(w, flatten=True) control = np.array([(1, 2, 3.0), (4, 5, 6.0)], dtype=[("a", int), ("ba", float), ("bb", int)]) assert_equal(test, control)
def test_wmasked_arrays(self): # Test merge_arrays masked arrays (_, x, _, _) = self.data mx = ma.array([1, 2, 3], mask=[1, 0, 0]) test = merge_arrays((x, mx), usemask=True) control = ma.array([(1, 1), (2, 2), (-1, 3)], mask=[(0, 1), (0, 0), (1, 0)], dtype=[("f0", int), ("f1", int)]) assert_equal(test, control) test = merge_arrays((x, mx), usemask=True, asrecarray=True) assert_equal(test, control) assert_(isinstance(test, MaskedRecords))
def test_flatten(self): # Test standard & flexible (_, x, _, z) = self.data test = merge_arrays((x, z), flatten=True) control = np.array([(1, "A", 1.0), (2, "B", 2.0)], dtype=[("f0", int), ("A", "|S3"), ("B", float)]) assert_equal(test, control) test = merge_arrays((x, z), flatten=False) control = np.array( [(1, ("A", 1.0)), (2, ("B", 2.0))], dtype=[("f0", int), ("f1", [("A", "|S3"), ("B", float)])] ) assert_equal(test, control)
def test_flatten(self): # Test standard & flexible (_, x, _, z) = self.data test = merge_arrays((x, z), flatten=True) control = np.array([(1, 'A', 1.), (2, 'B', 2.)], dtype=[('f0', int), ('A', '|S3'), ('B', float)]) assert_equal(test, control) test = merge_arrays((x, z), flatten=False) control = np.array([(1, ('A', 1.)), (2, ('B', 2.))], dtype=[('f0', int), ('f1', [('A', '|S3'), ('B', float)])]) assert_equal(test, control)
def test_flatten_wflexible(self): # Test flatten standard & nested (w, x, _, _) = self.data test = merge_arrays((x, w), flatten=True) control = np.array( [(1, 1, 2, 3.0), (2, 4, 5, 6.0)], dtype=[("f0", int), ("a", int), ("ba", float), ("bb", int)] ) assert_equal(test, control) test = merge_arrays((x, w), flatten=False) controldtype = [("f0", int), ("f1", [("a", int), ("b", [("ba", float), ("bb", int)])])] control = np.array([(1.0, (1, (2, 3.0))), (2, (4, (5, 6.0)))], dtype=controldtype) assert_equal(test, control)
def test_w_shorter_flex(self): # Test merge_arrays w/ a shorter flexndarray. z = self.data[-1] # Fixme, this test looks incomplete and broken # test = merge_arrays((z, np.array([10, 20, 30]).view([('C', int)]))) # control = np.array([('A', 1., 10), ('B', 2., 20), ('-1', -1, 20)], # dtype=[('A', '|S3'), ('B', float), ('C', int)]) # assert_equal(test, control) # Hack to avoid pyflakes warnings about unused variables merge_arrays((z, np.array([10, 20, 30]).view([("C", int)]))) np.array([("A", 1.0, 10), ("B", 2.0, 20), ("-1", -1, 20)], dtype=[("A", "|S3"), ("B", float), ("C", int)])
def test_standard(self): # Test standard & standard # Test merge arrays (_, x, y, _) = self.data test = merge_arrays((x, y), usemask=False) control = np.array([(1, 10), (2, 20), (-1, 30)], dtype=[("f0", int), ("f1", int)]) assert_equal(test, control) test = merge_arrays((x, y), usemask=True) control = ma.array( [(1, 10), (2, 20), (-1, 30)], mask=[(0, 0), (0, 0), (1, 0)], dtype=[("f0", int), ("f1", int)] ) assert_equal(test, control) assert_equal(test.mask, control.mask)
def test_solo(self): # Test merge_arrays on a single array. (_, x, _, z) = self.data test = merge_arrays(x) control = np.array([(1,), (2,)], dtype=[('f0', int)]) assert_equal(test, control) test = merge_arrays((x,)) assert_equal(test, control) test = merge_arrays(z, flatten=False) assert_equal(test, z) test = merge_arrays(z, flatten=True) assert_equal(test, z)
def test_flatten(self): # Test standard & flexible (_, x, _, z) = self.data test = merge_arrays((x, z), flatten=True) control = np.array( [(1, "A", 1.0), (2, "B", 2.0)], dtype=[("f0", int), ("A", "|S3"), ("B", float)], ) assert_equal(test, control) test = merge_arrays((x, z), flatten=False) control = np.array( [(1, ("A", 1.0)), (2, ("B", 2.0))], dtype=[("f0", int), ("f1", [("A", "|S3"), ("B", float)])], ) assert_equal(test, control)
def computeFunction(self, data): # If we haven't specified a domain if self._domain == None or self._range == None: # We look for a domain in our parents up = self._parent while up != None: if isinstance(up, FilterModule): if up._domain != None and up._range != None: self._domain = up._domain self._range = up._range up = up._parent if self._domain != None and self._range != None: if self._range == "Custom": try: #print "custom = " + self._customFunction exec("custom = " + self._customFunction) except: custom = np.zeros(data.shape[0], dtype=[('Custom', 'f4')]) func = merge_arrays((data[self._domain], custom), asrecarray=True, flatten=True).copy().view(HDFunction) func.normalize(norm=self._norm) #print func.dtype.names,func.dtype.names[:-1] + ('Custom',) func.dtype.names = func.dtype.names[:-1] + ('Custom', ) return func else: return data.function(self._domain + [self._range], norm=self._norm) else: return None
def readHSTsample(self): """ # read the HST galaxy training sample """ if os.path.isfile(self.finName): catfinal = fitsio.read(self.finName) else: cosmos_cat = galsim.COSMOSCatalog(self.catName, dir=self.directory) # used index index_use = cosmos_cat.orig_index # used catalog paracat = cosmos_cat.param_cat[index_use] # parametric catalog oricat = fitsio.read( cosmos_cat.real_cat.getFileName())[index_use] ra = oricat['RA'] dec = oricat['DEC'] indexNew = np.arange(len(ra), dtype=int) __tmp = np.stack([ra, dec, indexNew]).T radec = np.array([tuple(__t) for __t in __tmp], dtype=[('ra', '>f8'), ('dec', '>f8'), ('index', 'i8')]) catfinal = rfn.merge_arrays([paracat, radec], flatten=True, usemask=False) fitsio.write(self.finName, catfinal) self.catused = catfinal return
def write_tree(self): array = [] all_branches = default_branches + ["label", "train_id"] for aux in all_branches: print aux a = self.append_arrays(aux) if aux == "label" or aux == "train_id": tree_name = aux elif "evt_weight" in aux: tree_name = "weight" else: tree_name = aux[:-1] a_ = self.create_structured_array(a, tree_name) array.append(a_) for mva in self.mva_helpers.keys(): a = [ y for x in [ self.mva_helpers[mva]["prediction"]["train"], self.mva_helpers[mva]["prediction"]["test"], self.mva_helpers[mva]["prediction"]["data"] ] for y in x ] a_ = self.create_structured_array(a, mva) array.append(a_) merged_array = rfn.merge_arrays(array, flatten=True, usemask=False) self.output_root = self.output + ".root" os.system("rm %s" % self.output_root) root_numpy.array2root(merged_array, self.output_root, treename="t")
def chromosome_index(self): snps = self.snps n = len(snps) index = np.arange(n).astype([('index',int)]) chroms = snps['chromosome'].astype('S10') chromset = set(chroms) chroms = chroms.astype([('chromosome','S10')]) locs = snps['location'].astype(int).astype([('location',int)]) snps = rfn.merge_arrays([chroms,locs,index]) snps.sort() index = {} for name in chromset: mask = snps['chromosome']==name indices = snps['index'][mask] pos = snps['location'][mask] if name.startswith('chr'): name = name[3:] if name.upper()=='MT': name = 'M' index[name] = pos,indices return index
def load_as_recarr(self, filename, fields=None, formatmarker='# format:'): """Warning: if there's missing value, it will be filled with default value. See: https://docs.scipy.org/doc/numpy/user/basics.rec.html numpy.lib.recfunctions.merge_arrays() -1 for integers -1.0 for floating point numbers '-' for characters '-1' for strings True for boolean values """ cols = self.load(filename, fields=fields, formatmarker=formatmarker) _fields, cols_data = cols.keys(), cols.values( ) # None fields is updated during self.load cols_type = [ self.name2fmter[field_name]._type_ for field_name in _fields ] #for name,x,t in zip(fields, cols_data, cols_type): # print '***', name,x,t cols_nparr = [ np.array(x, dtype=t) for name, x, t in zip(_fields, cols_data, cols_type) ] unnamed_recarr = rfn.merge_arrays(cols_nparr, flatten=True, usemask=False).view(np.recarray) return rfn.rename_fields( unnamed_recarr, dict(zip(unnamed_recarr.dtype.names, _fields)))
def __label_ner(self, ners_offset, ners_label, tokens_start, tokens_end): # add sublabel to ner labels. If ner has multiple token the first token label starts with I-, the next tokens starts with B-. One token ner's label always starts with I- # ners info: label, start and end chars positions. ners_lbl = [] ners_st_id = [] ners_end_id = [] for offset, label in zip(ners_offset, ners_label): # get ner start/stop chars ids (from xml) pos_ = [p.split("-") for p in offset.split(";")] # if the ner has multiple non-consecutive tokens, then len(pos_)>1; i.e. i>0 (in the below loop) for i, pos in enumerate(pos_): # i>0; multiple non-consecutive tokens ner sublabels = ["I-", "I-"] if i > 0 else ["B-", "I-"] # We have start/stop indices for all tokens in the sentence. Now we want to get ner start/stop indices from this all tokens indices. (relate tokens with ner) # Find entity's start/stop chars ids (from xml) in tokens start/stop ids lists. i.e. search "pos" in "tokens_start" and "tokens_end" lists # if the ner consists of multiple (consecutive) tokens, both of the start and the stop index will have the same location in "tokens_start" and "tokens_end" lists. # accounts for start sequence // add 9 to ids ner_st_id = tokens_start.index(int(pos[0]) + 9) # this piece of code handel the dataset issue discussed in README.md: section 1.3.2. Plural Entity if int(pos[1]) + 1 + 9 in tokens_end: ner_end_id = tokens_end.index(int(pos[1]) + 1 + 9) elif int(pos[1]) + 1 + 1 + 9 in tokens_end: ner_end_id = tokens_end.index(int(pos[1]) + 1 + 1 + 9) # multiple (non-consecutive or consecutive) tokens ner, sublabel depends on the value of i if ner_st_id != ner_end_id: labels = [label] * (ner_end_id - ner_st_id + 1) labels = [sublabels[0] + labels[0]] + \ [sublabels[1] + lb for lb in labels[1:]] # single token ner; non-consecutive ner elif i > 0: labels = ["I-" + label] # single token ner else: labels = ["B-" + label] # store and return ner info ners_lbl.extend(labels) ners_st_id.extend(tokens_start[ner_st_id:ner_end_id + 1]) ners_end_id.extend(tokens_end[ner_st_id:ner_end_id + 1]) ner_data = rfn.merge_arrays((ners_st_id, ners_end_id, ners_lbl)) # token could have multiple tag, see README.MD. ner_data = np.unique(ner_data, axis=0) ner_start = ner_data[ner_data.dtype.names[0]] ner_end = ner_data[ner_data.dtype.names[1]] ner_labels = ner_data[ner_data.dtype.names[2]] # check that there are no overllaping in start/stop ids # temp = np.unique(np.column_stack((ner_start, ner_end)).flatten()) # assert np.all(temp[:-1] <= temp[1:]) return ner_start, ner_end, ner_labels
def write_html(r, render_to='table.html'): types = numpy.zeros(len(r), dtype=[ ('Type', 'S60'), ]) for i, row in enumerate(r[type_fields]): for j, k in enumerate(row): if k == True: types[i][0] += type_fields[j] + ' | ' r = rfn.merge_arrays([r, types], flatten=True)[used_columns] x = PrettyTable(r.dtype.names) for row in r: x.add_row(row) with open('index.html', 'r') as template: with open(render_to, 'w') as output: s = template.read() s = s.replace( "{{ table }}", x.get_html_string( attributes={"class": "table table-hover table-striped"})) output.write(s)
def test_w_shorter_flex(self): # Test merge_arrays w/ a shorter flexndarray. z = self.data[-1] # Fixme, this test looks incomplete and broken # test = merge_arrays((z, np.array([10, 20, 30]).view([('C', int)]))) # control = np.array([('A', 1., 10), ('B', 2., 20), ('-1', -1, 20)], # dtype=[('A', '|S3'), ('B', float), ('C', int)]) # assert_equal(test, control) # Hack to avoid pyflakes warnings about unused variables merge_arrays((z, np.array([10, 20, 30]).view([("C", int)]))) np.array( [("A", 1.0, 10), ("B", 2.0, 20), ("-1", -1, 20)], dtype=[("A", "|S3"), ("B", float), ("C", int)], )
def findMovingAverage(outputCollection, collectionName, column, windowSize): """Find the average of column based on a window size Parameters ---------- outputCollection : str The collection name we store the final result as collectionName : str The collection name we use to take average column : str The column name we want the average of windowSize : int The number of rows we average at a moment """ wholeColumn = allCollections[collectionName][column] movingAverage = [] sumTillHere = 0 for rowNum in range(len(wholeColumn)): if rowNum < windowSize: sumTillHere += wholeColumn[rowNum] movingAverage.append(sumTillHere / (rowNum + 1)) else: sumTillHere -= wholeColumn[rowNum - windowSize] sumTillHere += wholeColumn[rowNum] movingAverage.append(sumTillHere / windowSize) movAvgTable = np.array(movingAverage, dtype=[('movavg(' + column + ')', 'float_')]) allCollections[outputCollection] = rfn.merge_arrays( (allCollections[collectionName], movAvgTable), flatten=True, usemask=False) printTable(allCollections[outputCollection]) outputToFile(outputCollection, 'allOperations', 'a+')
def produceBinWeighter(self, filenames): weighter = self.make_empty_weighter() branches = [self.weightbranchX, self.weightbranchY] showprog = ShowProgress(5, len(filenames)) counter = 0 if self.remove or self.weight: for fname in filenames: # Read truths truth_array = self.readTreeFromRootToTuple( fname, branches=self.truthclasses) # Use defined reduced truths reduced_array = self.reduceTruth(truth_array) dtype = zip(self.reducedtruthclasses, [col.dtype for col in reduced_array.transpose()]) reduced_array = reduced_array.ravel().view(dtype) # Read variables to weight along weight_along_array = self.readTreeFromRootToTuple( fname, branches=branches) # Merge two arrays import numpy.lib.recfunctions as rfn nparray = rfn.merge_arrays([reduced_array, weight_along_array], flatten=True, usemask=False) weighter.addDistributions(nparray, referenceclass=self.referenceclass) del nparray del truth_array del weight_along_array del reduced_array showprog.show(counter) counter = counter + 1 weighter.createRemoveProbabilitiesAndWeights(self.referenceclass) return weighter
def restrict(self, restriction): restriction = np.asarray(restriction, bool) base = rsgame.empty_copy(self).restrict(restriction) size_mask = restriction.repeat(self._sizes) sizes = self._sizes[restriction] profiles = self._profiles[size_mask] lengths = self._lengths[restriction] zeros = (profiles[:, ~restriction] / lengths[:, ~restriction].repeat(sizes, 0)) removed = np.exp(-np.einsum('ij,ij->i', zeros, zeros) / 2) # pylint: disable=invalid-unary-operand-type uprofs, inds = np.unique(recfunctions.merge_arrays([ np.arange(restriction.sum()).repeat(sizes).view([('s', int)]), utils.axis_to_elem(profiles[:, restriction]) ], flatten=True), return_inverse=True) new_alpha = np.bincount(inds, removed * self._alpha[size_mask]) new_sizes = np.diff( np.concatenate([[-1], np.flatnonzero(np.diff(uprofs['s'])), [new_alpha.size - 1]])) return _RbfGpGame(base.role_names, base.strat_names, base.num_role_players, self._offset[restriction], self._coefs[restriction], lengths[:, restriction], new_sizes, uprofs['axis'], new_alpha)
def update_phase(self): """Perform all the updates on the modules. The update phase occurs N times, based on the user configuration for the experiment. This function loops over the instruments and post-processing modules (based on their priority) and calls their update method. One file will be written for each update. """ for update_number in range(self.config['updates']): current_data = np.array([(np.datetime64('now'), )], dtype=[('time', 'datetime64[us]')]) for module in self.modules: class_ = module.__class__ print("...{}: updating {}...".format( update_number, module.__class__.__name__)) if issubclass(class_, Instrument): try: module_data = module.update(update_number) except RuntimeError: self.cleanup_phase(abort=True) raise if module_data is not None: current_data = rfn.merge_arrays( [current_data, module_data], flatten=True) elif issubclass(class_, PostProcessing): current_data = module.update(update_number, current_data.copy()) filename = '{}/scan_data_{:03d}.npy'.format( self.config['directory'], update_number) with open(filename, 'xb') as data_file: np.save(data_file, current_data.copy(), allow_pickle=False)
def test_flatten_wflexible(self): # Test flatten standard & nested (w, x, _, _) = self.data test = merge_arrays((x, w), flatten=True) control = np.array([(1, 1, 2, 3.0), (2, 4, 5, 6.0)], dtype=[('f0', int), ('a', int), ('ba', float), ('bb', int)]) assert_equal(test, control) test = merge_arrays((x, w), flatten=False) controldtype = [('f0', int), ('f1', [('a', int), ('b', [('ba', float), ('bb', int)])])] control = np.array([(1., (1, (2, 3.0))), (2, (4, (5, 6.0)))], dtype=controldtype) assert_equal(test, control)
def run(self, outputs_requested, **kwargs): arrays = [kwargs[input_key].to_np() for input_key in self.__input_keys] # http://stackoverflow.com/questions/15815854/how-to-add-column-to-numpy-array out = UObject(UObjectPhase.Write) out.from_np(merge_arrays(arrays, flatten=True)) return {'output': out}
def restrict(self, restriction): restriction = np.asarray(restriction, bool) base = rsgame.empty_copy(self).restrict(restriction) size_mask = restriction.repeat(self._sizes) sizes = self._sizes[restriction] profiles = self._profiles[size_mask] lengths = self._lengths[restriction] zeros = (profiles[:, ~restriction] / lengths[:, ~restriction].repeat(sizes, 0)) removed = np.exp(-np.einsum('ij,ij->i', zeros, zeros) / 2) # pylint: disable=invalid-unary-operand-type uprofs, inds = np.unique( recfunctions.merge_arrays([ np.arange(restriction.sum()).repeat(sizes).view([('s', int)]), utils.axis_to_elem(profiles[:, restriction])], flatten=True), return_inverse=True) new_alpha = np.bincount(inds, removed * self._alpha[size_mask]) new_sizes = np.diff(np.concatenate([ [-1], np.flatnonzero(np.diff(uprofs['s'])), [new_alpha.size - 1]])) return _RbfGpGame( base.role_names, base.strat_names, base.num_role_players, self._offset[restriction], self._coefs[restriction], lengths[:, restriction], new_sizes, uprofs['axis'], new_alpha)
def parseLogs(self, infile): pprint("Parsing file"); # data is reconstructed as array of tuples self.data = np.genfromtxt(infile, delimiter=" ", comments="#", #dtype="float,float,int,int,int,int,int,int,int,int", #names=["ts", "ots", "cpu", "fid", "st0", "st1", "len", "count", "tot", "dlt"] dtype="float,float,int,int,int,int,int,int,int,int,int,int,int", names=["ts", "ots", "cpu", "fid", "st0", "st1", "len", "count", "tot", "dlt", "wrdlt", "wbps", "rbps"] ) # iterate; create and append new columns if 0: # (note, takes a while, so avoiding) pprint("Got %d entries; processing" % (len(self.data)) ) i = 0; rdtot=0; wrtot=0; rbps = 0; wbps = 0; wt=[]; #wd = []; wa =[]; ra=[]; for ix in self.data: if ix['fid'] == 1: wrtot = ix['tot'] wbps=wrtot/ix['ts']; if ix['fid'] == 2: rdtot = ix['tot'] rbps = rdtot/ix['ts'] #wd.append( wrtot-rdtot) #wa.append( wbps ); #ra.append( rbps ); wt.append ( (wrtot-rdtot, wbps, rbps) ) #my_appended_array = append_fields( self.data, names=['wrdlt','wbps','rbps'], dtypes="int,int,int", data=[wd, wa, ra] ) # problem; "expected a readable buffer" self.data = merge_arrays([self.data, np.array(wt, dtype=[('wrdlt', int), ('wbps', int), ('rbps', int)]) ], flatten=True) # cast to int works from here #end if 0 pprint("Done.") #(self.data)
def test_singlerecord(self): (_, x, y, z) = self.data test = merge_arrays((x[0], y[0], z[0]), usemask=False) control = np.array([(1, 10, ('A', 1))], dtype=[('f0', int), ('f1', int), ('f2', [('A', '|S3'), ('B', float)])]) assert_equal(test, control)
def test_w_singlefield(self): # Test single field test = merge_arrays((np.array([1, 2]).view([("a", int)]), np.array([10.0, 20.0, 30.0]))) control = ma.array( [(1, 10.0), (2, 20.0), (-1, 30.0)], mask=[(0, 0), (0, 0), (1, 0)], dtype=[("a", int), ("f1", float)] ) assert_equal(test, control)
def create_pointcloud(data, mode=None, groups=None, batch_columns=None): if mode is None: mode = Mode.STANDARD if groups is None: groups = {} groups = merge_dicts(DEFAULT_GROUPS, groups) if batch_columns is None: batch_columns = [] columns = [] def add(name, data): if name == 'position': columns.append(PositionColumn(name, data, mode)) elif name.lower() in FeatureColumn.TYPES: columns.append(FeatureColumn(name.lower(), data)) else: columns.append(BatchColumn(name, data)) # Remap columns based off their groupings grouped = set() for name, selection in groups.iteritems(): add(name, data[selection]) grouped.update( selection if isinstance(selection, list) else [selection]) for column in (set(data.dtype.names) - grouped): add(column, data[column]) # Find all mapped values and replace it with their mapping if len(batch_columns) > 0: r_columns, r_names = [], [] for column in batch_columns: if column not in columns: raise Exception('Column %s does not exist' % column) r_column = columns[columns.index(column)] r_columns.append(r_column) r_names.extend(r_column.names()) merged_data = merge_arrays([x.data() for x in r_columns], flatten=True, usemask=False) merged_data.dtype.names = r_names batch_groups, batch_ids = np.unique(merged_data, axis=0, return_inverse=True) batch_ids = batch_ids.astype(np.uint16) idx_offset = 0 for column in r_columns: column.is_instanced = True d_names = column.names() selector = d_names[0] if column.count() == 1 else d_names column._data = batch_groups[selector] columns.append( FeatureColumn('batch_id', batch_ids, {'BATCH_LENGTH': len(batch_groups)})) return PointcloudTile(columns)
def test_w_singlefield(self): # Test single field test = merge_arrays( (np.array([1, 2]).view([('a', int)]), np.array([10., 20., 30.])), ) control = ma.array([(1, 10.), (2, 20.), (-1, 30.)], mask=[(0, 0), (0, 0), (1, 0)], dtype=[('a', int), ('f1', float)]) assert_equal(test, control)
def __init__(self, size=default_size, freq="1m"): super(ArrayManager, self).__init__(size=size) dt_int = np.array([(0, )] * size, dtype=np.dtype([('datetimeint', np.int64)])) self.array = rfn.merge_arrays([dt_int, self.array], flatten=True, usemask=False) self._freq = freq
def fix_hj_fracgood_colmns(objects): nrows = objects.size dtypes = np.dtype([('FRACGOOD', '>f4')]) dummy = np.empty(nrows, dtype=dtypes) dummy[:] = 1.0 obj = rfn.merge_arrays([objects,dummy], flatten=True, usemask=False) # return obj
def test_standard(self): # Test standard & standard # Test merge arrays (_, x, y, _) = self.data test = merge_arrays((x, y), usemask=False) control = np.array([(1, 10), (2, 20), (-1, 30)], dtype=[("f0", int), ("f1", int)]) assert_equal(test, control) test = merge_arrays((x, y), usemask=True) control = ma.array( [(1, 10), (2, 20), (-1, 30)], mask=[(0, 0), (0, 0), (1, 0)], dtype=[("f0", int), ("f1", int)], ) assert_equal(test, control) assert_equal(test.mask, control.mask)
def test_w_singlefield(self): # Test single field test = merge_arrays((np.array([1, 2]).view([('a', int)]), np.array([10., 20., 30.])),) control = ma.array([(1, 10.), (2, 20.), (-1, 30.)], mask=[(0, 0), (0, 0), (1, 0)], dtype=[('a', int), ('f1', float)]) assert_equal(test, control)
def test_singlerecord(self): (_, x, y, z) = self.data test = merge_arrays((x[0], y[0], z[0]), usemask=False) control = np.array( [(1, 10, ("A", 1))], dtype=[("f0", int), ("f1", int), ("f2", [("A", "|S3"), ("B", float)])], ) assert_equal(test, control)
def finalizeObjects(self, objects): objs = numpy.recarray(len(objects), dtype=[('NAME','S24'), ('TS','f4'), ('GLON','f4'), ('GLAT','f4'), ('RA','f4'), ('DEC','f4'), ('MODULUS','f4'), ('DISTANCE','f4'), ('RICHNESS','f4'), ('MASS','f4'), ('NANNULUS','i4'), ('NINTERIOR','i4'), ]) objs['TS'] = self.values[objects['IDX_MAX'],objects['ZIDX_MAX']] lon,lat = objects['X_MAX'],objects['Y_MAX'] coordsys = self.config['coords']['coordsys'] if coordsys.lower() == 'gal': print("GAL coordintes") objs['GLON'],objs['GLAT'] = lon,lat objs['RA'],objs['DEC'] = gal2cel(lon,lat) else: print("CEL coordintes") objs['RA'],objs['DEC'] = lon,lat objs['GLON'],objs['GLAT'] = cel2gal(lon,lat) modulus = objects['Z_MAX'] objs['MODULUS'] = modulus objs['DISTANCE'] = mod2dist(modulus) nside = healpy.npix2nside(len(self.nannulus)) pix = ang2pix(nside,lon,lat) richness = self.richness[objects['IDX_MAX'],objects['ZIDX_MAX']] objs['RICHNESS'] = richness objs['MASS'] = richness * self.stellar[pix] objs['NANNULUS'] = self.nannulus[pix].astype(int) objs['NINTERIOR'] = self.ninterior[pix].astype(int) # Default name formatting # http://cdsarc.u-strasbg.fr/ftp/pub/iau/ # http://cds.u-strasbg.fr/vizier/Dic/iau-spec.htx fmt = "J%(hour)02i%(hmin)04.1f%(deg)+03i%(dmin)02i" for obj,_ra,_dec in zip(objs,objs['RA'],objs['DEC']): hms = dec2hms(_ra); dms = dec2dms(_dec) params = dict(hour=hms[0],hmin=hms[1]+hms[2]/60., deg=dms[0],dmin=dms[1]+dms[2]/60.) obj['NAME'] = fmt%params out = recfuncs.merge_arrays([objs,objects],usemask=False, asrecarray=True,flatten=True) return out
def test_flatten_wflexible(self): # Test flatten standard & nested (w, x, _, _) = self.data test = merge_arrays((x, w), flatten=True) control = np.array( [(1, 1, 2, 3.0), (2, 4, 5, 6.0)], dtype=[("f0", int), ("a", int), ("ba", float), ("bb", int)], ) assert_equal(test, control) test = merge_arrays((x, w), flatten=False) controldtype = [ ("f0", int), ("f1", [("a", int), ("b", [("ba", float), ("bb", int)])]), ] control = np.array([(1.0, (1, (2, 3.0))), (2, (4, (5, 6.0)))], dtype=controldtype) assert_equal(test, control)
def get_series(data, series_name): desired = data['calibration_%s_record_desired_pose' % series_name] desired,_ = transform_from_start(desired) d_i = time_cluster(desired) desired = desired[numpy.hstack([numpy.diff(d_i)>0,[True]])] desired = rfn.merge_arrays([desired,quat_to_rpq(desired)],flatten=True) estimated = data['calibration_%s_record_estimated_pose' % series_name] estimated,_ = transform_from_start(estimated) e_i = time_cluster(estimated) return desired, estimated, e_i
def extend(data, const, duplicate): if len(duplicate) == 0: return data new_vals = tuple([const[f][0] for f in duplicate]) new_fields = [gen_name(field, data.dtype.names) for field in duplicate] dtype = [(name, np.dtype(type(val))) for (name, val) in zip(new_fields, new_vals)] extension = np.empty(len(data), dtype=dtype) extension.fill(new_vals) data = merge_arrays((data, extension), usemask=False, flatten=True) return data
def test_w_singlefield(self): # Test single field test = merge_arrays((np.array([1, 2]).view( [("a", int)]), np.array([10.0, 20.0, 30.0])), ) control = ma.array( [(1, 10.0), (2, 20.0), (-1, 30.0)], mask=[(0, 0), (0, 0), (1, 0)], dtype=[("a", int), ("f1", float)], ) assert_equal(test, control)
def test_sql(self): # Make sure we don't accidentally corrupt our test database db_path, db_file_name = self._tmp_files.tmp_copy(path_of_data( 'small.db')) db_url = 'sqlite:///{}'.format(db_path) q_sel_employees = 'CREATE TABLE {tmp_emp} AS SELECT * FROM employees;' # We have to be careful about the datetime type in sqlite3. It will # forget if we don't keep reminding it, and if it forgets sqlalchemy # will be unhappy. Hence, we can't use CREATE TABLE AS if our table # has a DATETIME q_sel_hours = ('CREATE TABLE {tmp_hrs} ' '(id INT, employee_id INT, time DATETIME, ' ' event_type TEXT); ' 'INSERT INTO {tmp_hrs} SELECT * FROM hours;') q_join = ('CREATE TABLE {joined} ' '(id INT, last_name TEXT, salary REAL, time DATETIME, ' ' event_type TEXT); ' 'INSERT INTO {joined} ' 'SELECT {tmp_emp}.id, last_name, salary, time, event_type ' 'FROM {tmp_emp} JOIN {tmp_hrs} ON ' '{tmp_emp}.id = {tmp_hrs}.employee_id;') p = Pipeline() get_emp = p.add(RunSQL(db_url, q_sel_employees, [], ['tmp_emp'], {})) get_hrs = p.add(RunSQL(db_url, q_sel_hours, [], ['tmp_hrs'], {})) join = p.add(RunSQL(db_url, q_join, ['tmp_emp', 'tmp_hrs'], ['joined'], {})) csv_out = p.add(CSVWrite(self._tmp_files('out.csv'))) get_emp['tmp_emp'] > join['tmp_emp'] get_hrs['tmp_hrs'] > join['tmp_hrs'] join['joined'] > csv_out['input'] self.run_pipeline(p) ctrl = csv_read(path_of_data('test_transform_test_sql_ctrl.csv')) result = self._tmp_files.csv_read('out.csv') # Because Numpy insists on printing times with local offsets, but # not every computer has the same offset, we have to force it back # into UTC for i, dt in enumerate(result['time']): # .item() makes a datetime, which we can format correctly later # http://stackoverflow.com/questions/25134639/how-to-force-python-print-numpy-datetime64-with-specified-timezone result['time'][i] = np.datetime64(dt).item().strftime( '%Y-%m-%dT%H:%M:%S') # Then we have to make the string field smaller new_cols = [] for col in result.dtype.names: new_cols.append(result[col].astype(ctrl.dtype[col])) result = merge_arrays(new_cols, flatten=True) result.dtype.names = ctrl.dtype.names self.assertTrue(np.array_equal(result, ctrl))
def test_sql(self): # Make sure we don't accidentally corrupt our test database db_path, db_file_name = self._tmp_files.tmp_copy( path_of_data('small.db')) db_url = 'sqlite:///{}'.format(db_path) q_sel_employees = 'CREATE TABLE {tmp_emp} AS SELECT * FROM employees;' # We have to be careful about the datetime type in sqlite3. It will # forget if we don't keep reminding it, and if it forgets sqlalchemy # will be unhappy. Hence, we can't use CREATE TABLE AS if our table # has a DATETIME q_sel_hours = ('CREATE TABLE {tmp_hrs} ' '(id INT, employee_id INT, time DATETIME, ' ' event_type TEXT); ' 'INSERT INTO {tmp_hrs} SELECT * FROM hours;') q_join = ('CREATE TABLE {joined} ' '(id INT, last_name TEXT, salary REAL, time DATETIME, ' ' event_type TEXT); ' 'INSERT INTO {joined} ' 'SELECT {tmp_emp}.id, last_name, salary, time, event_type ' 'FROM {tmp_emp} JOIN {tmp_hrs} ON ' '{tmp_emp}.id = {tmp_hrs}.employee_id;') p = Pipeline() get_emp = p.add(RunSQL(db_url, q_sel_employees, [], ['tmp_emp'], {})) get_hrs = p.add(RunSQL(db_url, q_sel_hours, [], ['tmp_hrs'], {})) join = p.add( RunSQL(db_url, q_join, ['tmp_emp', 'tmp_hrs'], ['joined'], {})) csv_out = p.add(CSVWrite(self._tmp_files('out.csv'))) get_emp['tmp_emp'] > join['tmp_emp'] get_hrs['tmp_hrs'] > join['tmp_hrs'] join['joined'] > csv_out['input'] self.run_pipeline(p) ctrl = csv_read(path_of_data('test_transform_test_sql_ctrl.csv')) result = self._tmp_files.csv_read('out.csv') # Because Numpy insists on printing times with local offsets, but # not every computer has the same offset, we have to force it back # into UTC for i, dt in enumerate(result['time']): # .item() makes a datetime, which we can format correctly later # http://stackoverflow.com/questions/25134639/how-to-force-python-print-numpy-datetime64-with-specified-timezone result['time'][i] = np.datetime64(dt).item().strftime( '%Y-%m-%dT%H:%M:%S') # Then we have to make the string field smaller new_cols = [] for col in result.dtype.names: new_cols.append(result[col].astype(ctrl.dtype[col])) result = merge_arrays(new_cols, flatten=True) result.dtype.names = ctrl.dtype.names self.assertTrue(np.array_equal(result, ctrl))
def finalizeObjects(self, objects): objs = numpy.recarray(len(objects), dtype=[('NAME','S24'), ('TS','f4'), ('GLON','f4'), ('GLAT','f4'), ('RA','f4'), ('DEC','f4'), ('MODULUS','f4'), ('DISTANCE','f4'), ('RICHNESS','f4'), ('MASS','f4'), ('NANNULUS','i4'), ('NINTERIOR','i4'), ]) objs['TS'] = self.values[objects['IDX_MAX'],objects['ZIDX_MAX']] glon,glat = objects['X_MAX'],objects['Y_MAX'] objs['GLON'],objs['GLAT'] = glon,glat ra,dec = gal2cel(glon,glat) objs['RA'],objs['DEC'] = ra,dec modulus = objects['Z_MAX'] objs['MODULUS'] = modulus objs['DISTANCE'] = mod2dist(modulus) #ninterior = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'NINSIDE') #nannulus = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'NANNULUS') #stellar = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'STELLAR') nside = healpy.npix2nside(len(self.nannulus)) pix = ang2pix(nside,glon,glat) richness = self.richness[objects['IDX_MAX'],objects['ZIDX_MAX']] objs['RICHNESS'] = richness objs['MASS'] = richness * self.stellar[pix] objs['NANNULUS'] = self.nannulus[pix].astype(int) objs['NINTERIOR'] = self.ninterior[pix].astype(int) # Default name formatting # http://cdsarc.u-strasbg.fr/ftp/pub/iau/ # http://cds.u-strasbg.fr/vizier/Dic/iau-spec.htx fmt = "J%(hour)02i%(hmin)04.1f%(deg)+03i%(dmin)02i" for obj,_ra,_dec in zip(objs,ra,dec): hms = dec2hms(_ra); dms = dec2dms(_dec) params = dict(hour=hms[0],hmin=hms[1]+hms[2]/60., deg=dms[0],dmin=dms[1]+dms[2]/60.) obj['NAME'] = fmt%params out = recfuncs.merge_arrays([objs,objects],usemask=False,asrecarray=True,flatten=True) # This is safer than viewing as FITS_rec return pyfits.new_table(out).data
def list_shank_units(bird, sess, shank, shank_file=None, sorted=False): meta_dt = np.dtype([('sess', 'S32', 1), ('shank', np.int, 1), ('is_good', 'b', 1)]) kwik_file = et.open_kwik(bird, sess) if shank_file is None else et.open_kwik(bird, sess, shank_file) group = int(shank) all_units = kwf.list_units(kwik_file, group=group, sorted=False) n_units = all_units.size all_meta = np.recarray(n_units, dtype=meta_dt) all_meta['sess'] = sess all_meta['shank'] = shank all_meta['is_good'] = True return rfn.merge_arrays((all_meta, all_units), asrecarray=True, flatten=True)
def classify_line(filename, classifier): """ Use `classifier` to classify data stored in `filename` Args: filename (str): filename of stored results classifier (sklearn classifier): pre-trained classifier """ z = np.load(filename) rec = z['record'] if rec.shape[0] == 0: logger.debug('No records in {f}. Continuing'.format(f=filename)) return # Rescale intercept term coef = rec['coef'].copy() # copy so we don't transform npz coef coef[:, 0, :] = (coef[:, 0, :] + coef[:, 1, :] * ((rec['start'] + rec['end']) / 2.0)[:, np.newaxis]) # Include RMSE for full X matrix newdim = (coef.shape[0], coef.shape[1] * coef.shape[2]) X = np.hstack((coef.reshape(newdim), rec['rmse'])) # Create output and classify classes = classifier.classes_ classified = np.zeros(rec.shape[0], dtype=[ ('class', 'u2'), ('class_proba', 'float32', classes.size) ]) classified['class'] = classifier.predict(X) classified['class_proba'] = classifier.predict_proba(X) # Replace with new classification if exists, or add by merging if ('class' in rec.dtype.names and 'class_proba' in rec.dtype.names and rec['class_proba'].shape[1] == classes.size): rec['class'] = classified['class'] rec['class_proba'] = classified['class_proba'] else: # Drop incompatible classified results if needed # e.g., if the number of classes changed if 'class' in rec.dtype.names and 'class_proba' in rec.dtype.names: rec = nprfn.drop_fields(rec, ['class', 'class_proba']) rec = nprfn.merge_arrays((rec, classified), flatten=True) # Create dict for re-saving `npz` file (only way to append) out = {} for k, v in z.iteritems(): out[k] = v out['classes'] = classes out['record'] = rec np.savez(filename, **out)
def __init__(self, ax, map_fig, zplot_fig, cmdplot_fig, ccdplot_fig, avplot_fig, cmd_ax, ccd_ax, avhist_ax, zhist_ax, data, point_list = {}): self.previous_point = [] self.start_point = [] self.end_point = [] self.line = None if point_list == {}: self.point_list = point_list else: self.point_list = pickle.load(open(point_list,'rb')) self.map_fig = map_fig self.map_fig.canvas.draw() self.map_fig.canvas.set_window_title('Spatial Map') self.data = data self.zplot_fig = zplot_fig self.zplot_fig.canvas.set_window_title('Metallicity Histogram') self.avplot_fig = avplot_fig self.avplot_fig.canvas.set_window_title('Av Histogram') self.cmdplot_fig = cmdplot_fig self.cmdplot_fig.canvas.set_window_title('Color-Magnitude Diagram') self.ccdplot_fig = ccdplot_fig self.ccdplot_fig.canvas.set_window_title('Color-Color Diagram') self.cmd_ax = cmd_ax self.ccd_ax = ccd_ax self.avhist_ax = avhist_ax self.zhist_ax = zhist_ax self.ax = ax self.star_arr = np.zeros((len(self.data),1),dtype=[('REGION_NUM',int)]) self.star_arr['REGION_NUM'] += 999 self.region_arr = rfn.merge_arrays([self.data,self.star_arr],flatten = True) self.region_arr = np.ma.masked_array(self.region_arr,np.isnan(self.region_arr['Z'])) self.region_counter = 0 self.draw_spatial() self.color_list = ['red','blue','green','purple','black'] self.color_num = 0 self.color_mod = 0 self.region_data = 0 self.region_check = False
def __init__(self, *arrays): """Constructs DataObject from arbitrary number of ndarrays. Each ndarray can have an arbitrary number of fields. Field names should all be capitalized and words in multi-word field names should be separated by underscores if necessary. ndarrays have a 'size' property---their sizes should all be equivalent. Args: arrays (numpy.ndarray): ndarrays with equivalent sizes. """ self._ndarray = merge_arrays(arrays, flatten=True)
def _build_nodes(cls, matching_problem: MatchingProblem) -> np.ndarray: """Method to build the nodes based on a matching problem""" courier_ids, route_ids = cls._get_entities_ids(matching_problem) route_demands = np.array(-1 * np.ones(route_ids.shape), dtype=[('demand', '<i8')]) courier_demands = np.zeros(courier_ids.shape, dtype=[('demand', '<i8')]) courier_nodes = rfn.merge_arrays([courier_ids, courier_demands], flatten=True, usemask=False) route_nodes = rfn.merge_arrays([route_ids, route_demands], flatten=True, usemask=False) supply_node = np.array([('supply', len(route_ids))], dtype=[('id', '<U100'), ('demand', '<i8')]) return np.concatenate((courier_nodes, route_nodes, supply_node), axis=0)
def extend_list(runlist, *newcol_name): newlist = runlist for index, value in enumerate(newcol_name): newcol = np.zeros(np.size(runlist), dtype={ 'names': [value], 'formats': ['f8'] }) newlist = rfn.merge_arrays((newlist, newcol), flatten=True, usemask=False) return newlist
def create_pointcloud(data, mode=None, groups=None, batch_columns=None): if mode is None: mode = Mode.STANDARD if groups is None: groups = {} groups = merge_dicts(DEFAULT_GROUPS, groups) if batch_columns is None: batch_columns = [] columns = [] def add(name, data): if name == 'position': columns.append(PositionColumn(name, data, mode)) elif name.lower() in FeatureColumn.TYPES: columns.append(FeatureColumn(name.lower(), data)) else: columns.append(BatchColumn(name, data)) # Remap columns based off their groupings grouped = set() for name, selection in groups.iteritems(): add(name, data[selection]) grouped.update(selection if isinstance(selection, list) else [selection]) for column in (set(data.dtype.names) - grouped): add(column, data[column]) # Find all mapped values and replace it with their mapping if len(batch_columns) > 0: r_columns, r_names = [], [] for column in batch_columns: if column not in columns: raise Exception('Column %s does not exist' % column) r_column = columns[columns.index(column)] r_columns.append(r_column) r_names.extend(r_column.names()) merged_data = merge_arrays([ x.data() for x in r_columns ], flatten=True, usemask=False) merged_data.dtype.names = r_names batch_groups, batch_ids = np.unique(merged_data, axis=0, return_inverse=True) batch_ids = batch_ids.astype(np.uint16) idx_offset = 0 for column in r_columns: column.is_instanced = True d_names = column.names() selector = d_names[0] if column.count() == 1 else d_names column._data = batch_groups[selector] columns.append(FeatureColumn('batch_id', batch_ids, { 'BATCH_LENGTH': len(batch_groups) })) return PointcloudTile(columns)
def addTermToMatrix(term,ndMat): rows = len(ndMat) if (rows == 0): rows=1 newfield = np.zeros((rows,1),dtype=[(term,'i8')]) newarr = merge_arrays([ndMat,newfield],flatten=True) if ('f0' in newarr.dtype.names): #this clears up the first column if the array started as 0x0 #newarr.dtype.names is a tuple, needs to be a list for remove names=list(newarr.dtype.names) names.remove('f0') newarr=newarr[names] return newarr
def read_oneproc(self,i): f = open(datadir+'proc'+str(i)+'/'+inputfilename,'rb') procdata = [] ndump = 0 ndump_recorded = 0 nstalk_tot_thisproc = 0 while f.read(1): f.seek(-1,1) ndump = ndump +1 header= np.fromfile(f,dtype=headertype,count=1) #print('header',header) nstalk_loc = header['nstalk_loc'][0] if (nstalk_loc>0): pad = np.fromfile(f,dtype=np.int32,count=1) ipar_stalk = np.fromfile(f,dtype=np.dtype([('ipar',np.int32)]),count=nstalk_loc) #read off after-padding pad = np.fromfile(f,dtype=np.int32,count=1) pad = np.fromfile(f,dtype=np.int32,count=1) if not(pad==nstalk_loc*stalktype.itemsize): print('proc',nproc,'dump',ndump,'padding at data ',pad,' while nstalk_loc*stalktype.itemze ', nstalk_loc*stalktype.itemsize) exit() values_stalk = np.fromfile(f,dtype=stalktype,count=nstalk_loc) pad = np.fromfile(f,dtype=np.int32,count=1) if ( header['tstalk'][0] >= min_time_to_record ): merged = rfn.merge_arrays( [ipar_stalk, values_stalk], flatten = True , usemask = False, asrecarray=True) merged = rfn.append_fields( merged, 'tstalk', data = header['tstalk'][0]*np.ones(nstalk_loc) ) #compress on max ipar condition merged = merged.compress(merged['ipar'] < max_ipar_to_record) if (ndump==1): nstalk_tot_thisproc = len(merged) ndump_recorded = ndump_recorded + 1 procdata.append([header['tstalk'][0],merged]) if (len(procdata) >= MAX_QUEUE_CHUNK): self.result_queue.put(procdata) procdata = [] elif (ndump==1): #compress on max ipar condition nstalk_tot_thisproc = len(ipar_stalk.compress(ipar_stalk['ipar'] < max_ipar_to_record)) continue elif (ndump ==1): #nothing to read form this proc on first dump, still need to count particles recorded nstalk_tot_thisproc = 0 f.close() if (len(procdata) >0): self.result_queue.put(procdata) self.result_queue2.put([nstalk_tot_thisproc, ndump_recorded])
def test_measure_latency(): data = numpy.ndarray(shape=(NUM_SAMPLES), dtype=RECORD) for n, (frame, _) in enumerate(stbt.frames(50)): if n >= NUM_SAMPLES: break stbt_receive_time = numpy.array( [(frame.time, time.time())], dtype=OWN_TIMESTAMPS) timestamps = read_timestamps(frame) data[n] = merge_arrays([timestamps, stbt_receive_time], flatten=True, usemask=False) # Sometimes we'll lose a row but no biggie numpy.savetxt("latency-test.txt", data[:n])
def updateArray(data): ''' Adds the results containers to the data product. ''' newData = np.zeros(data.size) data = rfns.append_fields(data, ['ML_pred_1d', 'ML_pred_2d', 'ML_pred_3d'], [newData, newData, newData], dtypes='>f4', usemask=False) newnewData = np.zeros(data.size, dtype=[('ML_pred_1d_err', '>f4', (2,)), ('ML_pred_2d_err', '>f4', (2,)), ('ML_pred_3d_err', '>f4', (2,)),]) data = rfns.merge_arrays((data, newnewData), usemask=False, asrecarray=False, flatten=True) return data
def visitOffsets(visits, zpOff=1.): """ Convert an opsim array to have more generic names and add any extra columns we may want """ dnames = visits.dtype.names dnames = ['ra' if (x=='fieldRA' or x=='ditheredRA') else x for x in dnames] dnames = ['dec' if (x=='fieldDec' or x=='ditheredDec') else x for x in dnames] dnames = ['visitID' if (x=='obsHistID') else x for x in dnames] visits.dtype.names = dnames zp = (np.random.uniform(size=visits.size)*zpOff).astype(zip(['zpOff'],[float])) visits = rfn.merge_arrays([visits, zp], flatten=True, usemask=False) return visits
def makeDataValues(size=100, min=0., max=1., nd=3, random=True): """Generate a simple array of numbers, evenly arranged between min/max, in nd dimensions, but (optional) random order.""" data = [] for d in range(nd): datavalues = np.arange(0, size, dtype='float') datavalues *= (float(max) - float(min)) / (datavalues.max() - datavalues.min()) datavalues += min if random: randorder = np.random.rand(size) randind = np.argsort(randorder) datavalues = datavalues[randind] datavalues = np.array(zip(datavalues), dtype=[('testdata'+ '%d' %(d), 'float')]) data.append(datavalues) data = rfn.merge_arrays(data, flatten=True, usemask=False) return data
def getInfo(self,catalog=None): """ Returns info for the source from either a specified catalog or all of the catalogs that contain the object Arguments: catalog: optional pointer to catalog object that contains the source. If this field is missing the function will return infomation from every catalog containing the object Returns: Structured np.array """ # to get the name of each field use self.data.dtype.names myInfo=np.array(self.data,copy=True) if catalog is None: for n,cat in enumerate(self.catalogs): catalog=openCatalog(cat) objects=np.sort(catalog[objects],order='id')['objects'] myInfo=rfn.merge_arrays(myInfo,obj) return myInfo
def _run_plugin_update(self, plugin, update_number, data): """Run the update phase on one PLACE plugin""" class_ = plugin.__class__ elm_name = plugin.elm_module_name try: if issubclass(class_, Instrument): new_data = plugin.update( update_number, self.progress.experiment['plugins'][elm_name]['progress']) if new_data is not None: data = rfn.merge_arrays([data, new_data], flatten=True) elif issubclass(class_, PostProcessing): data = plugin.update(update_number, data.copy()) except RuntimeError as err: self.progress.message = str(err) self.cleanup_phase(abort=True) raise return data