def peaks_ridges1d(wavetrset, ridgesset, scalevals, padfirstdimintoarray=True, minridgelength=3, minchildlength=0., maxscale_localmax=None, minridgewtsum=100., minchildwtsum=0., verbose=False): """ peaks_ridges1d(wavetrset, ridgesset, scalevals, padfirstdimintoarray=True, minridgelength=3, minchildlength=0., maxscale_localmax=None, minridgewtsum=100., minchildwtsum=0., verbose=False) wavetrset, ridgesset, scalevals are from previous function runs, see example.py minridgelength: minimum length (i.e. number of wavelet scales) of ridge for it to count as a peak minchildlength: same thing but for length of the child ridge alone, not count its mother's length maxscale_localmax: the largest scale in which the ridge can be locally maximum and still count as a peak minridgewtsum: critical value for the sum of the wavelet transform over the ridge. This will depend on the number of wavelet scales used and the nominal intensities of the original data. minchildwtsum: same thing except for the child ridge. The child ridge must have this total intensity to count as a peak, but for the "minridgewtsum", the child and mother intensities are added together. """ minridgelength=max(1, minridgelength) ridgescalevals=scalevals[::-1] #ordered big->small if maxscale_localmax is None: maxscale_localmax=max(scalevals) ridgescalecritind=numpy.where(ridgescalevals<=maxscale_localmax)[0] if len(ridgescalecritind)<2: print 'aborted: the set of qscales does not include more than 1 point in the specified qwidthrange' return 'aborted: the set of qscales does not include more than 1 point in the specified qwidthrange' ridgescalecritind=ridgescalecritind[0] peaks=[perform_peaks_ridges1d(wt, ridges, ridgescalecritind=ridgescalecritind, minridgelength=minridgelength, minchildlength=minchildlength, minridgewtsum=minridgewtsum, minchildwtsum=minchildwtsum, verbose=verbose) for wt, ridges in zip(wavetrset, ridgesset)] numpks=[len(p) for p in peaks] maxnp=max(numpks) filler=[[32767]*2]*maxnp if padfirstdimintoarray: for p in peaks: p+=filler[:len(filler)-len(p)] return numpy.uint32([pksort(numpy.uint32(p).T) for p in peaks]) else: return [pksort(numpy.uint32(p).T) for p in peaks]#list of 2 x numpeaks arrays,m the 2 are scaleind and posnind
def calculateComplexDerefOpAddress(complexDerefOp, registerMap): match = re.match("((?:\\-?0x[0-9a-f]+)?)\\(%([a-z0-9]+),%([a-z0-9]+),([0-9]+)\\)", complexDerefOp) if match != None: offset = 0L if len(match.group(1)) > 0: offset = long(match.group(1), 16) regA = RegisterHelper.getRegisterValue(match.group(2), registerMap) regB = RegisterHelper.getRegisterValue(match.group(3), registerMap) mult = long(match.group(4), 16) # If we're missing any of the two register values, return None if regA == None or regB == None: if regA == None: return (None, "Missing value for register %s" % match.group(2)) else: return (None, "Missing value for register %s" % match.group(3)) if RegisterHelper.getBitWidth(registerMap) == 32: val = int32(uint32(regA)) + int32(uint32(offset)) + (int32(uint32(regB)) * int32(uint32(mult))) else: # Assume 64 bit width val = int64(uint64(regA)) + int64(uint64(offset)) + (int64(uint64(regB)) * int64(uint64(mult))) return (long(val), None) return (None, "Unknown failure.")
def testInt(self): num = np.int(2562010) self.assertEqual(np.int(ujson.decode(ujson.encode(num))), num) num = np.int8(127) self.assertEqual(np.int8(ujson.decode(ujson.encode(num))), num) num = np.int16(2562010) self.assertEqual(np.int16(ujson.decode(ujson.encode(num))), num) num = np.int32(2562010) self.assertEqual(np.int32(ujson.decode(ujson.encode(num))), num) num = np.int64(2562010) self.assertEqual(np.int64(ujson.decode(ujson.encode(num))), num) num = np.uint8(255) self.assertEqual(np.uint8(ujson.decode(ujson.encode(num))), num) num = np.uint16(2562010) self.assertEqual(np.uint16(ujson.decode(ujson.encode(num))), num) num = np.uint32(2562010) self.assertEqual(np.uint32(ujson.decode(ujson.encode(num))), num) num = np.uint64(2562010) self.assertEqual(np.uint64(ujson.decode(ujson.encode(num))), num)
def radixSortKeysOnly(self, keyBits): i = numpy.uint32(0) bitStep = self.bitStep while (keyBits > i*bitStep): self.radixSortStepKeysOnly(bitStep, i*bitStep) i+=numpy.uint32(1)
def testIntMax(self): num = np.int(np.iinfo(np.int).max) self.assertEqual(np.int(ujson.decode(ujson.encode(num))), num) num = np.int8(np.iinfo(np.int8).max) self.assertEqual(np.int8(ujson.decode(ujson.encode(num))), num) num = np.int16(np.iinfo(np.int16).max) self.assertEqual(np.int16(ujson.decode(ujson.encode(num))), num) num = np.int32(np.iinfo(np.int32).max) self.assertEqual(np.int32(ujson.decode(ujson.encode(num))), num) num = np.uint8(np.iinfo(np.uint8).max) self.assertEqual(np.uint8(ujson.decode(ujson.encode(num))), num) num = np.uint16(np.iinfo(np.uint16).max) self.assertEqual(np.uint16(ujson.decode(ujson.encode(num))), num) num = np.uint32(np.iinfo(np.uint32).max) self.assertEqual(np.uint32(ujson.decode(ujson.encode(num))), num) if platform.architecture()[0] != '32bit': num = np.int64(np.iinfo(np.int64).max) self.assertEqual(np.int64(ujson.decode(ujson.encode(num))), num) # uint64 max will always overflow as it's encoded to signed num = np.uint64(np.iinfo(np.int64).max) self.assertEqual(np.uint64(ujson.decode(ujson.encode(num))), num)
def load_stack(folder_name): stack = None input_files = sorted(glob.glob(os.path.join(folder_name, '*'))) for i, file_name in enumerate(input_files): print file_name if file_name.endswith('h5') or file_name.endswith('hdf5'): infile = h5py.File(file_name) im = infile['/probabilities'][...] else: im = mahotas.imread(file_name) if len(im.shape) == 3: im = np.uint32(im[ :, :, 0 ]) + np.uint32(im[ :, :, 1 ]) * 2**8 + np.uint32(im[ :, :, 2 ]) * 2**16 if im.shape[0] > 400: im = im[60:60+400, 210:210+400] if rot != 0: im = np.rot90(im, rot) if stack is None: stack = np.zeros((len(input_files), im.shape[0], im.shape[1]), dtype=im.dtype) print 'Stack size={0}, dtype={1}.'.format(stack.shape, stack.dtype) stack[i,:,:] = im #print file_name return stack
def _set_kernel_arguments(self): """Tie arguments of OpenCL kernel-functions to the actual kernels set_kernel_arguments() is a private method, called by configure(). It uses the dictionary _cl_kernel_args. Note that by default, since TthRange is disabled, the integration kernels have tth_min_max tied to the tthRange argument slot. When setRange is called it replaces that argument with tthRange low and upper bounds. When unsetRange is called, the argument slot is reset to tth_min_max. """ self._cl_kernel_args["bsort_vertical"] = [self._cl_mem["input_data"].data, None] self._cl_kernel_args["bsort_horizontal"] = [self._cl_mem["input_data"].data, None] self._cl_kernel_args["filter_vertical"] = [self._cl_mem["input_data"].data, self._cl_mem["vector_vertical"].data, numpy.uint32(self.npt_width), numpy.uint32(self.npt_height), numpy.float32(0), numpy.float32(0.5), ] self._cl_kernel_args["filter_horizontal"] = [self._cl_mem["input_data"].data, self._cl_mem["vector_horizontal"].data, numpy.uint32(self.npt_width), numpy.uint32(self.npt_height), numpy.float32(0), numpy.float32(0.5), ]
def add_bucket_entry(uhash, pieces, first_bucket_vector, second_bucket_vector, point_index): h_index = np.uint64(first_bucket_vector[0]) + np.uint64(second_bucket_vector[0 + 2]) if h_index >= const.prime_default: h_index -= const.prime_default assert(h_index < const.prime_default) h_index = np.uint32(h_index) h_index = h_index % uhash.table_size control = np.uint64(first_bucket_vector[1]) + np.uint64(second_bucket_vector[1 + 2]) if control >= const.prime_default: control -= const.prime_default assert(control < const.prime_default) control = np.uint32(control) if uhash.t == 1: b = uhash.ll_hash_table[h_index] while b and b.control_value != control: b = b.next_bucket_in_chain # if bucket does not exist if b is None: uhash.buckets += 1 uhash.ll_hash_table[h_index] = lsh_structs.bucket(control, point_index, uhash.ll_hash_table[h_index]) else: bucket_entry = lsh_structs.bucket_entry(point_index, b.first_entry.next_entry) b.first_entry.next_entry = bucket_entry uhash.points += 1
def get_bucket(uhash, pieces, first_bucket_vector, second_bucket_vector): h_index = np.uint64(first_bucket_vector[0]) + np.uint64(second_bucket_vector[0 + 2]) if h_index >= const.prime_default: h_index -= const.prime_default assert(h_index < const.prime_default) h_index = np.uint32(h_index) h_index = h_index % uhash.table_size control = np.uint64(first_bucket_vector[1]) + np.uint64(second_bucket_vector[1 + 2]) if control >= const.prime_default: control -= const.prime_default assert(control < const.prime_default) control = np.uint32(control) if uhash.t == 2: index_hybrid = uhash.hybrid_hash_table[h_index] while index_hybrid: if index_hybrid.control_value == control: index_hybrid = C.pointer(index_hybrid)[1] return index_hybrid else: index_hybrid = C.pointer(index_hybrid)[1] if index_hybrid.point.is_last_bucket: return None l = index_hybrid.point.bucket_length index_hybrid = C.pointer(index_hybrid)[l] return None
def arm_timed_latch(self, latch_name, time=None, force=False): ''' Arm a timed latch. Use force=True to force even if already armed @param fpga host @param latch_name: base name for latch @param time: time in adc samples since epoch @param force: force arm of latch that is already armed ''' status = self.host.device_by_name('%s_status' %(latch_name)).read()['data'] # get armed, arm and load counts armed_before = status['armed'] arm_count_before = status['arm_count'] load_count_before = status['load_count'] # if not forcing it, check for already armed first if armed_before == True: if force == False: LOGGER.info('forcing arm of already armed timed latch %s' %latch_name) else: LOGGER.error('timed latch %s already armed, use force=True' %latch_name) return # we load immediate if not given time if time == None: self.host.device_by_name('%s_control0' %(latch_name)).write(arm=0, load_immediate=1) self.host.device_by_name('%s_control0' %(latch_name)).write(arm=1, load_immediate=1) LOGGER.info('Timed latch %s arm-for-immediate-loading attempt' %latch_name) else: # TODO check time values time_samples = numpy.uint64(time) time_msw = numpy.uint32((time_samples & 0x0000FFFFFFFFFFFF) >> 32) time_lsw = numpy.uint32((time_samples & 0x00000000FFFFFFFF)) self.host.device_by_name('%s_control' %(latch_name)).write(load_time_lsw=time_lsw) self.host.device_by_name('%s_control0' %(latch_name)).write(arm=0, load_immediate=0, load_time_msw=time_msw) self.host.device_by_name('%s_control0' %(latch_name)).write(arm=1, load_immediate=0, load_time_msw=time_msw) LOGGER.info('Timed latch %s arm-for-loading-at-time attempt' %latch_name) # TODO check that arm count increased as expected status = self.host.device_by_name('%s_status' %(latch_name)).read()['data'] # get armed, arm and load counts armed_after = status['armed'] arm_count_after = status['arm_count'] load_count_after = status['load_count'] # armed count did not succeed if arm_count_after != (arm_count_before+1): # TODO check time LOGGER.error('Timed latch %s arm count at %i instead of %i' %(latch_name, arm_count_after, (arm_count_before+1))) else: # check load count increased as expected if time == None: if load_count_after != (load_count_before+1): LOGGER.error('Timed latch %s load count at %i instead of %i' %(latch_name, load_count_after, (load_count_before+1))) else: LOGGER.info('Timed latch %s successfully armed' %(latch_name))
def combine_scaled(r, g, b, a): """Combine components in [0, 1] to rgba uint32""" r2 = min(255, np.uint32(r * 255)) g2 = min(255, np.uint32(g * 255)) b2 = min(255, np.uint32(b * 255)) a2 = min(255, np.uint32(a * 255)) return np.uint32((a2 << 24) | (b2 << 16) | (g2 << 8) | r2)
def test_valid(self): prop = bcpp.Int() assert prop.is_valid(None) assert prop.is_valid(0) assert prop.is_valid(1) assert prop.is_valid(np.int8(0)) assert prop.is_valid(np.int8(1)) assert prop.is_valid(np.int16(0)) assert prop.is_valid(np.int16(1)) assert prop.is_valid(np.int32(0)) assert prop.is_valid(np.int32(1)) assert prop.is_valid(np.int64(0)) assert prop.is_valid(np.int64(1)) assert prop.is_valid(np.uint8(0)) assert prop.is_valid(np.uint8(1)) assert prop.is_valid(np.uint16(0)) assert prop.is_valid(np.uint16(1)) assert prop.is_valid(np.uint32(0)) assert prop.is_valid(np.uint32(1)) assert prop.is_valid(np.uint64(0)) assert prop.is_valid(np.uint64(1)) # TODO (bev) should fail assert prop.is_valid(False) assert prop.is_valid(True)
def __call__(self, queue, tgt, src, shape): w, h = shape assert w % block_size == 0 assert h % block_size == 0 return self.kernel(queue, (w, h), None, tgt, src, numpy.uint32(w), numpy.uint32(h))
def set_data(self, time, lcids=None, pbids=None, nsamples=None, exptimes=None): mf = cl.mem_flags if self._b_time is not None: self._b_time.release() self._b_lcids.release() self._b_pbids.release() self._b_nsamples.release() self._b_etimes.release() self.nlc = uint32(1 if lcids is None else unique(lcids).size) self.npb = uint32(1 if pbids is None else unique(pbids).size) self.nptb = time.size self.time = asarray(time, dtype='float32') self.lcids = zeros(time.size, 'uint32') if lcids is None else asarray(lcids, dtype='uint32') self.pbids = zeros(self.nlc, 'uint32') if pbids is None else asarray(pbids, dtype='uint32') self.nsamples = ones(self.nlc, 'uint32') if nsamples is None else asarray(nsamples, dtype='uint32') self.exptimes = ones(self.nlc, 'float32') if exptimes is None else asarray(exptimes, dtype='float32') self._b_time = cl.Buffer(self.ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=self.time) self._b_lcids = cl.Buffer(self.ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=self.lcids) self._b_pbids = cl.Buffer(self.ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=self.pbids) self._b_nsamples = cl.Buffer(self.ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=self.nsamples) self._b_etimes = cl.Buffer(self.ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=self.exptimes)
def pad(self, max_pad): """ Pad the timeseries data so that there are no missing values. We fill in missing power values using the previous power value in the series. """ width = self.times[-1] - self.times[0] + 1 padded_array = np.rec.array((0, 2), dtype=[('time', np.uint32), ('power', np.float32)]) padded_array.resize(width) cnt = 0 for i in xrange(len(self.times)-1): padded_array[cnt] = (np.uint32(self.times[i]), self.powers[i]) cnt += 1 if self.times[i+1] - self.times[i] > max_pad: continue for t in xrange(self.times[i]+1, self.times[i+1]): padded_array[cnt] = (np.uint32(t), self.powers[i]) cnt += 1 padded_array[cnt] = (np.uint32(self.times[-1]), self.powers[-1]) padded_array.resize(cnt + 1) self.array = padded_array
def X_div_Y(self, alpha, X, x0, Y, y0, beta, result = None): ''' retutn alpha*(X[,] + x0) / (Y[,] + y0) + beta X_div_Y(float alpha, float* X, float x0, float* Y, float y0, float beta, float* result, uint Y_col, uint Y_row) ''' if result is None: Y_col = Y.shape[0] Y_row = Y.shape[1] self.X_div_Y_kernel(np.float32(alpha), X.gpudata, np.float32(x0), \ Y.gpudata, np.float32(y0), \ np.float32(beta), Y.gpudata, \ np.uint32(Y_col), np.uint32(Y_row), \ block = (32, 32, 1), \ grid = (int(Y_row / 32) + 1, int(Y_col / 32) + 1) \ ) else: Y_col = Y.shape[0] Y_row = Y.shape[1] self.X_div_Y_kernel(np.float32(alpha), X.gpudata, np.float32(x0), \ Y.gpudata, np.float32(y0), \ np.float32(beta), result.gpudata, \ np.uint32(Y_col), np.uint32(Y_row), \ block = (32, 32, 1), \ grid = (int(Y_row / 32) + 1, int(Y_col / 32) + 1) \ )
def index_to_array(self, index, dim, result = None): ''' index_to_array(float* index, float* result, uint r_col, uint r_row) ''' if result is None: result = gpuarray.GPUArray([index.size, dim], dtype = self.dtype) self.index_to_array_kernel(index.gpudata, result.gpudata, \ np.uint32(r_col), np.uint32(r_row), \ block = self._2d_block, \ grid = self._2d_grid(r_col, r_row) \ ) return result else: r_col = result.shape[0] r_row = result.shape[1] if r_col != index.size or r_row != dim: raise ValueError('index_to_array: the dim of the result dont match the input') self.index_to_array_kernel(index.gpudata, result.gpudata, \ np.uint32(r_col), np.uint32(r_row), \ block = self._2d_block, \ grid = self._2d_grid(r_col, r_row) \ )
def _fold_exp_and_coh(t_array, w, tz, tau_arr): if tz != 0.: t_array -= tz shape = t_array.shape t_array = t_array.astype(np.float32) t_arr_gpu = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=t_array) tau_buf = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=(1/tau_arr).astype(np.float32)) shape = (shape[0], shape[1], tau_arr.size) shape_coh = (shape[0], shape[1], 3) out = cl_array.empty(queue, shape=shape, dtype=np.float32) out_coh = cl_array.empty(queue, shape=shape_coh, dtype=np.float32) global_work_size = t_array.size + (work_size[0] - t_array.size % work_size[0]) prg.fold_exp(queue, (global_work_size, tau_arr.size), work_size, t_arr_gpu, np.float32(w), tau_buf, out.data, np.uint32(t_array.size)) coh_no_div.coh_gauss(queue, (global_work_size, 3), work_size, t_arr_gpu, np.float32(w/1.4142), out_coh.data, np.uint32(t_array.size)) queue.finish() a = out.get(async_=True) b = out_coh.get(async_=True) b /= np.abs(b).max(0) queue.finish() return a, b
def geo_mask(self): """Masking the space pixels from geometry info.""" cfac = np.uint32(self.proj_info['CFAC']) lfac = np.uint32(self.proj_info['LFAC']) coff = np.float32(self.proj_info['COFF']) loff = np.float32(self.proj_info['LOFF']) nlines = int(self.data_info['number_of_lines']) ncols = int(self.data_info['number_of_columns']) # count starts at 1 local_coff = 1 local_loff = (self.total_segments - self.segment_number) * nlines + 1 xmax, ymax = get_geostationary_angle_extent(self.area) pixel_cmax = np.rad2deg(xmax) * cfac * 1.0 / 2**16 pixel_lmax = np.rad2deg(ymax) * lfac * 1.0 / 2**16 def ellipse(line, col): return ((line / pixel_lmax) ** 2) + ((col / pixel_cmax) ** 2) <= 1 cols_idx = da.arange(-(coff - local_coff), ncols - (coff - local_coff), dtype=np.float, chunks=CHUNK_SIZE) lines_idx = da.arange(nlines - (loff - local_loff), -(loff - local_loff), -1, dtype=np.float, chunks=CHUNK_SIZE) return ellipse(lines_idx[:, None], cols_idx[None, :])
def _minmax_impl(a_gpu, axis, min_or_max, stream=None): ''' Returns both max and argmax (min/argmin) along an axis.''' assert len(a_gpu.shape) < 3 if iscomplextype(a_gpu.dtype): raise ValueError("Cannot compute min/max of complex values") if axis is None: ## Note: PyCUDA doesn't have an overall argmax/argmin! if min_or_max == 'max': return gpuarray.max(a_gpu).get() else: return gpuarray.min(a_gpu).get() else: if axis < 0: axis += 2 assert axis in (0, 1) global _global_cublas_allocator alloc = _global_cublas_allocator n, m = a_gpu.shape if a_gpu.flags.c_contiguous else (a_gpu.shape[1], a_gpu.shape[0]) col_kernel, row_kernel = _get_minmax_kernel(a_gpu.dtype, min_or_max) if (axis == 0 and a_gpu.flags.c_contiguous) or (axis == 1 and a_gpu.flags.f_contiguous): target = gpuarray.empty(m, dtype=a_gpu.dtype, allocator=alloc) idx = gpuarray.empty(m, dtype=np.uint32, allocator=alloc) col_kernel(a_gpu, target, idx, np.uint32(m), np.uint32(n), block=(32, 1, 1), grid=(m, 1, 1), stream=stream) else: target = gpuarray.empty(n, dtype=a_gpu, allocator=alloc) idx = gpuarray.empty(n, dtype=np.uint32, allocator=alloc) row_kernel(a_gpu, target, idx, np.uint32(m), np.uint32(n), block=(32, 1, 1), grid=(n, 1, 1), stream=stream) return target, idx
def _build_synth_assignments(self,f_lo,phase_offset=0,tag='a'): """ Build phase increment words for synthesizer. Parameters ---------- f_lo : float LO frequency. phase_offset : float LO phase offset in radians. tag : string Tag associated with the downconverter, should be one of {'a','b','c','d','e','f'}. Default is 'a'. Returns ------- assign_synth : dict Each (key,val) pair can be used to assign devices as in roach2.write_int(key,val) """ # calculate discretized parameter values phase_res = 2**self.PHASE_LOOKUP_DEPTH mask = phase_res-1 dphi = uint32(f_lo/self.ADC_SAMPLE_RATE*phase_res) dphi_demux = uint32((dphi*self.DEMUX) % phase_res) phi0 = uint32(phase_offset*phase_res/(2*pi)) # build single 30bit value word = ((dphi_demux&mask)<<(self.PHASE_LOOKUP_DEPTH*2)) | ((dphi&mask)<<self.PHASE_LOOKUP_DEPTH) | (phi0&mask) assign_synth = {'ddc_1st_' + tag + '_synth_input_dphi':word} return assign_synth
def getTimeBaseIndices(self, name, tBegin, tEnd): """ Return time indices of name corresponding to tBegin and tEnd """ if not self.status: raise Exception('Shotfile not open!') try: sigName = ctypes.c_char_p(name) except TypeError: sigName = ctypes.c_char_p(name.encode()) error = ctypes.c_int32(0) info = self.getTimeBaseInfo(name) if tEnd < tBegin: temp = tEnd tEnd = tBegin tBegin = temp if tBegin < info.tBegin: tBegin = info.tBegin if tEnd > info.tEnd: tEnd = info.tEnd try: time1 = ctypes.c_float(tBegin) except TypeError: time1 = ctypes.c_float(tBegin.value) try: time2 = ctypes.c_float(tEnd) except TypeError: time2 = ctypes.c_float(tEnd.value) k1 = ctypes.c_uint32(0) k2 = ctypes.c_uint32(0) lname = ctypes.c_uint64(len(name)) __libddww__.ddtindex_(ctypes.byref(error), ctypes.byref(self.diaref), sigName, ctypes.byref(time1), ctypes.byref(time2), ctypes.byref(k1), ctypes.byref(k2), lname) getError(error.value) return numpy.uint32(k1.value), numpy.uint32(k2.value)
def decode(self, server, block_header, target, job_id = None, extranonce2 = None): if block_header: job = Object() binary_data = block_header.decode('hex') data0 = np.zeros(64, np.uint32) data0 = np.insert(data0, [0] * 16, unpack('IIIIIIIIIIIIIIII', binary_data[:64])) job.target = np.array(unpack('IIIIIIII', target.decode('hex')), dtype=np.uint32) job.header = binary_data[:68] job.merkle_end = np.uint32(unpack('I', binary_data[64:68])[0]) job.time = np.uint32(unpack('I', binary_data[68:72])[0]) job.difficulty = np.uint32(unpack('I', binary_data[72:76])[0]) job.state = sha256(STATE, data0) job.f = np.zeros(8, np.uint32) job.state2 = partial(job.state, job.merkle_end, job.time, job.difficulty, job.f) job.targetQ = 2**256 / int(''.join(list(chunks(target, 2))[::-1]), 16) job.job_id = job_id job.extranonce2 = extranonce2 job.server = server calculateF(job.state, job.merkle_end, job.time, job.difficulty, job.f, job.state2) if job.difficulty != self.difficulty: self.set_difficulty(job.difficulty) return job
def calculate_pressure(self): # Calculate atmospheric pressure in [Pa] self.B6 = self.B5 - 4000 print("B6=",self.B6) X1 = (self.B2 * (self.B6 * self.B6 / 2**12)) / 2**11 print("X1=",X1) X2 = self.AC2 * self.B6 / 2**11 print("X2=",X2) X3 = X1 + X2 print("X3=",X3) self.B3 = (((self.AC1 * 4 + X3) << self.BMP183_CMD['OVERSAMPLE_3']) + 2 ) / 4 print("B3=",self.B3) X1 = self.AC3 * self.B6 / 2**13 print("X1=",X1) X2 = (self.B1 * (self.B6 * self.B6 / 2**12)) / 2**16 print("X2=",X2) X3 = ((X1 + X2) + 2) / 2**2 print("X3=",X3) self.B4 = numpy.uint32 (self.AC4 * (X3 + 32768) / 2**15) print("B4=",self.B4) self.B7 = (numpy.uint32 (self.UP) - self.B3) * (50000 >> self.BMP183_CMD['OVERSAMPLE_3']) print("B7=",self.B7) p = numpy.uint32 ((self.B7 * 2) / self.B4) print("p=",p) X1 = (p / 2**8) * ( p / 2**8) print("X1=",X1) X1 = int (X1 * 3038) / 2**16 print("X1=",X1) X2 = int (-7357 * p) / 2**16 print("X2=",X2) self.pressure = p + (X1 + X2 +3791) / 2**4 print("pressure=",self.pressure)
def communion_encode(msg): assert msg["mode"] in ("request", "response") m = 'SEAMLESS'.encode() tip = b'\x00' if msg["mode"] == "request" else b'\x01' m += tip m += np.uint32(msg["id"]).tobytes() remainder = msg.copy() remainder.pop("mode") remainder.pop("id") remainder.pop("content") if len(remainder.keys()): rem = json.dumps(remainder).encode() nrem = np.uint32(len(rem)) m += nrem m += rem else: m += b'\x00\x00\x00\x00' content = msg["content"] if content is None: m += b'\x00' else: assert isinstance(content, (str, bytes, bool)), content if isinstance(content, bool): is_str = b'\x01' else: is_str = b'\x03' if isinstance(content, str) else b'\x02' m += is_str if isinstance(content, str): content = content.encode() elif isinstance(content, bool): content = b'\x01' if content else b'\x00' m += content #assert communion_decode(m) == msg, (communion_decode(m), msg) return m
def set_dac(self): @command() def set_dac_data(self, data): pass dac_data_1 = np.uint32(np.mod(np.floor(8192 * self.dac[0, :]) + 8192, 16384) + 8192) dac_data_2 = np.uint32(np.mod(np.floor(8192 * self.dac[1, :]) + 8192, 16384) + 8192) set_dac_data(self, dac_data_1 + 65536 * dac_data_2)
def set_demod(self, data): @command() def set_demod_buffer(self, data): pass data1 = np.uint32(np.mod(np.floor(8192 * data[0, :]) + 8192,16384) + 8192) data2 = np.uint32(np.mod(np.floor(8192 * data[1, :]) + 8192,16384) + 8192) set_demod_buffer(self, data1 + data2 * 2**16)
def color_count(image): '''Considering a (w,h,3) image of (dtype=uint8), compute the number of unique colors Encoding (i,j,k) into single index N = i+R*j+R*C*k Decoding N into (i,j,k) = (N-k*R*C-j*R, (N-k*R*C)/R, N/(R*C)) using integer division\n Inputs: image Returns: NumPy array of unique colors, number of pixels of each unique color in image ''' #Need to convert image to uint32 before multiplication so numbers are not truncated F = np.uint32(image[...,0])*256*256 + np.uint32(image[...,1])*256 + np.uint32(image[...,2]) unique, counts = np.unique(F, return_counts=True) colors = np.empty(shape=(len(unique),3),dtype=np.uint32) numcol = np.empty(len(unique),dtype=np.uint32) i = 0 for col,num in zip(unique,counts): R = col/(256*256) G = (col-R*256*256)/256 B = (col-R*256*256-G*256) colors[i] = (R,G,B) numcol[i] = num i+=1 return colors, numcol
def createSplitPerm(self, size, subset_ratio=0.8, seed=None): rng = np.random.RandomState(np.uint32((time.time()))) if seed is not None: rng.seed(np.uint32(seed)) perm = rng.permutation(size) k = int(size * subset_ratio) return perm[:k]
def __extractRegionIndices(self): ''' Function generates, and returns, a dictionary of indices for regions in self.regionFile.''' # Create variable to store region indices and loop through file regionIndices = {} with open(self.regionFile, 'r') as infile: for line in infile: # Extract region data and find indices of matching bins chrom, start, end, region = line.strip().split('\t') acceptableBins = ( (self.binChr == chrom) & (self.binStart >= np.uint32(start)) & (self.binEnd <= np.uint32(end))) indices = np.where(acceptableBins)[0] # Add region indices to dictionary if region in regionIndices: regionIndices[region] = np.concatenate( (regionIndices[region], indices)) else: regionIndices[region] = indices # Check region dictionary for absent or erronous regions for region in regionIndices: # Extract indices, sort, and update dictionary indices = regionIndices[region] indices.sort() regionIndices[region] = indices # Check for absent or duplicate indices if len(indices) == 0: raise IOError('{} has no bins'.format(region)) if len(set(indices)) != len(indices): raise IOError('{} has overlapping segments'.format(region)) # Return region index data return(regionIndices)
def xorshift32(t): x=t x=x^(x<<np.uint32(13)) x=x^(x>>np.uint32(17)) x=x^(x<<np.uint32(5)) return np.uint32(x)
def _pq_compute_single_core(proc_id, gt_jsons_set, pred_jsons_set, gt_pans_set, pred_pans_set, gt_image_jsons_set, categories): OFFSET = 256 * 256 * 256 VOID = 0 pq_stat = PQStat() for idx, (gt_json, pred_json, gt_pan, pred_pan, gt_image_json) in enumerate( zip(gt_jsons_set, pred_jsons_set, gt_pans_set, pred_pans_set, gt_image_jsons_set)): # if idx % 100 == 0: # logger.info('Compute pq -> Core: {}, {} from {} images processed'.format(proc_id, idx, len(gt_jsons_set))) gt_pan, pred_pan = np.uint32(gt_pan), np.uint32(pred_pan) pan_gt = gt_pan[:, :, 0] + gt_pan[:, :, 1] * 256 + gt_pan[:, :, 2] * 256 * 256 pan_pred = pred_pan[:, :, 0] + pred_pan[:, :, 1] * 256 + pred_pan[:, :, 2] * 256 * 256 gt_segms = {el['id']: el for el in gt_json['segments_info']} pred_segms = {el['id']: el for el in pred_json['segments_info']} # predicted segments area calculation + prediction sanity checks pred_labels_set = set(el['id'] for el in pred_json['segments_info']) labels, labels_cnt = np.unique(pan_pred, return_counts=True) for label, label_cnt in zip(labels, labels_cnt): if label not in pred_segms: if label == VOID: continue raise KeyError( 'In the image with ID {} segment with ID {} is presented in PNG and not presented in JSON.' .format(gt_ann['image_id'], label)) pred_segms[label]['area'] = label_cnt pred_labels_set.remove(label) if pred_segms[label]['category_id'] not in categories: raise KeyError( 'In the image with ID {} segment with ID {} has unknown category_id {}.' .format(gt_ann['image_id'], label, pred_segms[label]['category_id'])) if len(pred_labels_set) != 0: raise KeyError( 'In the image with ID {} the following segment IDs {} are presented in JSON and not presented in PNG.' .format(gt_ann['image_id'], list(pred_labels_set))) # confusion matrix calculation pan_gt_pred = pan_gt.astype(np.uint64) * OFFSET + pan_pred.astype( np.uint64) gt_pred_map = {} labels, labels_cnt = np.unique(pan_gt_pred, return_counts=True) for label, intersection in zip(labels, labels_cnt): gt_id = label // OFFSET pred_id = label % OFFSET gt_pred_map[(gt_id, pred_id)] = intersection # count all matched pairs gt_matched = set() pred_matched = set() tp = 0 fp = 0 fn = 0 for label_tuple, intersection in gt_pred_map.items(): gt_label, pred_label = label_tuple if gt_label not in gt_segms: continue if pred_label not in pred_segms: continue if gt_segms[gt_label]['iscrowd'] == 1: continue if gt_segms[gt_label]['category_id'] != pred_segms[pred_label][ 'category_id']: continue union = pred_segms[pred_label]['area'] + gt_segms[gt_label][ 'area'] - intersection - gt_pred_map.get( (VOID, pred_label), 0) iou = intersection / union if iou > 0.5: pq_stat[gt_segms[gt_label]['category_id']].tp += 1 pq_stat[gt_segms[gt_label]['category_id']].iou += iou gt_matched.add(gt_label) pred_matched.add(pred_label) tp += 1 # count false positives crowd_labels_dict = {} for gt_label, gt_info in gt_segms.items(): if gt_label in gt_matched: continue # crowd segments are ignored if gt_info['iscrowd'] == 1: crowd_labels_dict[gt_info['category_id']] = gt_label continue pq_stat[gt_info['category_id']].fn += 1 fn += 1 # count false positives for pred_label, pred_info in pred_segms.items(): if pred_label in pred_matched: continue # intersection of the segment with VOID intersection = gt_pred_map.get((VOID, pred_label), 0) # plus intersection with corresponding CROWD region if it exists if pred_info['category_id'] in crowd_labels_dict: intersection += gt_pred_map.get( (crowd_labels_dict[pred_info['category_id']], pred_label), 0) # predicted segment is ignored if more than half of the segment correspond to VOID and CROWD regions if intersection / pred_info['area'] > 0.5: continue pq_stat[pred_info['category_id']].fp += 1 fp += 1 # logger.info('Compute pq -> Core: {}, all {} images processed'.format(proc_id, len(gt_jsons_set))) return pq_stat
def _converter_2ch_single_core(proc_id, pan_2ch_set, color_gererator): sys.path.insert( 0, os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..', 'lib', 'dataset_devkit')) from panopticapi.utils import rgb2id OFFSET = 1000 VOID = 255 annotations, pan_all = [], [] for idx in range(len(pan_2ch_set)): pan_2ch = np.uint32(pan_2ch_set[idx]) pan = OFFSET * pan_2ch[:, :, 0] + pan_2ch[:, :, 1] pan_format = np.zeros((pan_2ch.shape[0], pan_2ch.shape[1], 3), dtype=np.uint8) l = np.unique(pan) segm_info = [] for el in l: sem = el // OFFSET if sem == VOID: continue mask = pan == el if vis_panoptic: color = color_gererator.categories[sem]['color'] else: color = color_gererator.get_color(sem) pan_format[mask] = color index = np.where(mask) x = index[1].min() y = index[0].min() width = index[1].max() - x height = index[0].max() - y segm_info.append({ "category_id": sem.item(), "iscrowd": 0, "id": int(rgb2id(color)), "bbox": [x.item(), y.item(), width.item(), height.item()], "area": mask.sum().item() }) annotations.append({"segments_info": segm_info}) if vis_panoptic: pan_format = Image.fromarray(pan_format) draw = ImageDraw.Draw(pan_format) for el in l: sem = el // OFFSET if sem == VOID: continue if color_gererator.categories[sem][ 'isthing'] and el % OFFSET != 0: mask = ((pan == el) * 255).astype(np.uint8) _, contour, _ = cv2.findContours( mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) for c in contour: c = c.reshape(-1).tolist() if len(c) < 4: print('warning: invalid contour') continue draw.line(c, fill='white', width=2) pan_format = np.array(pan_format) pan_all.append(pan_format) return annotations, pan_all
def process(self, val): if self.processNumber(val): self.trace(self.__format.format(np.uint32(self.value))) return True else: return False
def img_spikes_from_to(path, num_neurons, start_file_idx, end_file_idx, on_time_ms, off_time_ms, start_time, delete_before=0, ext='txt.bz2', noise=True, noise_prob=0.): start = start_file_idx end = end_file_idx spikes = [] spk_files = glob.glob(os.path.join(path, "*.%s" % (ext))) spk_files.sort() # print(path) # print(len(spk_files)) f = None spks = [[] for i in range(num_neurons)] if len(spk_files) == 0: raise Exception("Unable to locate files in dir:\n\t\t%s" % path) t = float(start_time) max_t = start_time for fname in spk_files[start:end]: # print(fname) # spks[:] = [ [] for i in range(num_neurons) ] n_lines = file_len(fname, compressed=True) # f = open(fname, 'r') f = bz2.BZ2File(fname, 'rb') line_n = 2 np.random.seed() for line in f: vals = line.split(' ') nrn_id, spk_time = np.uint32(float(vals[0])), int(float(vals[1])) line_n += 1 rand_dt = np.random.randint(-2, 3) #[-2, -1, 0, 1, 2] or [..., 3) # print("id = %s, t = %s"%(vals[0], vals[1])) if nrn_id > num_neurons: raise Exception( "Neuron Id from file is greater than number of " "neurons given in the argument (%d > %d)" % (nrn_id, num_neurons)) if noise: np.random.seed() dice_roll = np.random.uniform(0., 1.) if dice_roll <= noise_prob: continue rspk_time = spk_time + rand_dt if rspk_time <= delete_before: continue rspk_time += t rspk_time -= delete_before if rspk_time < 0: continue sys.stdout.write( "\r%05.2f%%\tbase t %06d\tneuron %06d\ttime %06d\tmax t %06d" % (100. * float(line_n) / n_lines, t, nrn_id, rspk_time, on_time_ms)) sys.stdout.flush() if rspk_time > on_time_ms: break if rspk_time in spks[nrn_id]: continue spks[nrn_id].append(rspk_time) if rspk_time > max_t: max_t = rspk_time f.close() # print(fname, t) # t += on_time_ms + off_time_ms t = max_t t += off_time_ms # spikes.append(spks) sys.stdout.write("\n") sys.stdout.flush() for nrn_id in range(num_neurons): nspk = len(spks[nrn_id]) if nspk == 0: continue spks[nrn_id][:] = list(set(spks[nrn_id])) # remove duplicates # random noise (spike loss) # np.random.seed(np.uint32(time.time()*(10**10))) # to_remove = np.random.choice(np.arange(nspk), size=int(nspk*0.1), # replace=False) # for i in sorted(to_remove, reverse=True): # del spks[nrn_id][i] spks[nrn_id].sort() print() return spks
def qc_fpga_task(fpga_trials, alf_trials): """ :fpga_task is the dictionary output of ibllib.io.extractors.ephys_fpga.extract_behaviour_sync : bpod_trials is the dictionary output of ibllib.io.extractors.ephys_trials.extract_all : alf_trials is the ALF _ibl_trials object after extraction (alf.io.load_object) :return: qc_session, qc_trials, True means QC passes while False indicates a failure """ GOCUE_STIMON_DELAY = 0.01 # -> 0.1 FEEDBACK_STIMFREEZE_DELAY = 0.01 # -> 0.1 VALVE_STIM_OFF_DELAY = 1 VALVE_STIM_OFF_JITTER = 0.1 ITI_IN_STIM_OFF_JITTER = 0.1 ERROR_STIM_OFF_DELAY = 2 ERROR_STIM_OFF_JITTER = 0.1 RESPONSE_FEEDBACK_DELAY = 0.0005 def strictly_after(t0, t1, threshold): """ returns isafter, iswithinthreshold""" return (t1 - t0) > 0, np.abs((t1 - t0)) <= threshold ntrials = fpga_trials['stimOn_times'].size qc_trials = Bunch({}) """ First Check consistency of the dataset: whithin each trial, all events happen after trial start should not be NaNs and increasing. This is not a QC but an assertion. """ status = True for k in [ 'response_times', 'stimOn_times', 'response_times', 'goCueTrigger_times', 'goCue_times', 'feedback_times' ]: if k.endswith('_bpod'): tstart = alf_trials['intervals_bpod'][:, 0] else: tstart = alf_trials['intervals'][:, 0] selection = ~np.isnan(alf_trials[k]) status &= np.all(alf_trials[k][selection] - tstart[selection] > 0) status &= np.all(np.diff(alf_trials[k][selection]) > 0) assert status """ This part of the function uses only fpga_trials information """ # check number of feedbacks: should always be one qc_trials['n_feedback'] = ( np.uint32(~np.isnan(fpga_trials['valveOpen_times'])) + np.uint32(~np.isnan(fpga_trials['errorCue_times']))) # check for non-Nans qc_trials['stimOn_times_nan'] = ~np.isnan(fpga_trials['stimOn_times']) qc_trials['goCue_times_nan'] = ~np.isnan(fpga_trials['goCue_times']) # stimOn before goCue qc_trials['stimOn_times_before_goCue_times'], qc_trials['stimOn_times_goCue_times_delay'] =\ strictly_after(fpga_trials['stimOn_times'], fpga_trials['goCue_times'], GOCUE_STIMON_DELAY) # stimFreeze before feedback qc_trials['stim_freeze_before_feedback'], qc_trials['stim_freeze_feedback_delay'] = \ strictly_after(fpga_trials['stimFreeze_times'], fpga_trials['feedback_times'], FEEDBACK_STIMFREEZE_DELAY) # stimOff 1 sec after valve, with 0.1 as acceptable jitter qc_trials['stimOff_delay_valve'] = np.less( np.abs(fpga_trials['stimOff_times'] - fpga_trials['valveOpen_times'] - VALVE_STIM_OFF_DELAY), VALVE_STIM_OFF_JITTER, out=np.ones(ntrials, dtype=np.bool), where=~np.isnan(fpga_trials['valveOpen_times'])) # iti_in whithin 0.01 sec of stimOff qc_trials['iti_in_delay_stim_off'] = \ np.abs(fpga_trials['stimOff_times'] - fpga_trials['itiIn_times']) < ITI_IN_STIM_OFF_JITTER # stimOff 2 secs after errorCue_times with jitter # noise off happens 2 secs after stimm, with 0.1 as acceptable jitter qc_trials['stimOff_delay_noise'] = np.less( np.abs(fpga_trials['stimOff_times'] - fpga_trials['errorCue_times'] - ERROR_STIM_OFF_DELAY), ERROR_STIM_OFF_JITTER, out=np.ones(ntrials, dtype=np.bool), where=~np.isnan(fpga_trials['errorCue_times'])) """ This part uses only alf_trials information """ # TEST Response times (from session start) should be increasing continuously # Note: RT are not durations but time stamps from session start # 1. check for non-Nans qc_trials['response_times_nan'] = ~np.isnan(alf_trials['response_times']) # 2. check for positive increase qc_trials['response_times_increase'] = \ np.diff(np.append([0], alf_trials['response_times'])) > 0 # TEST Response times (from goCue) should be positive qc_trials['response_times_goCue_times_diff'] = \ alf_trials['response_times'] - alf_trials['goCue_times'] > 0 # TEST 1. Response_times should be before feedback qc_trials['response_before_feedback'] = \ alf_trials['feedback_times'] - alf_trials['response_times'] > 0 # 2. Delay between wheel reaches threshold (response time) and # feedback is 100us, acceptable jitter 500 us qc_trials['response_feedback_delay'] = \ alf_trials['feedback_times'] - alf_trials['response_times'] < RESPONSE_FEEDBACK_DELAY # Test output at session level qc_session = {k: np.all(qc_trials[k]) for k in qc_trials} return qc_session, qc_trials
da.arange(10, chunks=-1, whatsthis=1) assert da.arange(10).chunks == ((10, ), ) @pytest.mark.parametrize( "start,stop,step,dtype", [ (0, 1, 1, None), # int64 (1.5, 2, 1, None), # float64 (1, 2.5, 1, None), # float64 (1, 2, 0.5, None), # float64 (np.float32(1), np.float32(2), np.float32(1), None), # promoted to float64 (np.int32(1), np.int32(2), np.int32(1), None), # promoted to int64 (np.uint32(1), np.uint32(2), np.uint32(1), None), # promoted to int64 (np.uint64(1), np.uint64(2), np.uint64(1), None), # promoted to float64 (np.uint32(1), np.uint32(2), np.uint32(1), np.uint32), (np.uint64(1), np.uint64(2), np.uint64(1), np.uint64), # numpy.arange gives unexpected results # https://github.com/numpy/numpy/issues/11505 # (1j, 2, 1, None), # (1, 2j, 1, None), # (1, 2, 1j, None), # (1+2j, 2+3j, 1+.1j, None), ], ) def test_arange_dtypes(start, stop, step, dtype): a_np = np.arange(start, stop, step, dtype=dtype) a_da = da.arange(start, stop, step, dtype=dtype, chunks=-1)
def get_mydata_correl(data_pars): state_num = data_pars['state_num'] time_len = data_pars['time_len'] signal_dimension = data_pars['signal_dimension'] CNR = data_pars['CNR'] window_len = data_pars['window_len'] half_window_len = data_pars['half_window_len'] a = np.ones(shape=(state_num, state_num)) alpha = np.ones(10) * 10 alpha[5:] = 1 base_prob = np.random.dirichlet(alpha) * 0.1 for t in range(state_num): a[t, :] = base_prob a[t, t] += 0.9 # simulate states state = np.zeros(time_len, dtype=np.uint8) p = np.random.uniform() state[0] = np.floor(p * state_num) for t in range(0, time_len - 1): p = np.random.uniform() for s in range(state_num): if (p <= np.sum(a[state[t], :s + 1])): state[t + 1] = s break freq = np.zeros(state_num) for t in range(state_num): freq[t] = np.sum(state == t) loading = np.random.randint(-1, 2, size=(state_num, signal_dimension)) cov = np.zeros((state_num, signal_dimension, signal_dimension)) for t in range(state_num): cov[t, :, :] = np.matmul(np.transpose([loading[t, :]]), [loading[t, :]]) # generate BOLD signal signal = np.zeros((time_len, signal_dimension)) for t in range(0, time_len): signal[t, :] = np.random.multivariate_normal( np.zeros((signal_dimension)), cov[state[t], :, :]) signal += np.random.normal(size=signal.shape) / CNR original_dim = np.uint32(signal_dimension * (signal_dimension - 1) / 2) x_train = np.zeros(shape=(time_len - window_len * 2, np.uint32(original_dim))) sum_corr = np.zeros(shape=(state_num, original_dim)) occupancy = np.zeros(state_num) for t in range(window_len, time_len - window_len): corr_matrix = np.corrcoef( np.transpose(signal[t - half_window_len:t + half_window_len + 1, :])) upper = corr_matrix[np.triu_indices(signal_dimension, k=1)] x_train[t - window_len, :] = np.squeeze(upper) if (np.sum(state[t - half_window_len:t + half_window_len + 1] == state[t]) == window_len): sum_corr[state[t], :] += x_train[t - window_len, :] occupancy[state[t]] += 1 return x_train
n_workers = (cat.size[0] * cat.size[1], ) prg.sep_channels(queue, n_workers, None, pix_buf, r_buf, g_buf, b_buf) wgs = cl.Kernel(prg, 'blur_channel').get_work_group_info( cl.kernel_work_group_info.WORK_GROUP_SIZE, ctx.get_info(cl.context_info.DEVICES)[0]) n_local = (16, 12) if n_local[0] * n_local[1] > wgs: print "Reduce the n_local variable size please!" nn_buf = cl.LocalMemory((n_local[0] + 2) * (n_local[1] + 2)) n_workers = (cat.size[0], cat.size[1]) prg.blur_channel(queue, n_workers, n_local, r_buf, rb_buf, nn_buf, np.uint32(cat.size[0]), np.uint32(cat.size[1])) prg.blur_channel(queue, n_workers, n_local, g_buf, gb_buf, nn_buf, np.uint32(cat.size[0]), np.uint32(cat.size[1])) prg.blur_channel(queue, n_workers, n_local, b_buf, bb_buf, nn_buf, np.uint32(cat.size[0]), np.uint32(cat.size[1])) #show_single_buffer(queue, n_pix, r_buf) show_single_buffer(queue, n_pix, rb_buf) n_workers = (cat.size[0] * cat.size[1], ) prg.mer_channels(queue, n_workers, None, pixb_buf, rb_buf, gb_buf, bb_buf) cl.enqueue_copy(queue, result, pixb_buf) im_data = [(p[0], p[1], p[2], p[3]) for p in result] cat.putdata(im_data)
ind=ind+h2 h=4 h2=h*h static_plain=True if (static_plain): s_plain = "HELLO WORLD!!!!!!!!!" size_mesg = len(s_plain) else: size_mesg=36 lenH=np.uint32((size_mesg+h2-1)/h2) print(lenH) rp=1 seed=123 #np.uint32(time.time()) #seed=xorshift32(seed) #print(seed) #seed=xorshift32(seed) #print(seed) #sys.exit(0) DK = np.empty([LSC_SKEY],dtype=np.uint8) Nonce = np.empty([LSC_SKEY],dtype=np.uint8) sc = np.empty([256],dtype=np.uint8) PboxRM = np.empty([h2],dtype=np.uint8)
'''chroma.bvh: Bounding Volume Hierarchy generation and manipulation.''' import numpy as np from pycuda.gpuarray import vec uint4 = vec.uint4 # pylint: disable-msg=C0103,E1101 CHILD_BITS = 28 NCHILD_MASK = np.uint32(0xFFFF << CHILD_BITS) def unpack_nodes(nodes): '''Creates a numpy record array with the contents of nodes array unpacked into separate fields. ``nodes``: ndarray(shape=n, dtype=uint4) BVH node array in the packed x,y,z,w format. Returns ndarray(shape=n, dtype=[('xlo', np.uint16), ('xhi', np.uint16), ('ylo', np.uint16), ('yhi', np.uint16), ('zlo', np.uint16), ('zhi', np.uint16), ('child', np.uint32), ('nchild', np.uint16)]) ''' unpacked_dtype = np.dtype([('xlo', np.uint16), ('xhi', np.uint16), ('ylo', np.uint16), ('yhi', np.uint16), ('zlo', np.uint16), ('zhi', np.uint16), ('child', np.uint32), ('nchild', np.uint16)]) unpacked = np.empty(shape=len(nodes), dtype=unpacked_dtype) for axis in ['x', 'y', 'z']: unpacked[axis + 'lo'] = nodes[axis] & 0xFFFF
def feature_vecs_DOC_W2V(train_pos, train_neg, test_pos, test_neg): """ Returns the feature vectors for all text in the train and test datasets. """ # Load the pre-trained word2vec model word2vec_model = word2vec.Word2Vec.load(path_to_pretrained_w2v) # Doc2Vec requires TaggedDocument objects as input. # Turn the datasets from lists of words to lists of TaggedDocument objects. labeled_train_pos = [ TaggedDocument(words, ["TRAIN_POS_" + str(i)]) for i, words in enumerate(train_pos) ] labeled_train_neg = [ TaggedDocument(words, ["TRAIN_NEG_" + str(i)]) for i, words in enumerate(train_neg) ] labeled_test_pos = [ TaggedDocument(words, ["TEST_POS_" + str(i)]) for i, words in enumerate(test_pos) ] labeled_test_neg = [ TaggedDocument(words, ["TEST_NEG_" + str(i)]) for i, words in enumerate(test_neg) ] sentences = labeled_train_pos + labeled_train_neg + labeled_test_pos + labeled_test_neg # Use modified doc2vec codes for applying the pre-trained word2vec model model = doc2vec_modified.Doc2Vec(dm=0, dm_mean=1, alpha=0.025, min_alpha=0.0001, min_count=1, size=1000, hs=1, workers=4, train_words=False, train_lbls=True) model.reset_weights() # Copy wiki word2vec model into doc2vec model model.vocab = word2vec_model.vocab model.syn0 = word2vec_model.syn0 model.syn1 = word2vec_model.syn1 model.index2word = word2vec_model.index2word print("# of pre-trained vocab = " + str(len(model.vocab))) # Extract sentence labels for the training and test data train_pos_labels = [ "TRAIN_POS_" + str(i) for i in range(len(labeled_train_pos)) ] train_neg_labels = [ "TRAIN_NEG_" + str(i) for i in range(len(labeled_train_neg)) ] test_pos_labels = [ "TEST_POS_" + str(i) for i in range(len(labeled_test_pos)) ] test_neg_labels = [ "TEST_NEG_" + str(i) for i in range(len(labeled_test_neg)) ] sentence_labels = train_pos_labels + train_neg_labels + test_pos_labels + test_neg_labels new_syn0 = empty((len(sentences), model.layer1_size), dtype=REAL) new_syn1 = empty((len(sentences), model.layer1_size), dtype=REAL) syn_index = 0 # Initialize and add a vector of syn0 (i.e. input vector) and syn1 (i.e. output vector) for a vector of a label for label in sentence_labels: v = model.append_label_into_vocab( label) # I made this function in the doc2vec code random.seed( uint32(model.hashfxn(model.index2word[v.index] + str(model.seed)))) new_syn0[syn_index] = (random.rand(model.layer1_size) - 0.5) / model.layer1_size new_syn1[syn_index] = zeros((1, model.layer1_size), dtype=REAL) syn_index += 1 model.syn0 = vstack([model.syn0, new_syn0]) model.syn1 = vstack([model.syn1, new_syn1]) model.precalc_sampling() # Train the model # This may take a bit to run for i in range(5): start_time = time.time() print("Training iteration %d" % (i)) random.shuffle(sentences) model.train(sentences) print("Done - Training") print("--- %s minutes ---" % ((time.time() - start_time) / 60)) start_time = time.time() # Convert "nan" values into "0" in vectors indices_nan = isnan(model.syn0) model.syn0[indices_nan] = 0.0 indices_nan = isnan(model.syn1) model.syn1[indices_nan] = 0.0 # Extract the feature vectors for the training and test data train_pos_vec = [ model.syn0[model.vocab["TRAIN_POS_" + str(i)].index] for i in range(len(labeled_train_pos)) ] train_neg_vec = [ model.syn0[model.vocab["TRAIN_NEG_" + str(i)].index] for i in range(len(labeled_train_neg)) ] test_pos_vec = [ model.syn0[model.vocab["TEST_POS_" + str(i)].index] for i in range(len(labeled_test_pos)) ] test_neg_vec = [ model.syn0[model.vocab["TEST_NEG_" + str(i)].index] for i in range(len(labeled_test_neg)) ] print("Done - Extracting the feature vectors") print("--- %s minutes ---" % ((time.time() - start_time) / 60)) # Return the four feature vectors return train_pos_vec, train_neg_vec, test_pos_vec, test_neg_vec
borealis_site_rawacf_data = OrderedDict([(str(1558583991060), { "borealis_git_hash": np.unicode_('v0.2-61-gc13ab34'), "experiment_id": np.int64(100000000), "experiment_name": np.unicode_('TestScheme9ACFs'), "experiment_comment": np.unicode_(''), "num_slices": np.int64(1), "slice_comment": np.unicode_(''), "station": np.unicode_('sas'), "num_sequences": np.int64(29), "range_sep": np.float32(44.96887), "first_range_rtt": np.float32(1200.8307), "first_range": np.float32(180.0), "rx_sample_rate": np.float64(3333.3333333333335), "scan_start_marker": np.bool_(True), "int_time": np.float32(3.000395), "tx_pulse_len": np.uint32(300), "tau_spacing": np.uint32(2400), "main_antenna_count": np.uint32(16), "intf_antenna_count": np.uint32(4), "freq": np.uint32(10500), "samples_data_type": np.unicode_('complex float'), "pulses": np.array([0, 9, 12, 20, 22, 26, 27]).astype(np.uint32), "lags": np.array([[0, 0], [26, 27], [20, 22], [9, 12], [22, 26], [22, 27], [20, 26], [20, 27], [12, 20],
def eval(self, pars): tic() _ctx, queue = card() self.res[:] = 0 cl.enqueue_copy(queue, self.res_b, self.res) radius, length, cyl_theta, cyl_phi = \ [GaussianDispersion(int(pars[base+'_pd_n']), pars[base+'_pd'], pars[base+'_pd_nsigma']) for base in GpuCylinder.PD_PARS] #Get the weights for each radius.value, radius.weight = radius.get_weights( pars['radius'], 0, 10000, True) length.value, length.weight = length.get_weights( pars['length'], 0, 10000, True) cyl_theta.value, cyl_theta.weight = cyl_theta.get_weights( pars['cyl_theta'], -np.inf, np.inf, False) cyl_phi.value, cyl_phi.weight = cyl_phi.get_weights( pars['cyl_phi'], -np.inf, np.inf, False) #Perform the computation, with all weight points sum, norm, norm_vol, vol = 0.0, 0.0, 0.0, 0.0 size = len(cyl_theta.weight) sub = pars['sldCyl'] - pars['sldSolv'] real = np.float32 if self.qx.dtype == np.dtype( 'float32') else np.float64 #Loop over radius, length, theta, phi weight points for i in xrange(len(radius.weight)): for j in xrange(len(length.weight)): vol += radius.weight[i] * length.weight[j] * pow( radius.value[i], 2) * length.value[j] norm_vol += radius.weight[i] * length.weight[j] for k in xrange(len(cyl_theta.weight)): for l in xrange(len(cyl_phi.weight)): self.prg.CylinderKernel(queue, self.qx.shape, None, self.qx_b, self.qy_b, self.res_b, real(sub), real(radius.value[i]), real(length.value[j]), real(pars['scale']), real(radius.weight[i]), real(length.weight[j]), real(cyl_theta.weight[k]), real(cyl_phi.weight[l]), real(cyl_theta.value[k]), real(cyl_phi.value[l]), np.uint32(self.qx.size), np.uint32(size)) norm += radius.weight[i] * length.weight[ j] * cyl_theta.weight[k] * cyl_phi.weight[l] # if size > 1: # norm /= math.asin(1.0) cl.enqueue_copy(queue, self.res, self.res_b) sum = self.res if vol != 0.0 and norm_vol != 0.0: sum *= norm_vol / vol print toc() * 1000, self.qx.shape[0] return sum / norm + pars['background']
def get_compute_kernels(self, runner, full_output, bulk): gpu_rho = runner.gpu_field(self.rho) gpu_phi = runner.gpu_field(self.phi) gpu_v = runner.gpu_field(self.v) gpu_map = runner.gpu_geo_map() gpu_dist1a = runner.gpu_dist(0, 0) gpu_dist1b = runner.gpu_dist(0, 1) gpu_dist2a = runner.gpu_dist(1, 0) gpu_dist2b = runner.gpu_dist(1, 1) options = 0 if full_output: options |= 1 if bulk: options |= 2 options = np.uint32(options) # Primary. args1a = ([gpu_map, gpu_dist1a, gpu_dist1b, gpu_rho, gpu_phi] + gpu_v + [options]) args1b = ([gpu_map, gpu_dist2a, gpu_dist2b, gpu_rho, gpu_phi] + gpu_v + [options]) # Secondary. args2a = ([gpu_map, gpu_dist1b, gpu_dist1a, gpu_rho, gpu_phi] + gpu_v + [options]) args2b = ([gpu_map, gpu_dist2b, gpu_dist2a, gpu_rho, gpu_phi] + gpu_v + [options]) macro_args1 = ([gpu_map, gpu_dist1a, gpu_dist2a, gpu_rho, gpu_phi] + gpu_v + [options]) macro_args2 = ([gpu_map, gpu_dist1b, gpu_dist2b, gpu_rho, gpu_phi] + gpu_v + [options]) args_a_signature = 'P' * (len(args1a) - 1) + 'i' args_b_signature = 'P' * (len(args1b) - 1) + 'i' macro_signature = 'P' * (len(macro_args1) - 1) + 'i' if runner.gpu_scratch_space is not None: macro_args1.append(runner.gpu_scratch_space) macro_args2.append(runner.gpu_scratch_space) macro_signature += 'P' args1a.append(runner.gpu_scratch_space) args2a.append(runner.gpu_scratch_space) args1b.append(runner.gpu_scratch_space) args2b.append(runner.gpu_scratch_space) args_a_signature += 'P' args_b_signature += 'P' macro = runner.get_kernel( 'ShanChenPrepareMacroFields', macro_args1, macro_signature, needs_iteration=self.config.needs_iteration_num) if self.config.access_pattern == 'AB': macro_secondary = runner.get_kernel( 'ShanChenPrepareMacroFields', macro_args2, macro_signature, needs_iteration=self.config.needs_iteration_num) macro_pair = KernelPair(macro, macro_secondary) else: macro_pair = KernelPair(macro, macro) # TODO(michalj): These kernels can actually run in parallel. primary = [ runner.get_kernel('ShanChenCollideAndPropagate0', args1a, args_a_signature, needs_iteration=self.config.needs_iteration_num), runner.get_kernel('ShanChenCollideAndPropagate1', args1b, args_b_signature, needs_iteration=self.config.needs_iteration_num) ] if self.config.access_pattern == 'AB': secondary = [ runner.get_kernel( 'ShanChenCollideAndPropagate0', args2a, args_a_signature, needs_iteration=self.config.needs_iteration_num), runner.get_kernel( 'ShanChenCollideAndPropagate1', args2b, args_b_signature, needs_iteration=self.config.needs_iteration_num) ] sim_pair = KernelPair(primary, secondary) else: sim_pair = KernelPair(primary, primary) return zip(macro_pair, sim_pair)
def write_uint32(self, value): return self.write(np.uint32(value))
# BSD 3-Clause License; see https://github.com/scikit-hep/uproot4/blob/master/LICENSE from __future__ import absolute_import import numpy # used in unmarshaling kByteCountMask = numpy.int64(0x40000000) kByteCountVMask = numpy.int64(0x4000) kClassMask = numpy.int64(0x80000000) kNewClassTag = numpy.int64(0xFFFFFFFF) kIsOnHeap = numpy.uint32(0x01000000) kIsReferenced = numpy.uint32(1 << 4) kMapOffset = 2 # not used? kNullTag = 0 kNotDeleted = numpy.uint32(0x02000000) kZombie = numpy.uint32(0x04000000) kBitMask = numpy.uint32(0x00FFFFFF) kDisplacementMask = numpy.uint32(0xFF000000) ############# core/zip/inc/Compression.h kZLIB = 1 kLZMA = 2 kOldCompressionAlgo = 3 kLZ4 = 4 kZSTD = 5
def threefry_2x32(keypair, count): """Apply the Threefry 2x32 hash. Args: keypair: a pair of 32bit unsigned integers used for the key. count: an array of dtype uint32 used for the counts. Returns: An array of dtype uint32 with the same shape as `count`. """ # Based on ThreeFry2x32 by phawkins@ in //.../xla/client/lib/prng.cc key1, key2 = keypair if not lax._dtype(key1) == lax._dtype(key2) == lax._dtype( count) == onp.uint32: msg = "threefry_2x32 requires uint32 arguments, got {}" raise TypeError( msg.format([lax._dtype(x) for x in [key1, key2, count]])) rotate_left = _make_rotate_left(lax._dtype(count)) def apply_round(v, rot): v = v[:] v[0] = v[0] + v[1] v[1] = rotate_left(v[1], rot) v[1] = v[0] ^ v[1] return v odd_size = count.size % 2 if odd_size: x = list(np.split(np.concatenate([count.ravel(), onp.uint32([0])]), 2)) else: x = list(np.split(count.ravel(), 2)) rotations = [13, 15, 26, 6, 17, 29, 16, 24] ks = [key1, key2, key1 ^ key2 ^ onp.uint32(0x1BD11BDA)] x[0] = x[0] + ks[0] x[1] = x[1] + ks[1] for r in rotations[:4]: x = apply_round(x, r) x[0] = x[0] + ks[1] x[1] = x[1] + ks[2] + onp.uint32(1) for r in rotations[4:]: x = apply_round(x, r) x[0] = x[0] + ks[2] x[1] = x[1] + ks[0] + onp.uint32(2) for r in rotations[:4]: x = apply_round(x, r) x[0] = x[0] + ks[0] x[1] = x[1] + ks[1] + onp.uint32(3) for r in rotations[4:]: x = apply_round(x, r) x[0] = x[0] + ks[1] x[1] = x[1] + ks[2] + onp.uint32(4) for r in rotations[:4]: x = apply_round(x, r) x[0] = x[0] + ks[2] x[1] = x[1] + ks[0] + onp.uint32(5) out = np.concatenate(x) assert out.dtype == onp.uint32 return lax.reshape(out[:-1] if odd_size else out, count.shape)
plt.rc('text', usetex=True) plt.rc('font', **font) # family='serif') # params = {'text.latex.preamble': ['\usepackage{upgreek}']} # plt.rcParams.update(params) plt.rc ############################### ### data in nam_data = 'Sedimentation_velocities.dat' ############################### ### load data data_In = np.loadtxt(nam_data) n_meas_tot = np.uint32(data_In[:,0]) v_front = data_In[:,1] ## front velocity uv_front = data_In[:,2] v_xdfa = data_In[:,3] uv_xdfa = data_In[:,4] ###### list of pumprates pumprate = [750, 700, 650, 600, 550, 500, 450, 400, 350] N_rate = len(pumprate) N_meas = [3, 3, 3, 3, 3, 3, 3, 3, 3] ### number of measurements per pump rate ############################## ### plot attenuation vs z_tilde
def get_compute_kernels(self, runner, full_output, bulk): gpu_rho = runner.gpu_field(self.rho) gpu_phi = runner.gpu_field(self.phi) gpu_lap = runner.gpu_field(self.phi_laplacian) gpu_v = runner.gpu_field(self.v) gpu_map = runner.gpu_geo_map() gpu_dist1a = runner.gpu_dist(0, 0) gpu_dist1b = runner.gpu_dist(0, 1) gpu_dist2a = runner.gpu_dist(1, 0) gpu_dist2b = runner.gpu_dist(1, 1) options = 0 if full_output: options |= 1 if bulk: options |= 2 if hasattr( self, '_force_term_for_eq') and self._force_term_for_eq.get(1) == 0: phi_args = [gpu_rho, gpu_phi] else: phi_args = [gpu_phi] options = np.uint32(options) # Primary. args1a = ([gpu_map, gpu_dist1a, gpu_dist1b, gpu_rho, gpu_phi] + gpu_v + [gpu_lap, options]) args1b = ([gpu_map, gpu_dist2a, gpu_dist2b] + phi_args + gpu_v + [gpu_lap, options]) # Secondary. args2a = ([gpu_map, gpu_dist1b, gpu_dist1a, gpu_rho, gpu_phi] + gpu_v + [gpu_lap, options]) args2b = ([gpu_map, gpu_dist2b, gpu_dist2a] + phi_args + gpu_v + [gpu_lap, options]) macro_args1 = [ gpu_map, gpu_dist1a, gpu_dist2a, gpu_rho, gpu_phi, options ] macro_args2 = [ gpu_map, gpu_dist1b, gpu_dist2b, gpu_rho, gpu_phi, options ] args_a_signature = 'P' * (len(args1a) - 1) + 'i' args_b_signature = 'P' * (len(args1b) - 1) + 'i' macro_signature = 'P' * (len(macro_args1) - 1) + 'i' if runner.gpu_scratch_space is not None: macro_args1.append(runner.gpu_scratch_space) macro_args2.append(runner.gpu_scratch_space) macro_signature += 'P' args1a.append(runner.gpu_scratch_space) args2a.append(runner.gpu_scratch_space) args1b.append(runner.gpu_scratch_space) args2b.append(runner.gpu_scratch_space) args_a_signature += 'P' args_b_signature += 'P' macro = runner.get_kernel( 'FreeEnergyPrepareMacroFields', macro_args1, macro_signature, needs_iteration=self.config.needs_iteration_num) if self.config.access_pattern == 'AB': macro_secondary = runner.get_kernel( 'FreeEnergyPrepareMacroFields', macro_args2, macro_signature, needs_iteration=self.config.needs_iteration_num) macro_pair = KernelPair(macro, macro_secondary) else: macro_pair = KernelPair(macro, macro) # Note: these two kernels need to be executed in order. primary = [ runner.get_kernel('FreeEnergyCollideAndPropagateFluid', args1a, args_a_signature, needs_iteration=self.config.needs_iteration_num), runner.get_kernel('FreeEnergyCollideAndPropagateOrderParam', args1b, args_b_signature, needs_iteration=self.config.needs_iteration_num) ] if self.config.access_pattern == 'AB': secondary = [ runner.get_kernel( 'FreeEnergyCollideAndPropagateFluid', args2a, args_a_signature, needs_iteration=self.config.needs_iteration_num), runner.get_kernel( 'FreeEnergyCollideAndPropagateOrderParam', args2b, args_b_signature, needs_iteration=self.config.needs_iteration_num) ] sim_pair = KernelPair(primary, secondary) else: sim_pair = KernelPair(primary, primary) return zip(macro_pair, sim_pair)
#print("RM1") #print(RM1) rc4keyperm(DK[LSC_SKEY/4:2*LSC_SKEY/4], h2, rp, PboxRM, LSC_SKEY/4); #print("PboxRM") #print(PboxRM) rc4key(DK[2*LSC_SKEY/4:3*LSC_SKEY/4], Sbox1, LSC_SKEY/4); rc4key(DK[3*LSC_SKEY/4:LSC_SKEY], Sbox2, LSC_SKEY/4); #print("Sbox1") #print(Sbox1) #print("Sbox2") #print(Sbox2) RM2=np.copy(RM1) RMorig=np.copy(RM1) myrand=np.uint32(0) for i in range(min(LSC_SKEY,32)): myrand=myrand|(DK[i]&1); myrand=np.uint32(myrand<<1) # with following setting: # # LSC_DETERMINISTIC=True # LSC_SKEY=16 # LSC_STATIC_KEY=True # LSC_WMIC=True # LSC_MICv=2 # # when fcount=0 #
def extract_data(oname, coords, obsname, T0, period, inst, SDSS, comp_mags=None, myLoc='.', fnames=None, lower_phase=-0.5, upper_phase=0.5, no_calibration=False): ''' Takes a set of *CAM observations (and data on the system), and produces a set of phase folded lightcurves. If we're in the SDSS field, each .log file needs a corresponding .coords file that contains the RA and Dec of each aperture: <CCD1 ap1 RA> <Dec> <CCD1 ap2 RA> <Dec> <CCD2 ap1 RA> <Dec> <CCD2 ap2 RA> <Dec> <CCD3 ap1 RA> <Dec> <CCD3 ap2 RA> <Dec> <CCD3 ap3 RA> <Dec> If not, I need a standard star reduction, and each .log file needs a corresponding .log reduction that uses the same parameters to ensure an accurate match. These should be specified in comp_fnames. If none are supplied, try searching for a Arguments: ---------- oname: str Template for written files. Applied to coords: str RA, Dec of target. Must in in a format astropy can understand. obsname: str Observatory location. See astropy for a list of valid names T0: float Ephemeris data period: float Ephemeris data SDSS: bool, optional If True, I'll do an SDSS lookup for the comparison star magnitudes. If False, use a standard star to calibrate myLoc: str, optional Working directory. If not supplied, default to current working directory fnames: list, optional List of target reduction files. If not supplied, searches for log files Returns: -------- written_files: list List of created .calib files. ''' printer("\n\n# # # # # # # # # # # # # # # # # # # BEGIN BATCH CALIBRATION # # # # # # # # # # # # # # # # # # #\n\n") # Writing out lc_dir = os.path.join(myLoc, 'MCMC_LIGHTCURVES') try: os.mkdir(lc_dir) except: pass printer("Lightcurves will go in: {}".format(lc_dir)) figs_dir = os.path.join(myLoc, 'MCMC_LIGHTCURVES', "FIGS") try: os.mkdir(figs_dir) except: pass printer("Figures will go in: {}".format(figs_dir)) # Report the things we're working with printer(" Using these log files: ") for i, fname in enumerate(fnames): printer(" {:2d} - {}".format(i, fname)) printer(' ') printer(" I'll write out to {}*\n".format(oname)) #Correct to BMJD printer(" Correcting observations from MJD to BMJD (observed from '{}')".format(obsname)) printer(" Phase folding data for a T0: {:}, period: {:}".format(T0, period)) # Data masking stuff FLAG = np.uint32(0) for f in FLAGS_TO_IGNORE: FLAG = FLAG | f if FLAG: printer(" Ignoring bad data flags: {}".format(FLAGS_TO_IGNORE)) printer("List of keys:") printer(hcam.FLAGS) # Where are we? try: observatory = coord.EarthLocation.of_site(obsname) except: lat, lon = obsname.split(',') printer(" Attempting to get the earth observatory from latitude and longitude") observatory = coord.EarthLocation.from_geodetic(lat=lat, lon=lon) star_loc = coord.SkyCoord( coords, unit=(u.hourangle, u.deg), frame='icrs' ) # I want to know what instrument I'm using, since each has a different number of cameras inst = inst.lower() if inst == 'uspec': nCCD = 1 bands = ['???'] c = ['black'] elif inst == 'ucam': nCCD = 3 bands = ['r', 'g', 'u'] c = ['red', 'green', 'blue'] elif inst == 'hcam': nCCD = 5 bands = ['u', 'g', 'r', 'i', 'z'] c = ['blue', 'green', 'red', 'magenta', 'black'] printer(" I'm using the instrument {}, which has {} CCDS in the following order:".format(inst, nCCD)) for n, col in zip(range(nCCD), c): printer(" -> CCD {}: plotted in {}".format(n+1, col)) written_files = [] # Plotting # ADU_lightcurves = {fname: [] for fname in fnames} print("Making plotting area...", end='') plt.ion() fig, ax = plt.subplots(nCCD, figsize=[11.69, 8.27], sharex=True) # If we only have one CCD, axes still need to be a lists if nCCD == 1: ax = [ax] twinAx = [] for i, a in enumerate(ax): a.set_ylabel('Flux, mJy') twinAx.append(a.twinx()) twinAx[i].set_ylabel('Count Ratio') twinAx[i].yaxis.tick_right() ax[-1].set_xlabel('Phase, days') ax[0].set_title('Waiting for data...') fig.tight_layout() compFig, compAx = plt.subplots(nCCD, figsize=[11.69,8.27], sharex=True) # If we only have one CCD, axes still need to be a lists if nCCD == 1: compAx = [compAx] compFig.tight_layout() plt.show() print(" Done!") # I want a master pdf file with all the nights' lightcurves plotted pdfname = os.path.join(figs_dir, oname+"_all_nights.pdf") with PdfPages(pdfname) as pdf: for fname in fnames: printer("\n----------------------------------------------------------------\n----------------------------------------------------------------\n") printer("Calibrating lightcurves for {}".format(fname)) printer("\n----------------------------------------------------------------\n----------------------------------------------------------------\n") print("CWD: {}".format(os.getcwd())) print("File: {}".format(fname)) data = hcam.hlog.Hlog.read(fname) if data == {}: data = hcam.hlog.Hlog.rulog(fname) if data == {}: data = hcam.hlog.Hlog.rfits(fname) if data == {}: raise Exception("Could not properly read in log file, {}".format(fname)) printer(" Read the data file!") # Get the apertures of this data set aps = data.apnames CCDs = [str(i) for i in aps] CCDs = sorted(CCDs) if CCDs == []: printer("ERROR! No data in the file!") printer(" The observations have the following CCDs: {}".format([int(ccd) for ccd in CCDs])) printer(" Am I flux calibrating the data? {}".format(not no_calibration)) if no_calibration: printer("\n!!! Not doing flux calibration! Setting reference magnitudes to correspond to a flux=1\n\n") # Reference stars is a dict, keyed by the CCD string reference_stars = {} for CCD in CCDs: mags = [] # The comparison star fluxes get added together, so each should have an even share of 1 mJy. individual_flx = 1.0 / len(aps[CCD][1:]) for ap in aps[CCD][1:]: mags.append(sdss_flux2mag(individual_flx)) reference_stars[CCD] = np.array(mags) printer("'Unit' Reference stars have a magnitude of {:.2f}\n".format(sdss_flux2mag(1.0))) elif SDSS: printer(" Looking up SDSS magnitudes from the database") comparison_coord_files = comp_mags[fname] reference_stars = construct_reference(comparison_coord_files) else: reference_stars = {} comparisons = comp_mags[fname] bands = list(comparisons.keys()) printer(" For each of these CCDs, I've been given comparison stars of the following magnitudes:") for i, (b, comps) in enumerate(comparisons.items()): # I need to capture 'none' strings here, and store them as np.nans. # Later, when I construct the comparison star, these apertures must be ignored! comparison_list = [] for comp in comps: try: # Can I float? (we all float down here, georgie...) comp = float(comp) comparison_list.append(comp) except: # If I can't, Ignore me. comparison_list.append(np.nan) # Who doesnt love vectorised calculations? reference_stars[str(i+1)] = np.array(comparison_list) printer(" My comparison stars have the following apparent mags:") for b, mags in reference_stars.items(): printer(" - CCD{}, mags: {}".format(b, mags)) printer("\n\n") for a in ax: a.clear() a.set_ylabel('Flux, mJy') ax[-1].set_xlabel('Phase, days') ax[0].set_title('Waiting for data...') # Loop through the CCDs. # # For each CCD, grab the target lightcurve, and the comparisons for CCD in CCDs: lightcurve_metadata = '# This is data from the file {} CCD {}\n'.format(fname, CCD) CCD_int = int(CCD) - 1 printer("-> CCD {}".format(CCD)) # Plot the comparison we construct compAx[CCD_int].clear() compAx[CCD_int].set_title("CCD {}, comparison star".format(CCD)) compAx[CCD_int].set_ylabel("Counts per frame") # Get this frame's apertures ap = aps[CCD] printer(" This CCD has the apertures: {}".format(ap)) # Check that there is more than one aperture -- i.e. if a comparison star exists if len(ap) <= 1: printer("I can't do relative photometry with only one aperture!") printer("!!! Bad log file, '{}' !!!".format(fname)) raise LookupError("Not enough apertures in the log file!", fname) # Check for nans in the log files. for a in ap: star = data.tseries(CCD, a) if np.any(np.isnan(star.y)): printer("!!! Log file cannot contain nan values! File {}".format(fname)) raise ValueError("Log file cannot contain nan values!", fname) # Get some data on the """quality""" of the observations metadata = '#\n# Reduction info:\n\n' to_proc = data[CCD] ap_x = [header for header in to_proc.dtype.fields if "x_" in header] ap_y = [header for header in to_proc.dtype.fields if "y_" in header] ap_fwhm = [header for header in to_proc.dtype.fields if "fwhm_" in header] for x_label, y_label, fwhm_label in zip(ap_x, ap_y, ap_fwhm): x_pix_loc = to_proc[x_label].mean() y_pix_loc = to_proc[y_label].mean() fwhm_pix_loc = to_proc[fwhm_label].mean() aperture_number =x_label.replace("x_", "") metadata += '# Aperture {}\n'.format(aperture_number) metadata += "# x location: {:.0f}\n".format(x_pix_loc) metadata += "# y location: {:.0f}\n".format(y_pix_loc) metadata += "# fwhm: {:.2f}\n#\n#\n".format(fwhm_pix_loc) lightcurve_metadata += metadata # Grab the target data target = data.tseries(CCD, '1') # mags is a list of the relevant comparison star magnitudes. # For non-SDSS fields, this is the clipped mean magnitude of each object. mags = reference_stars[CCD] fluxs = sdss_mag2flux(mags) sumFlux = np.nansum(fluxs) sumMag = sdss_flux2mag(sumFlux) printer(" Comparison star mags: {}".format(mags)) if no_calibration: lightcurve_metadata += "# No flux calibration being done!!\n" lightcurve_metadata += "# simulated a dummy comparison magnitude of 1.00 mJy\n" else: lightcurve_metadata += "# Comparison star mags: {}\n".format(mags) # Add up the reference star fluxes N_comparisons = 0 comparison = "Dummy initialiser ( ͡° ͜ʖ ͡°)" count_ratios = [] for a, mag in zip(ap[1:], mags): if np.isnan(mag): printer(" The reference star in ap {} is being ignored!".format(a)) else: N_comparisons += 1 new_comparison = data.tseries(CCD, a) r = sdss_mag2flux(mag) / new_comparison.y.mean() r_err = sdss_mag2flux(mag) / new_comparison.y.std() try: comparison = comparison + new_comparison printer(" The reference star now includes data from aperture {}".format(a)) except: comparison = data.tseries(CCD, a) printer(" The comparison was initialised with aperture {}".format(a)) printer(" This star has mean count/frame of {:.3f}".format(new_comparison.y.mean())) printer(" and has a flux/count of {:.3g} +/- {:.3g} mJy/count".format(r, r_err)) printer(" The 'comparison star' I've construced from {} apertures now has a mean count/frame of {:.3f}".format(N_comparisons, np.mean(comparison.y))) # If we have SDSS stars too bright, get their mags from flux calibrating those that arent if SDSS and np.any(np.isinf(mags)): printer("\n\n I have some comparisons that saturated SDSS! Inferring their magnitudes from fainter stars.") lightcurve_metadata += "# Some comparison stars saturated the SDSS image.\n# Their magnitudes were inferred from fainter stars\n" printer(" Collecting fainter stars...") calibComp = None for mag, a in zip(mags, ap[1:]): if np.inf(mag): printer(" Skipping aperture {}, as it is nan".format(a)) else: if calibComp is None: calibComp = data.tseries(CCD, a) else: calibComp += data.tseries(CCD, a) if calibComp is None: raise Exception("All comparison stars saturated SDSS! Pick at least one that doesn't!") calibComp_counts = np.mean(calibComp.y) printer(" My non-saturated SDSS stars have a mean count/frame of {:.3f}".format(calibComp_counts)) lightcurve_metadata += "# My non-saturated SDSS stars have a mean count/frame of {:.3f}\n".format(calibComp_counts) printer(" My fluxes are {}".format([f for f in fluxs if not np.inf(f)])) printer(" with a sum flux of {:.3f} mJy".format(sumFlux)) printer(" and a sum mag of {:.3f} mag".format(sumMag)) lightcurve_metadata += "# My fluxes are {}\n".format([f for f in fluxs if not np.inf(f)]) lightcurve_metadata += "# with a sum flux of {:.3f} mJy\n".format(sumFlux) lightcurve_metadata += "# and a sum mag of {:.3f} mag\n".format(sumMag) for i, (mag, a) in enumerate(zip(mags, ap[1:])): if np.isinf(mag): cnts = data.tseries(CCD, a) meanCnts = np.mean(cnts.y) if np.any(np.isnan(cnts.y)): meanCnts = np.nanmean(cnts.y) printer("The file {} has nan counts! That's weird, and you should fix that.".format(fname)) printer("I'll continue ignoring the nan, BUT FIX IT!") lightcurve_metadata += "# The file {} has nan counts! That's wierd, and you should fix that.\n".format(fname) lightcurve_metadata += "# I'll continue ignoring the nan, BUT FIX IT!\n" # Calibrated against known SDSS stars. Observed through the same air column since they're the same frame, so no ext. corr. mag = sumMag - 2.5*np.log10(meanCnts/calibComp_counts) mags[i] = mag printer(" Star {} had no SDSS magnitude. Computed a magnitude of {:.3f} from an e- flux of {}".format(a, mag, meanCnts)) lightcurve_metadata += "# Star {} had no SDSS magnitude. Computed a magnitude of {:.3f} from an e- flux of {}\n".format(a, mag, meanCnts) printer("\n") # # # # # # # # # # # # # # # # # # # # # # Conversion of target lightcurve # # # # # # # # # # # # # # # # # # # # # # # Get the non-saturated fluxes fluxs = sdss_mag2flux(mags) comparison_flux = np.nansum(fluxs) ratio = target / comparison # counts / counts - ratio between target and comp printer("\n\n Correcting data to BMJD time...") ratio = tcorrect(ratio, star_loc, obsname) # If we're the first CCD, figure out what eclipse cycle we are if CCD == '1': meantime = np.mean(ratio.t) E = calc_E(meantime, T0, period) printer(" The mean time of this eclipse is {:.3f}.".format(meantime)) printer(" From ephemeris data, I get an eclipse Number,") printer(" E = ({:.3f} - [T0={:.3f}]) / [P={:.5f}]".format(meantime, T0, period)) printer(" E = {:.3f}".format(E)) E = np.rint(E) # The above can be off, if the eclipse isnt the minimum. in/decriment until it's within bounds while T0 + E*period < ratio.t[0]: printer(" !!! Eclipse time not within these data! Incrimenting E...") E += 1 while T0 + E*period > ratio.t[-1]: printer(" !!! Eclipse time not within these data! Decrimenting E...") E -= 1 printer(" I think that the eclipse spanning from {:.3f} to {:.3f} is cycle number {}".format( ratio.t[0], ratio.t[-1], E) ) eclTime = T0 + E*period printer(" The eclipse is then at time {:.3f}".format(eclTime)) printer("") # slice out the data between phase -0.5 and 0.5 printer(" Slicing out data between phase {} and {}".format(lower_phase, upper_phase)) slice_time = (ratio.t - eclTime) / period slice_args = (slice_time < upper_phase) * (slice_time > lower_phase) ratio = hcam.hlog.Tseries( slice_time[slice_args], ratio.y[slice_args], ratio.ye[slice_args], ratio.mask[slice_args] ) # Bad data has error = -1 mask = np.where(ratio.ye != -1) ratio = ratio[mask] meanRatio = np.mean(ratio.y) printer(" I sliced out {} data from the lightcurve about the eclipse.".format(len(ratio.t))) # Save the ratio for later ADU_lightcurves[fname].append(copy.deepcopy(ratio)) # Convert the ratio from ADU to mJy printer(" Multiplying the target count flux / comparison count flux by the comparison flux, {:.3f}".format(comparison_flux)) ratio = ratio * comparison_flux # Scale back up to actual flux. # Filter out flags I don't care about. ratio.mask = ratio.mask & (~ FLAG) ####################################################################################################### ############ IGNORE ME I'M BORING AND HARD TO READ. WHY READ ANYTHING HARD? JUST TRUST ME. ############ ####################################################################################################### # Reporting # lightcurve_metadata += "# I calculated an eclipse time of {} BMJD, and phase-folded around that\n".format(eclTime) lightcurve_metadata += "# with a T0 of {}, and a period of {}, making this eclipse {}.\n".format(T0, period, E) lightcurve_metadata += "# I also sliced out the phase {} -> {}!!\n#\n#\n".format(lower_phase, upper_phase) printer(" Comparison star apparent SDSS magnitudes:") lightcurve_metadata += "# Comparison star apparent SDSS magnitudes:\n" for m, mag in enumerate(mags): printer(" Star {} -> {:.3f} mag".format(m, mag)) lightcurve_metadata += "# Star {} -> {:.3f} mag\n".format(m, mag) printer("") lightcurve_metadata += "#\n#\n#\n" printer(" Apparent fluxes of the comparison stars:") lightcurve_metadata += "# Apparent fluxes of the comparison stars:\n" for i, flux in enumerate(fluxs): printer(" Star {} -> {:.3f} mJy".format(i, flux)) lightcurve_metadata += "# Star {} -> {:.3f} mJy\n".format(i, flux) lightcurve_metadata += "#\n" printer(' Sum apparent Flux: {:.3f} mJy\n'.format(comparison_flux)) printer(" Instrumental counts, summed per mean frame ({} frames) of {} comparison stars: {:.1f}".format( len(comparison.y), N_comparisons, np.mean(comparison.y) )) printer(" Instrumental counts per mean frame ({} frames) of target: {:.1f}".format( len(target.y), np.mean(target.y) )) printer(" Mean Target/comparison count ratio: {:.3f}".format(meanRatio)) printer(" Mean target magnitude: {:.3f}".format(sdss_flux2mag(meanRatio * comparison_flux))) lightcurve_metadata += '# Sum apparent Flux: {:.3f} mJy\n#\n#\n'.format(comparison_flux) lightcurve_metadata += "# Instrumental summed counts per mean frame ({} frames) of {} comparison stars: {:.1f}\n".format(len(comparison.y), len(ap[1:]), np.mean(comparison.y)) lightcurve_metadata += "# Instrumental counts per mean frame ({} frames) of target: {:.1f}\n#\n".format(len(target.y), np.mean(target.y)) lightcurve_metadata += "# Mean Target/comparison count ratio: {:.3f}\n".format(meanRatio) lightcurve_metadata += "# Mean target magnitude: {:.3f}\n".format(sdss_flux2mag(meanRatio * comparison_flux)) # Plotting management ax[CCD_int].clear() if CCD_int == 0: title = os.path.split(fname)[1] ax[0].set_title(title) compAx[0].set_title("{}\nCCD {}, comparison stars, normalised.".format(title, CCD)) ax[CCD_int].set_ylabel('Flux, mJy') # Plot the ratio ratio.mplot(ax[CCD_int], colour=c[CCD_int]) # Scale the right side labels twinAx[CCD_int].set_ylim( ax[CCD_int].get_ylim() / comparison_flux ) # Draw fig.canvas.draw_idle() compMin = 9e99 compMax = -9e99 if len(ap) == 2: # # Plot the mean count flux on the figure -- only used when single aperture, as not as useful as ratios compAx[CCD_int].errorbar(comparison.t, comparison.y, yerr=comparison.ye, label='Mean', color='black', linestyle='', marker='o', capsize=0) try: mean, _, _ = sigma_clipped_stats(comparison.y, maxiters=2, sigma=3) except: mean, _, _ = sigma_clipped_stats(comparison.y, iters=2, sigma=3) compAx[CCD_int].axhline(mean, linestyle='--', color='black') compMin = np.min(comparison.y) compMax = np.max(comparison.y) else: # Plot each combination of comparison star ratios, i.e. for 3 comparisons: 2/3, 2/4, 3/4 j = 0 for i, a in enumerate(ap[1:-1]): first = data.tseries(CCD, a) for b in ap[i+2:]: printer(" -> Plotting ap {}/{}".format(a, b)) toPlot = first / data.tseries(CCD, b) # Filter out flags I don't care about. toPlot.mask = toPlot.mask & (~ FLAG) # Apply the mask to the data if np.any(toPlot.mask): mask = np.where(toPlot.mask == 0) printer(" -> {} masked data!".format(np.sum(toPlot.mask != 0))) printer("\nMasked data:") printer(toPlot.mask) printer("\n\n") printer("Flags:") printer(hcam.FLAGS) if np.all(toPlot.mask != 0): print("ALL DATA ARE MASKED! Stopping...") exit() toPlot.t = toPlot.t[mask] toPlot.y = toPlot.y[mask] toPlot.ye = toPlot.ye[mask] toPlot.mask = toPlot.mask[mask] toPlot.y = toPlot.y / np.mean(toPlot.y) toPlot.y = toPlot.y + (j / 5) j += 1 # Get min and max axis limits if np.max(toPlot.y) > compMax: compMax = np.max(toPlot.y) if np.min(toPlot.y) < compMin: compMin = np.min(toPlot.y) # Fit a straight line to the data. Deviations indicate bad comparisons A,B = curve_fit(straight_line, toPlot.t, toPlot.y)[0] fit_X = np.linspace(toPlot.t[0], toPlot.t[-1], 3) fit_Y = straight_line(fit_X, A, B) # iters is depreciated. Try the new version, if that fails do the old version. yay, flexibility! try: mean, _, _ = sigma_clipped_stats(toPlot.y, maxiters=2, sigma=3) except: mean, _, _ = sigma_clipped_stats(toPlot.y, iters=2, sigma=3) compAx[CCD_int].axhline(mean, linestyle='--', color='black') compAx[CCD_int].plot(fit_X, fit_Y, color='red', linestyle=':') compAx[CCD_int].scatter(toPlot.t, toPlot.y, s=10, label="Aperture {}/{} - grad: {:.2f}".format(a, b, A), alpha=0.6 ) compFig.canvas.draw_idle() # Add in legend artist compAx[CCD_int].legend() pad = 0.05 * (compMax - compMin) compAx[CCD_int].set_ylim([compMin-pad, compMax+pad]) compAx[CCD_int].set_xlim([comparison.t[0], comparison.t[-1]]) # File handling stuff b = bands[CCD_int] while b == '???': b = input("What band are these data?: ") if b == '': print("PLEASE ENTER A BAND NAME for:\n{}\n".format(fname)) b = "???" date = time.Time(eclTime, format='mjd') date = date.strftime("%Y-%m-%d@%Hh%Mm") filename = oname filename = "{}_{}_{}.calib".format(filename, date, b) filename = os.path.join(lc_dir, filename) # Saving data printer(" These data have {} masked points.".format(np.sum(ratio.mask != 0))) if np.sum(ratio.mask != 0): printer("\n\n{}\n\n".format(ratio.mask)) with open(filename, 'w') as f: f.write(lightcurve_metadata) f.write("# Phase, Flux, Err_Flux\n") for t, y, ye, mask in zip(ratio.t, ratio.y, ratio.ye, ratio.mask): if not mask: f.write("{} {} {}\n".format(t, y, ye)) written_files.append(filename) printer(" Wrote data to {}".format(filename)) printer(" Finished CCD {}\n".format(CCD)) ax[-1].set_xlabel('Phase, days') x_range = [min(ratio.t), max(ratio.t)] ax[0].set_xlim(x_range) x_range = [min(comparison.t), max(comparison.t)] compAx[0].set_xlim(x_range) plt.tight_layout() plt.subplots_adjust(hspace=0.0) fig.canvas.draw_idle() compFig.canvas.draw_idle() input("\n Hit enter for next file\r") print() pdf.savefig(fig) pdf.savefig(compFig) plt.close(compFig) # Plot the ADU lightcurves for the user. for a in ax: a.clear() a.set_ylabel('Flux, ADU') ax[-1].set_xlabel('Phase, days') ax[0].set_title('Waiting for data...') for fname, lightcurves in ADU_lightcurves.items(): for i, tseries in enumerate(lightcurves): print("{} // CCD {}".format(fname, i)) flx = tseries.y phase = tseries.t ax[i].step(phase, flx, label=fname) for a in ax: a.legend() ax[0].set_title("ADU Lightcurves of all files") plt.tight_layout() fig.canvas.draw_idle() ADU_name = os.path.join(figs_dir, oname+'_ADU_lightcurves.pdf') fig.savefig(ADU_name) input("Hit enter to continue... ") printer(" ") printer(" Saved the plots to {}".format(pdfname)) plt.close('all') plt.ioff() return written_files
np.complex64(3j) np.complex64(A()) np.complex64(C()) np.complex128(3j) np.complex128(C()) np.complex128(None) np.complex64("1.2") np.complex128(b"2j") np.int8(4) np.int16(3.4) np.int32(4) np.int64(-1) np.uint8(B()) np.uint32() np.int32("1") np.int64(b"2") np.float16(A()) np.float32(16) np.float64(3.0) np.float64(None) np.float32("1") np.float16(b"2.5") if sys.version_info >= (3, 8): np.uint64(D()) np.float32(D()) np.complex64(D())
def LSC_process_pkt(lorapkt, dst, ptype, src, seq): #it seems that there is not a full reset when calling this function from a parent Python program #so we need to reset at the beginning of the function if LSC_DETERMINISTIC==True: RM1=np.copy(RMorig) RM2=np.copy(RM1) size_mesg = len(lorapkt) if LSC_WMIC: lenH=np.uint32((size_mesg+HEADER_SIZE+h2-1)/h2) else: lenH=np.uint32((size_mesg+h2-1)/h2) plain = np.empty([lenH*h2],dtype=np.uint8) cipher = np.empty([lenH*h2],dtype=np.uint8) check = np.empty([lenH*h2],dtype=np.uint8) for i in range(lenH*h2): cipher[i]=0 if LSC_WMIC: #add the packet header to compute MIC cipher[0]=dst cipher[1]=ptype cipher[2]=src cipher[3]=seq for i in range(size_mesg): cipher[HEADER_SIZE+i]=lorapkt[i] print "?LSC: received MIC: ", print (cipher[size_mesg+HEADER_SIZE-LSC_SMIC:size_mesg+HEADER_SIZE]) #encrypt received content: HEADER+CIPHER #but with fcount=seq+1 #here is use the plain buffer encrypt_ctr(cipher, plain, lenH, RM1, PboxRM, Sbox1, Sbox2, (seq+1) % 256) if LSC_MICv==1: #skip the first 4 bytes and take the next 4 bytes of encrypted HEADER+CIPHER plain[0]=plain[LSC_SMIC] plain[1]=plain[LSC_SMIC+1] plain[2]=plain[LSC_SMIC+2] plain[3]=plain[LSC_SMIC+3] elif LSC_MICv==2: #first, compute byte-sum of encrypted HEADER+CIPHER myMIC=np.sum(plain[:size_mesg+HEADER_SIZE-LSC_SMIC]) plain[0]=xorshift32(np.uint32(myMIC % 7)) plain[1]=xorshift32(np.uint32(myMIC % 13)) plain[2]=xorshift32(np.uint32(myMIC % 29)) plain[3]=xorshift32(np.uint32(myMIC % 57)) elif LSC_MICv==3: #should implement a better algorithm? #XTEA?: http://code.activestate.com/recipes/496737-python-xtea-encryption/ print "todo" print "?LSC: computed MIC: ", print (plain[:LSC_SMIC]) if np.array_equal(plain[:LSC_SMIC], cipher[size_mesg+HEADER_SIZE-LSC_SMIC:size_mesg+HEADER_SIZE]): print "?LSC: valid MIC" #print "?LSC: [cipher]: ", #print (cipher[HEADER_SIZE:size_mesg+HEADER_SIZE-LSC_SMIC]) #re-index cipher data to get rid of HEADER and MIC size_mesg -= LSC_SMIC for i in range(size_mesg): cipher[i]=lorapkt[i] else: return "###BADMIC###" else: for i in range(size_mesg): cipher[i]=lorapkt[i] #notice the usage of RM2 to decrypt as RM1 has changed encrypt_ctr(cipher, check, lenH, RM2, PboxRM, Sbox1, Sbox2, seq) #print "?LSC: [plain]: ", #print (check) s_plain = check.tostring() print "?LSC: plain payload is "+replchars.sub(replchars_to_hex, s_plain[:size_mesg]) return s_plain[:size_mesg]
def generateHeader(self, data, inShape): #data["values"] can be one of 3 shapes: dense 4d mat for activity, dense 6d mat for weights #scipy csr_sparse matrix for sparse activity header = {} values = data["values"] #If sparse matrix, write as sparse format if (sp.issparse(values)): if (inShape == None): raise ValueError( "Sparse values must have shape input when generating header" ) if len(inShape) != 3: raise ValueError( "Shape parameter must be a 3 tuple of (ny, nx, nf)") (ny, nx, nf) = inShape (numFrames, numFeat) = values.shape if (not numFeat == ny * nx * nf): raise ValueError( "Shape provided does not match the data shape (" + str(ny) + "*" + str(nx) + "*" + str(nf) + " vs " + str(numFeat) + ")") header["headersize"] = np.uint32(80) header["numparams"] = np.uint32(20) header["filetype"] = np.uint32(6) header["nx"] = np.uint32(nx) header["ny"] = np.uint32(ny) header["nf"] = np.uint32(nf) header["numrecords"] = np.uint32(1) header["recordsize"] = np.uint32(0) #Not used in sparse activity header["datasize"] = np.uint32(8) #Int/float are 4 bytes each header["datatype"] = np.uint32(4) #Type is location-value pair header["nxprocs"] = np.uint32(1) #No longer used header["nyprocs"] = np.uint32(1) header["nxExtended"] = np.uint32(nx) header["nyExtended"] = np.uint32(ny) header["kx0"] = np.uint32(0) header["ky0"] = np.uint32(0) header["nbatch"] = np.uint32(1) header["nbands"] = np.uint32(numFrames) header["time"] = np.float64(data["time"][0]) #If 4d dense matrix, write as dense format elif (values.ndim == 4): (numFrames, ny, nx, nf) = values.shape header["headersize"] = np.uint32(80) header["numparams"] = np.uint32(20) header["filetype"] = np.uint32(4) header["nx"] = np.uint32(nx) header["ny"] = np.uint32(ny) header["nf"] = np.uint32(nf) header["numrecords"] = np.uint32(1) header["recordsize"] = np.uint32(nx * ny * nf) #Not used in sparse activity header["datasize"] = np.uint32(4) #floats are 4 bytes header["datatype"] = np.uint32(3) #Type is float header["nxprocs"] = np.uint32(1) #No longer used header["nyprocs"] = np.uint32(1) header["nxExtended"] = np.uint32(nx) header["nyExtended"] = np.uint32(ny) header["kx0"] = np.uint32(0) header["ky0"] = np.uint32(0) header["nbatch"] = np.uint32(1) header["nbands"] = np.uint32(numFrames) header["time"] = np.float64(data["time"][0]) #If 6d dense matrix, write as weights format elif (values.ndim == 6): (numFrames, numArbors, numKernels, nyp, nxp, nfp) = values.shape header["headersize"] = np.uint32(104) header["numparams"] = np.uint32(26) header["filetype"] = np.uint32(5) header["nx"] = np.uint32(1) #size not used by weights header["ny"] = np.uint32(1) header["nf"] = np.uint32(numKernels) #Pre nf header["numrecords"] = np.uint32(numArbors) #Each data for arbor is preceded by nxp(2 bytes), ny (2 bytes) and offset (4 bytes) header["recordsize"] = np.uint32( 0) #weight files do not use recordsize header["datasize"] = np.uint32(4) #floats are 4 bytes header["datatype"] = np.uint32(3) #float type header["nxprocs"] = np.uint32(1) header["nyprocs"] = np.uint32(1) header["nxExtended"] = np.uint32(1) header["nyExtended"] = np.uint32(1) header["kx0"] = np.uint32(0) header["ky0"] = np.uint32(0) header["nbatch"] = np.uint32(1) header["nbands"] = np.uint32( numArbors ) #For weights, numArbors is stored in nbands, no field for numFrames #This field will be updated on write header["time"] = np.float64(data["time"][0]) #Weights have extended header header["nxp"] = np.uint32(nxp) header["nyp"] = np.uint32(nyp) header["nfp"] = np.uint32(nfp) header["wMax"] = np.uint32(1) #This field will be updated on write header["wMin"] = np.uint32(1) #This field will be updated on write header["numpatches"] = np.uint32(numKernels) return header
def __compile_kernels(self): ctype_indices = dtype_to_ctype(self.dtype_indices) ctype_labels = dtype_to_ctype(self.dtype_labels) ctype_counts = dtype_to_ctype(self.dtype_counts) ctype_samples = dtype_to_ctype(self.dtype_samples) n_labels = self.n_labels n_threads = self.COMPUTE_THREADS_PER_BLOCK n_shf_threads = self.RESHUFFLE_THREADS_PER_BLOCK """ DFS module """ dfs_module = compile_module("dfs_module.cu", (n_threads, n_shf_threads, n_labels, ctype_samples, ctype_labels, ctype_counts, ctype_indices, self.MAX_BLOCK_PER_FEATURE, self.debug)) const_stride = dfs_module.get_global("stride")[0] driver.memcpy_htod(const_stride, np.uint32(self.stride)) self.find_min_kernel = dfs_module.get_function("find_min_imp") self.find_min_kernel.prepare("PPPi") self.fill_kernel = dfs_module.get_function("fill_table") self.fill_kernel.prepare("PiiP") self.scan_reshuffle_tex = dfs_module.get_function("scan_reshuffle") self.scan_reshuffle_tex.prepare("PPii") tex_ref = dfs_module.get_texref("tex_mark") self.mark_table.bind_to_texref_ext(tex_ref) self.comput_total_2d = dfs_module.get_function("compute_2d") self.comput_total_2d.prepare("PPPPPPPii") self.reduce_2d = dfs_module.get_function("reduce_2d") self.reduce_2d.prepare("PPPPPi") self.scan_total_2d = dfs_module.get_function("scan_gini_large") self.scan_total_2d.prepare("PPPPii") self.scan_reduce = dfs_module.get_function("scan_reduce") self.scan_reduce.prepare("Pi") """ BFS module """ bfs_module = compile_module("bfs_module.cu", (self.BFS_THREADS, n_labels, ctype_samples, ctype_labels, ctype_counts, ctype_indices, self.debug)) const_stride = bfs_module.get_global("stride")[0] const_n_features = bfs_module.get_global("n_features")[0] const_max_features = bfs_module.get_global("max_features")[0] driver.memcpy_htod(const_stride, np.uint32(self.stride)) driver.memcpy_htod(const_n_features, np.uint16(self.n_features)) driver.memcpy_htod(const_max_features, np.uint16(self.max_features)) self.scan_total_bfs = bfs_module.get_function("scan_bfs") self.scan_total_bfs.prepare("PPPP") self.comput_bfs_2d = bfs_module.get_function("compute_2d") self.comput_bfs_2d.prepare("PPPPPPPPP") self.fill_bfs = bfs_module.get_function("fill_table") self.fill_bfs.prepare("PPPPP") self.reshuffle_bfs = bfs_module.get_function("scan_reshuffle") tex_ref = bfs_module.get_texref("tex_mark") self.mark_table.bind_to_texref_ext(tex_ref) self.reshuffle_bfs.prepare("PPP") self.reduce_bfs_2d = bfs_module.get_function("reduce") self.reduce_bfs_2d.prepare("PPPPPPi") self.get_thresholds = bfs_module.get_function("get_thresholds") self.get_thresholds.prepare("PPPPP") self.predict_kernel = mk_kernel( params = (ctype_indices, ctype_samples, ctype_labels), func_name = "predict", kernel_file = "predict.cu", prepare_args = "PPPPPPPii") self.bfs_module = bfs_module self.dfs_module = dfs_module
""" arrayops stores various array operations that will come in handy """ import bisect import numpy as np zero = np.uint32(0) one = np.uint32(1) def index(a, x): """ Given a sorted list, locate the leftmost value equal to x and return its index. This is the same as a.index(x) but uses the additional information that a is a sorted list. Example: >>> a = [0,1,2,3,4] >>> ind = index(a, 3) >>> ind 3 :param a: List to search :param x: value to find :return: """ i = bisect.bisect_left(a, x) if i != len(a) and a[i] == x: return i raise None
def write(self, data, shape=None, useExistingHeader=False): if (self.mode != 'w' and self.mode != 'a'): raise Exception("File not opened for writing") #Seek to end of file self.pvpFile.seek(0, os.SEEK_END) #Check data structure self.checkData(data) if not 'header' in data.keys(): if useExistingHeader: raise ValueError( "Must specify a \"header\" field if using existing header") #Generate header if it doesn't exist if (not self.header): if (useExistingHeader): self.header = data["header"] else: self.header = self.generateHeader(data, shape) #Write out full header self.writeHeader() #Otherwise, check header fields else: if (useExistingHeader): self.checkHeaders(self.header, data["header"]) else: self.checkHeaders(self.header, self.generateHeader(data, shape)) #Change nbands for number of frames self.updateHeader(data) if self.header['numparams'] == 26: hPattern = extendedHeaderPattern else: hPattern = headerPattern #Write out files based on data if self.header['filetype'] == 2: raise Exception('Filetype 2 not yet supported for write pvp') elif self.header['filetype'] == 4: (numFrames, ny, nx, nf) = data["values"].shape for dataFrame in range(numFrames): self.pvpFile.write(data["time"][dataFrame]) self.pvpFile.write(data["values"][dataFrame, :, :, :]) elif self.header['filetype'] == 5: (numFrames, numArbors, numKernels, nyp, nxp, nfp) = data["values"].shape # Type 5's have a header in each frame #Make a copy of header dictionary to avoid changing #the header field tmpHeader = self.header.copy() for dataFrame in range(numFrames): #Set header fields that change from frame to frame tmpHeader["time"] = np.float64(data["time"][dataFrame]) ##wMax and wMin are int32's, whereas the max and min might not be an int #tmpHeader["wMax"] = np.uint32(np.max(data["values"][dataFrame, :, :, :, :, :])) #tmpHeader["wMin"] = np.uint32(np.min(data["values"][dataFrame, :, :, :, :, :])) #We write headers here because we need a header per frame for headerEntry in hPattern: self.pvpFile.write(headerEntry[1]( tmpHeader[headerEntry[0]])) #Within each patch, we write out each nxp, nyp, and offset for dataArbor in range(numArbors): for dataKernel in range(numKernels): self.pvpFile.write(np.uint16(nxp)) self.pvpFile.write(np.uint16(nyp)) self.pvpFile.write( np.uint32(0)) #Offset is always 0 for kernels self.pvpFile.write(data["values"][dataFrame, dataArbor, dataKernel, :, :, :]) #Sparse values elif self.header['filetype'] == 6: (numFrames, numData) = data["values"].shape for dataFrame in range(numFrames): frameVals = data["values"].getrow(dataFrame) count = frameVals.nnz index = frameVals.indices value = frameVals.data #Write time first, followed by count, followed by values self.pvpFile.write(data["time"][dataFrame]) self.pvpFile.write(np.uint32(count)) npOut = np.zeros((count, 2)).astype(np.uint32) npOut[:, 0] = np.uint32(index) npOut[:, 1] = np.float32(value).view(np.uint32) self.pvpFile.write(npOut.flatten())
def test_3(self): sig = ['float64(float64, uint32)'] func = self.funcs['func2'] A = np.arange(100, dtype=np.float64) scale = np.uint32(3) self._run_and_compare(func, sig, A, scale, atol=1e-8)
def motif3struct_wei(W): ''' Structural motifs are patterns of local connectivity. Motif frequency is the frequency of occurrence of motifs around a node. Motif intensity and coherence are weighted generalizations of motif frequency. Parameters ---------- W : NxN np.ndarray weighted directed connection matrix (all weights between 0 and 1) Returns ------- I : 13xN np.ndarray motif intensity matrix Q : 13xN np.ndarray motif coherence matrix F : 13xN np.ndarray motif frequency matrix Notes ----- Average intensity and coherence are given by I./F and Q./F. ''' from scipy import io import os fname = os.path.join(os.path.dirname(__file__), motiflib) mot = io.loadmat(fname) m3 = mot['m3'] m3n = mot['m3n'] id3 = mot['id3'].squeeze() n3 = mot['n3'].squeeze() n = len(W) # number of vertices in W I = np.zeros((13, n)) # intensity Q = np.zeros((13, n)) # coherence F = np.zeros((13, n)) # frequency A = binarize(W, copy=True) # create binary adjmat As = np.logical_or(A, A.T) # symmetrized adjmat for u in range(n - 2): # v1: neighbors of u (>u) V1 = np.append(np.zeros((u, ), dtype=int), As[u, u + 1:n + 1]) for v1 in np.where(V1)[0]: # v2: neighbors of v1 (>u) V2 = np.append(np.zeros((u, ), dtype=int), As[v1, u + 1:n + 1]) V2[V1] = 0 # not already in V1 # and all neighbors of u (>v1) V2 = np.logical_or( np.append(np.zeros((v1, )), As[u, v1 + 1:n + 1]), V2) for v2 in np.where(V2)[0]: a = np.array( (A[v1, u], A[v2, u], A[u, v1], A[v2, v1], A[u, v2], A[v1, 2])) s = np.uint32(np.sum(np.power(10, np.arange(5, -1, -1)) * a)) ix = np.squeeze(s == m3n) w = np.array( (W[v1, u], W[v2, u], W[u, v1], W[v2, v1], W[u, v2], W[v1, v2])) M = w * m3[ix, :] id = id3[ix] - 1 l = n3[ix] x = np.sum(M, axis=1) / l # arithmetic mean M[M == 0] = 1 # enable geometric mean i = np.prod(M, axis=1)**(1 / l) # intensity q = i / x # coherence # add to cumulative counts I[id, u] += i I[id, v1] += i I[id, v2] += i Q[id, u] += q Q[id, v1] += q Q[id, v2] += q F[id, u] += 1 F[id, v1] += 1 F[id, v1] += 1 return I, Q, F