コード例 #1
0
def call_feature_demon(imageurl):
  result_dic = {}
  #import pdb; pdb.set_trace()
  try:
    fe_starttime = time.time()
    url = url_prefix % imageurl
    response = urllib2.urlopen(url)
    logging.info('extract_feature done, %.4f', time.time() - fe_starttime)
    if response <> None:
      retrieved_items = json.loads(response.read())
      if retrieved_items['result']:
        result_dic['url'] = retrieved_items['url']
        result_dic['predicted_category'] = retrieved_items['category']
        result_dic['scores'] = \
          np.trim_zeros((np.asarray(retrieved_items['score']) * 100).astype(np.uint8))
        #print(result_dic['scores'].shape)
        result_dic['predicted_category'] = result_dic['predicted_category'][0:result_dic['scores'].size]
        result_dic['feature'] = np.asarray(retrieved_items['feature'])

        result_dic['predicted_category_gc'] = retrieved_items['category_gc']
        result_dic['scores_gc'] = \
          np.trim_zeros((np.asarray(retrieved_items['score_gc']) * 100).astype(np.uint8))
        #print(result_dic['scores'].shape)
        result_dic['predicted_category_gc'] = result_dic['predicted_category_gc'][0:result_dic['scores_gc'].size]
        result_dic['sentence'] = retrieved_items['sentence']
        
  except Exception as err:
    logging.info('call_feature_demon error: %s', err)
    return {'result': False, 'feature': None}

  return result_dic
コード例 #2
0
def computePrecision(R_cv, B_cv, predictions, threshold, n):			
	'''
	Computes Precision@N metric on the cross-validation set.
	Precision@N is the percentage of movies the user rated above threshold in the recommendation list of size n
	'''
	
	cv_predictions = np.multiply(predictions, B_cv)
	sorted_predictions = np.fliplr(np.sort(cv_predictions))[:,:n]	
	top_indices = np.fliplr(np.argsort(cv_predictions))[:,:n]
	
	num_users = R_cv.shape[0]
	precision = np.zeros(num_users, dtype=float)
	
	for user_id in range(num_users):
		user_liked = np.ceil(threshold)<= np.trim_zeros(R_cv[user_id,top_indices[user_id]]) 
		user_disliked = np.trim_zeros(R_cv[user_id,top_indices[user_id]]) <= np.floor(threshold)
		
		# we think that a recommendation is good if the predicted rating 
		# is grater than threshold
		not_recommended = np.trim_zeros(sorted_predictions[user_id]) < threshold
		recommended = np.trim_zeros(sorted_predictions[user_id]) >= threshold
		
		tp = np.count_nonzero(np.logical_or(not_recommended, user_liked))
		fp = np.count_nonzero(np.logical_and(recommended, user_disliked))
		precision[user_id] = float(tp) / (tp+fp)
	
	mean_precision = np.mean(precision)
	return mean_precision
コード例 #3
0
ファイル: sequence.py プロジェクト: faircloth-lab/seqtools
 def trim(self, min_qual, clone = True):
     """return a new object with DNA trimmed according to some min_qual
         
     call using my_object.trim(clone=False) if you wish to 
     mask, trim, and return the current object.
     
     """
     self._qual_check()
     if clone: # pragma: no cover
         rval = self.clone()
     else: # pragma: no cover
         rval = self
     rval.min_qual = min_qual
     trim, comparison = rval._check()
     if trim:
         rval.snapshot()
         # use temp array so we don't change inner quality values (we are *not* masking)
         temp = deepcopy(rval.quality)
         temp[numpy.where(comparison)[0]] = 0
         l = len(rval.quality) - len(numpy.trim_zeros(temp,'f'))
         r = len(rval.quality) - len(numpy.trim_zeros(temp,'b'))
         rval.quality = rval.quality[l:len(rval.quality) - r]
         rval.sequence = rval.sequence[l:len(rval.sequence) - r]
         rval.trimming = 't'
     return rval
コード例 #4
0
ファイル: visual.py プロジェクト: iniverno/MIsim
def redundancyAnalysis(m):
  CHECK = False
  dims = m.shape
  vals = np.zeros((dims[1]*dims[2]*dims[3], (1<<PRECISION+1)))
  
  #if CHECK:
  aux = np.zeros(257)
  auxVal = np.zeros(257)

  index = 0
  for c in range(dims[1]):
    for x in range(dims[2]):
      for y in range(dims[3]):
        for i in range(dims[0]):
          if CHECK:
            if aux[ toInteger(m[i][c][x][y]) ] > 0:
              if m[i][c][x][y] != auxVal[  toInteger(m[i][c][x][y]) ]:
                print "ALARM"
            auxVal[ toInteger(m[i][c][x][y]) ] = m[i][c][x][y]
            aux[ toInteger(m[i][c][x][y]) ] += 1
          
          vals[index][ toInteger(m[i][c][x][y]) ] += 1;
        index += 1 # same value for all the filters
  vals = vals.flatten()
  vals = np.sort(vals)
  #print vals
  vals = np.trim_zeros(vals)
  vals -= 1
  vals = np.trim_zeros(vals)
  vals = vals[::-1]
  #print vals
  vals = np.cumsum(vals)
  vals /= dims[1]*dims[2]*dims[3]*dims[0]
  vals = sample(vals,SAMPLES)
  return vals
コード例 #5
0
ファイル: dem.py プロジェクト: davenquinn/Attitude
 def axis_data(axis):
     """Gets the bounds of a masked area along a certain axis"""
     x = mask.sum(axis)
     trimmed_front = N.trim_zeros(x,"f")
     offset = len(x)-len(trimmed_front)
     size = len(N.trim_zeros(trimmed_front,"b"))
     return offset,size
コード例 #6
0
ファイル: gasrank.py プロジェクト: RJT1990/pyflux
    def plot_abilities_one_components(self, team_ids, **kwargs):
        import matplotlib.pyplot as plt
        import seaborn as sns

        figsize = kwargs.get('figsize',(15,5))

        if self.latent_variables.estimated is False:
            raise Exception("No latent variables estimated!")
        else:
            plt.figure(figsize=figsize)

            if type(team_ids) == type([]):
                if type(team_ids[0]) == str:
                    for team_id in team_ids:
                        plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[self.team_dict[team_id]],
                            trim='b'), label=self.team_strings[self.team_dict[team_id]])
                else:
                    for team_id in team_ids:
                        plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[team_id],
                            trim='b'), label=self.team_strings[team_id])
            else:
                if type(team_ids) == str:
                    plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[self.team_dict[team_ids]],
                        trim='b'), label=self.team_strings[self.team_dict[team_ids]])
                else:
                    plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[team_ids],
                        trim='b'), label=self.team_strings[team_ids])

            plt.legend()
            plt.ylabel("Power")
            plt.xlabel("Games")
            plt.show()
コード例 #7
0
def calc_power(N, field, dk, Nshot, Lx, Ly, norm):

	dkx = (2.*np.pi)/Lx
	dky = (2.*np.pi)/Ly
	V= Lx*Ly

	power = np.zeros(N, dtype=float)
	Nmodes = np.zeros(N, dtype=int)
	
	for ix in range (0,N):
		if ix <=N/2.:
			kx = ix*dkx
		else:
			kx = (ix-N)*dkx

		for iy in range (0,N):
			if iy <=N/2.:
				ky = iy*dky
			else:
				ky = (iy-N)*dky
	
			kval = (kx*kx + ky*ky)**0.5

			if kval >0:
				power[int(kval/dk)] = power[int(kval/dk)] + field[ix][iy].real**2 + field[ix][iy].imag**2 - Nshot
				Nmodes[int(kval/dk)] = Nmodes[int(kval/dk)]+1
				
	iNonZeros = np.where(Nmodes != 0)	
	iZeros = np.where(Nmodes == 0)
	power[iNonZeros] = power[iNonZeros]/Nmodes[iNonZeros]

	k=np.linspace(dkx/2., ((N-1)*dkx)+dkx/2. ,num=N )	
	k[iZeros]= 0.
	return V*np.trim_zeros(power)/norm, np.trim_zeros(k)
コード例 #8
0
ファイル: array.py プロジェクト: titodalcanton/pycbc
 def trim_zeros(self):
     """Remove the leading and trailing zeros.
     """      
     tmp = self.numpy()
     f = len(self)-len(_numpy.trim_zeros(tmp, trim='f'))
     b = len(self)-len(_numpy.trim_zeros(tmp, trim='b'))
     return self[f:len(self)-b]
コード例 #9
0
ファイル: peak_finder.py プロジェクト: CSB5/vipr
def parse_coverage_bam(fbam, genomecoveragebed="genomeCoverageBed"):
    """
    
    Arguments:
    - `fbam`: file to read coverage from. Needs samtools and bedtools (genomeCoverageBed)
    - `genomecoveragebed`: path to genomeCoverageBed binary
    """

  
    # forward and reverse coverage
    # int is essential; see note below
    fw_cov = numpy.zeros((MAX_LEN), dtype=numpy.int32)
    rv_cov = numpy.zeros((MAX_LEN), dtype=numpy.int32)

    
    file_genome = tempfile.NamedTemporaryFile(delete=False)
    genome_from_bam(fbam, file_genome)
    file_genome.close()

    basic_cmd = "%s -ibam %s -g %s -d" % (
        genomecoveragebed, fbam, file_genome.name)



    for strand in ["+", "-"]:
        cmd = "%s -strand %s" % (basic_cmd, strand)
        try:
            process = subprocess.Popen(cmd.split(),
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
        except OSError:
            raise OSError, ("It seems %s is not installed" % cmd.split()[0])
        (stdoutdata, stderrdata) =  process.communicate()
        retcode = process.returncode
        if retcode != 0:
            raise OSError("Called command exited with error code '%d'." \
                          " Command was '%s'. stderr was: '%s'" % (
                              retcode, cmd, stderrdata))
        for line in str.splitlines(stderrdata):
            if len(line.strip()) == 0:
                continue
            LOG.warn("Got following stderr message from genomeCoverageBed: %s" % line)
        for line in str.splitlines(stdoutdata):
            if len(line) == 0:
                continue
            (chr, pos, cov) = line.split('\t')
            pos = int(pos)-1 # we use zero offset
            cov = int(cov)

            assert pos < MAX_LEN

            if strand == '+':
                fw_cov[pos] = cov
            elif strand == '-':
                rv_cov[pos] = cov

    os.unlink(file_genome.name)

    return (numpy.trim_zeros(fw_cov, trim='b'),
            numpy.trim_zeros(rv_cov, trim='b'))
コード例 #10
0
ファイル: FeatureFunctionLib.py プロジェクト: guille-c/FATS
    def fit(self, data):
        magnitude = data[0]
        time = data[1]

        global m_21
        global m_31
        global m_32

        Nsf = 100
        Np = 100
        sf1 = np.zeros(Nsf)
        sf2 = np.zeros(Nsf)
        sf3 = np.zeros(Nsf)
        f = interp1d(time, magnitude)

        time_int = np.linspace(np.min(time), np.max(time), Np)
        mag_int = f(time_int)

        for tau in np.arange(1, Nsf):
            sf1[tau-1] = np.mean(np.power(np.abs(mag_int[0:Np-tau] - mag_int[tau:Np]) , 1.0))
            sf2[tau-1] = np.mean(np.abs(np.power(np.abs(mag_int[0:Np-tau] - mag_int[tau:Np]) , 2.0)))
            sf3[tau-1] = np.mean(np.abs(np.power(np.abs(mag_int[0:Np-tau] - mag_int[tau:Np]) , 3.0)))
        sf1_log = np.log10(np.trim_zeros(sf1))
        sf2_log = np.log10(np.trim_zeros(sf2))
        sf3_log = np.log10(np.trim_zeros(sf3))

        m_21, b_21 = np.polyfit(sf1_log, sf2_log, 1)
        m_31, b_31 = np.polyfit(sf1_log, sf3_log, 1)
        m_32, b_32 = np.polyfit(sf2_log, sf3_log, 1)

        return m_21
コード例 #11
0
ファイル: agent.py プロジェクト: VinF/deer
 def avgEpisodeVValue(self):
     """ Returns the average V value on the episode (on time steps where a non-random action has been taken)
     """
     if (len(self._Vs_on_last_episode) == 0):
         return -1
     if(np.trim_zeros(self._Vs_on_last_episode)!=[]):
         return np.average(np.trim_zeros(self._Vs_on_last_episode))
     else:
         return 0
コード例 #12
0
 def assert_numden_almost_equal(self, n1, n2, d1, d2):
     n1[np.abs(n1) < 1e-10] = 0.
     n1 = np.trim_zeros(n1)
     d1[np.abs(d1) < 1e-10] = 0.
     d1 = np.trim_zeros(d1)
     n2[np.abs(n2) < 1e-10] = 0.
     n2 = np.trim_zeros(n2)
     d2[np.abs(d2) < 1e-10] = 0.
     d2 = np.trim_zeros(d2)
     np.testing.assert_array_almost_equal(n1, n2)
     np.testing.assert_array_almost_equal(d2, d2)
コード例 #13
0
ファイル: fpuwork.py プロジェクト: USStateDept/FPPWork
def printSummaryOutput(maincounts):
    print OUTPUT + "\\summaryoutput.txt"
    with open(OUTPUT + "\\summaryoutput.txt", 'wb') as f:
        for index in maincounts.keys():
            f.write(index + "  ***************\n")
            f.write("Total Count: " + str(maincounts[index]['totnum']) + '\n')
            f.write("Total Females: " + str(maincounts[index]['numfemales']) + '\n')
            f.write("Total Males: " + str(maincounts[index]['nummales']) + '\n')

            f.write("Mean Length of Stay: " + str(np.mean(np.trim_zeros(np.nan_to_num(maincounts[index]['meanlengthofstay'])))) + '\n')
            f.write("Mean Age: " + str(np.mean(np.trim_zeros(maincounts[index]['meanage']))) + '\n')
            f.write("Mean Travel Events: " + str(np.mean(np.trim_zeros(maincounts[index]['meantravelevents']))) + '\n')
コード例 #14
0
    def get_history(self, state, action):
        """
        :param state:
        :param action:
        :return: h(s,a), i.e. the number of times given action was selected in given state in the current episode
        """

        state = tuple(np.trim_zeros(state, 'f'))
        action = tuple(np.trim_zeros(action, 'f'))

        if (state, action) in self.state_action_history:
            return self.state_action_history[(state, action)]
        return 0
コード例 #15
0
ファイル: gasrank.py プロジェクト: RJT1990/pyflux
    def predict_two_components(self, team_1, team_2, team_1b, team_2b, neutral=False):
        """
        Returns team 1's probability of winning
        """
        if self.latent_variables.estimated is False:
            raise Exception("No latent variables estimated!")
        else:
            if type(team_1) == str:
                team_1_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[self.team_dict[team_1]], trim='b')[-1]
                team_2_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[self.team_dict[team_2]], trim='b')[-1]
                team_1_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[self.team_dict[team_1]], trim='b')[-1]
                team_2_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[self.team_dict[team_2]], trim='b')[-1]
  
            else:
                team_1_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[team_1], trim='b')[-1]
                team_2_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[team_2], trim='b')[-1]
                team_1_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[team_1_b], trim='b')[-1]
                team_2_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[team_2_b], trim='b')[-1]

        t_z = self.transform_z()

        if neutral is False:
            return self.link(t_z[0] + team_1_ability - team_2_ability + team_1_b_ability - team_2_b_ability)
        else:
            return self.link(team_1_ability - team_2_ability + team_1_b_ability - team_2_b_ability)
コード例 #16
0
def largestDistrict(data):
    largestDis=0
    data_new = data.dropna(subset=['Location','PoliceDistrict'])
    policeDis=np.array(data_new['PoliceDistrict'])
    Location=np.array(data_new['Location'])

    dic= collections.defaultdict(list)

    for i in range(policeDis.shape[0]):
        dic[policeDis[i]]+=[Location[i]]
    for key,value in dic.items():
        x=np.zeros(len(value))
        y=np.zeros(len(value))
        xi=0
        for i in range(len(value)):
            l=value[i][1:-2]
            loc=l.split(",")

            if float(loc[0])>26 and float(loc[0])<30 and float(loc[1])>-93 and float(loc[1])<-87:
                x[xi]=float(loc[0])
                y[xi]=float(loc[1])
                xi+=1
        x=np.trim_zeros(x)
        y=np.trim_zeros(y)
        stdX=np.std(x)
        stdY=np.std(y)
        meanX=np.mean(x)
        meanY=np.mean(y)

        bootom_la=math.radians(meanX-stdX)
        top_la=math.radians(meanX+stdX)
        delta_la=math.radians(2*stdX)
        delta_long=0

        a=math.sin(delta_la)*math.sin(delta_la)+math.cos(bootom_la)*math.cos(top_la)*math.sin(delta_long/2)*math.sin(delta_long/2)
        c=2* math.atan2(math.sqrt(a),math.sqrt(1-a))
        d_x=6371*c/2

        delta_la=0
        delta_long=math.radians(2*stdY)
        la=math.radians(meanX)

        a=math.sin(delta_la)*math.sin(delta_la)+math.cos(la)*math.cos(la)*math.sin(delta_long/2)*math.sin(delta_long/2)
        c=2*math.atan2(math.sqrt(a),math.sqrt(1-a))
        d_y=6371*c/2

        area=math.pi*d_x*d_y
        print area, key, meanX,meanY,stdX,stdY
        largestDis=max(area,largestDis)
    return largestDis
コード例 #17
0
ファイル: red_tools.py プロジェクト: CarlosBacigalupo/ipn
    def clean_flux(self,flux, xDef = 1, lambdas = np.array([])):
        '''clean a flux array to cross corralate to determine RV shift
            eliminates NaNs
            moving median to reduce peaks
            optional: increase resolution by xDef times
            
        '''	
        
        #Copy to output in case of no resampling
        fluxHD = flux
        newLambdas = lambdas
        
        
        #clean NaNs and median outliers	
        fluxHD[np.isnan(fluxHD)] = 0
        fluxNeat = fluxHD	
        fluxMed = signal.medfilt(fluxHD,5)
        w = np.where(abs((fluxHD-fluxMed)/np.maximum(fluxMed,50)) > 0.4)
        for ix in w[0]:
            fluxNeat[ix] = fluxMed[ix]

        #if enough data -> resample
        if ((xDef>1) and (len(lambdas)>0)):
            fFluxHD = interpolate.interp1d(lambdas,fluxNeat) 
            lambdas = np.arange(min(lambdas), max(lambdas),(max(lambdas)-min(lambdas))/len(lambdas)/xDef)
            fluxNeat = fFluxHD(lambdas)
        
        #remove trailing zeros, devide by fitted curve (flatten) and apply tukey window
        fluxNeat = np.trim_zeros(fluxNeat,'f') 
        lambdas = lambdas[-len(fluxNeat):]
        fluxNeat = np.trim_zeros(fluxNeat,'b') 
        lambdas = lambdas[:len(fluxNeat)]
        
        if ((lambdas.shape[0]>0) &  (fluxNeat.shape[0]>0)):
            fFluxNeat = optimize.curve_fit(self.cubic, lambdas, fluxNeat, p0 = [1,1,1,1])
            fittedCurve = self.cubic(lambdas, fFluxNeat[0][0], fFluxNeat[0][1], fFluxNeat[0][2], fFluxNeat[0][3])
        # 	plt.plot(fittedCurve)
        # 	plt.plot(fluxNeat)
        # 	plt.show()
        # 	plt.plot(fluxNeat/fittedCurve-1)
        # 	plt.show()
            
            fluxFlat = fluxNeat/fittedCurve-1
            
            fluxWindow = fluxFlat * self.tukey(0.1, len(fluxFlat))
        else:
            fluxWindow = np.array([])
            print 'empty after removing zeros'
            
        return lambdas, fluxWindow
コード例 #18
0
ファイル: tfuncs_praat.py プロジェクト: avashlin/ttslab
def trim_zeros(track, front=True, back=True):
    if front:
        values = np.trim_zeros(track.values, trim="f")
        times = track.times[len(track) - len(values):]
    else:
        values = track.values
        times = track.times
    if back:
        vallen = len(values)
        values = np.trim_zeros(values, trim="b")
        if vallen != len(values):
            times = times[:len(values) - vallen]

    track.values = values
    track.times = times
コード例 #19
0
    def add_to_history(self, state, action):
        """
        Adds a state, action pair to the history of the current episode, or increases its counter if already present
        :param state:
        :param action:
        :return:
        """

        state = tuple(np.trim_zeros(state, 'f'))
        action = tuple(np.trim_zeros(action, 'f'))

        if (state, action) in self.state_action_history:
            self.state_action_history[(state, action)] += 1
        else:
            self.state_action_history[(state, action)] = 1
コード例 #20
0
ファイル: FeatureFunctionLib.py プロジェクト: npcastro/FATS
    def slotted_autocorrelation(self, data, time, T, K,
                                second_round=False, K1=100):

        slots = np.zeros((K, 1))
        i = 1

        # make time start from 0
        time = time - np.min(time)

        # subtract mean from mag values
        m = np.mean(data)
        data = data - m

        prod = np.zeros((K, 1))
        pairs = np.subtract.outer(time, time)
        pairs[np.tril_indices_from(pairs)] = 10000000

        ks = np.int64(np.floor(np.abs(pairs) / T + 0.5))

        # We calculate the slotted autocorrelation for k=0 separately
        idx = np.where(ks == 0)
        prod[0] = ((sum(data ** 2) + sum(data[idx[0]] *
                   data[idx[1]])) / (len(idx[0]) + len(data)))
        slots[0] = 0

        # We calculate it for the rest of the ks
        if second_round is False:
            for k in np.arange(1, K):
                idx = np.where(ks == k)
                if len(idx[0]) != 0:
                    prod[k] = sum(data[idx[0]] * data[idx[1]]) / (len(idx[0]))
                    slots[i] = k
                    i = i + 1
                else:
                    prod[k] = np.infty
        else:
            for k in np.arange(K1, K):
                idx = np.where(ks == k)
                if len(idx[0]) != 0:
                    prod[k] = sum(data[idx[0]] * data[idx[1]]) / (len(idx[0]))
                    slots[i - 1] = k
                    i = i + 1
                else:
                    prod[k] = np.infty
            np.trim_zeros(prod, trim='b')

        slots = np.trim_zeros(slots, trim='b')
        return prod / prod[0], np.int64(slots).flatten()
コード例 #21
0
ファイル: rnn_inference.py プロジェクト: rsennrich/nematus
def sample(session, model, x, x_mask, graph=None):
    """Randomly samples translations from a RNNModel.

    Args:
        session: TensorFlow session.
        model: a RNNModel object.
        x: Numpy array with shape (factors, max_seq_len, batch_size).
        x_mask: Numpy array with shape (max_seq_len, batch_size).
        graph: a SampleGraph object (to allow reuse if sampling repeatedly).

    Returns:
        A list of NumPy arrays (one for each input sentence in x).
    """
    feed_dict = {model.inputs.x: x, model.inputs.x_mask: x_mask}
    if graph is None:
        graph = SampleGraph(model)
    sampled_ys = session.run(graph.outputs, feed_dict=feed_dict)
    sampled_ys = sampled_ys.T
    samples = []
    for sample in sampled_ys:
        sample = numpy.trim_zeros(list(sample), trim='b')
        sample.append(0)
        samples.append(sample)
    assert len(samples) == x.shape[-1]
    return samples
コード例 #22
0
ファイル: sequence.py プロジェクト: faircloth-lab/seqtools
 def mask_and_trim(self, min_qual, clone = True):
     """return a new object with DNA masked and trimmed according to some min_qual
         
     call using my_object.mask_and_trim(clone=False) if you wish to 
     mask, trim, and return the current object.
     
     """
     self._qual_check()
     if clone: # pragma: no cover
         rval = self.clone()
     else: # pragma: no cover
         rval = self
     rval.min_qual = min_qual
     trim, comparison = rval._check()
     if trim:
         rval.snapshot()
         sequence_array = numpy.array(list(rval.sequence))
         # get values where the qual is low and replace them w/ N
         sequence_array[numpy.where(comparison)[0]] = 'N'
         # trim and adjust rval.sequence
         rval.sequence = sequence_array.tostring().lstrip('N').rstrip('N')
         # trim and adjust rval.quality
         temp = deepcopy(rval.quality)
         temp[numpy.where(comparison)[0]] = 0
         rval.quality[numpy.where(comparison)[0]] = 0
         rval.quality = numpy.trim_zeros(rval.quality)
         # ensure the trimming didn't remove something inside the read
         assert len(rval.sequence) == len(rval.quality)
         rval.trimming = 'mt'
     return rval
コード例 #23
0
ファイル: font.py プロジェクト: manz/ff4
def get_max_width(char):
    max_width = 0
    for byte in char:
        trimmed = np.trim_zeros(byte, 'b')
        max_width = max(len(trimmed), max_width)

    return max_width
コード例 #24
0
ファイル: analysis.py プロジェクト: ronald-xie/SLM-Lab
def get_session_data(session):
    '''
    Gather data from session: MDP, Agent, Env data, hashed by aeb; then aggregate.
    @returns {dict, dict} session_mdp_data, session_data
    '''
    data_names = AGENT_DATA_NAMES + ENV_DATA_NAMES
    mdp_data_names = ['t', 'epi'] + data_names
    agg_data_names = ['epi'] + list(DATA_AGG_FNS.keys())
    data_h_v_dict = {data_name: session.aeb_space.get_history_v(data_name) for data_name in data_names}
    session_mdp_data, session_data = {}, {}
    for aeb in session.aeb_space.aeb_list:
        data_h_dict = {data_name: data_h_v[aeb] for data_name, data_h_v in data_h_v_dict.items()}
        # trim back to remove any incomplete sessions due to multienv termination
        complete_done_h = np.trim_zeros(data_h_dict['done'], 'b')
        # offset properly to bin separate episodes
        reset_bin = np.concatenate([[0.], complete_done_h[:-1]])
        data_len = len(reset_bin)
        reset_idx = reset_bin.astype('bool')
        nonreset_idx = ~reset_idx
        data_h_dict['t'] = np.ones(reset_idx.shape)
        data_h_dict['epi'] = reset_idx.astype(int).cumsum()
        mdp_df = pd.DataFrame({
            data_name: data_h_dict[data_name][:data_len]
            for data_name in mdp_data_names})
        mdp_df = mdp_df.reindex(mdp_data_names, axis=1)
        aeb_df = mdp_df[agg_data_names].groupby('epi').agg(DATA_AGG_FNS)
        aeb_df.reset_index(drop=False, inplace=True)
        session_mdp_data[aeb], session_data[aeb] = mdp_df, aeb_df
    logger.debug(f'{session_data}')
    data_size_in_bytes = util.memory_size(session_mdp_data)
    logger.debug(f'Size of session data: {data_size_in_bytes} MB')
    if data_size_in_bytes > 25:
        logger.warn(f'Session data > 25 MB')
    return session_mdp_data, session_data
コード例 #25
0
ファイル: area.py プロジェクト: davenquinn/Imagery
def extract_area(dataset, geometry, **kwargs):
    """
    Compute a spectrum using the basic GDAL driver to read a single band of the spectrum.
    This is faster than the rasterio version and is the default.
    """
    im = dataset.__gdal__
    bands = kwargs.pop("bands", list(range(im.RasterCount)))

    mask = create_mask(dataset,geometry)
    maskt, offset = offset_mask(mask)
    yo,xo = offset
    ys,xs = maskt.shape
    for i in (0,1):
        assert N.allclose(maskt.sum(axis=i), N.trim_zeros(mask.sum(axis=i)))

    maskt = maskt.transpose() # Conform to GDAL's fascist X-first expectations
    maska = N.repeat(
        N.expand_dims(maskt,0),
        len(bands),
        axis=0)
    buffer=im.ReadRaster(xo,yo,xs,ys,
        band_list=[b+1 for b in bands])
    arr = N.fromstring(buffer, dtype=dataset.dtype)
    arr = arr.reshape((len(bands), xs, ys))
    arr = N.ma.masked_array(arr, arr==dataset.nodata)
    arr[maska==False] = N.ma.masked
    xarr = N.array([xo+0.5 for i in range(xs)]).reshape((xs,0))
    yarr = N.array([yo+0.5 for i in range(ys)]).reshape((ys,1))

    import IPython; IPython.embed()
    return arr
コード例 #26
0
ファイル: models.py プロジェクト: JeffValenti/ExoCTK
    def _parse_coeffs(self, coeff_dict):
        """Convert dict of 'c#' coefficients into a list
        of coefficients in decreasing order, i.e. ['c2','c1','c0']

        Parameters
        ----------
        coeff_dict: dict
            The dictionary of coefficients

        Returns
        -------
        np.ndarray
            The sequence of coefficient values
        """
        params = {cN: coeff for cN, coeff in coeff_dict.items()
                  if cN.startswith('c') and cN[1:].isdigit()}
        self.parameters = Parameters(**params)

        # Parse 'c#' keyword arguments as coefficients
        coeffs = np.zeros(100)
        for k, v in self.parameters.dict.items():
            if k.lower().startswith('c') and k[1:].isdigit():
                coeffs[int(k[1:])] = v[0]

        # Trim zeros and reverse
        self.coeffs = np.trim_zeros(coeffs)[::-1]
コード例 #27
0
ファイル: __init__.py プロジェクト: alexellis/microdot-phat
def write_string(string, offset_x=0, offset_y=0, kerning=True):
    str_buf = []

    space = [0x00] * 5
    gap = [0x00] * 3

    if kerning:
        space = [0x00] * 2
        gap = [0x00]

    for char in string:
        if char == ' ':
            str_buf += space
        else:
            char_data = numpy.array(_get_char(char))
            if kerning:
                char_data = numpy.trim_zeros(char_data)
            str_buf += list(char_data)
        str_buf += gap # Gap between chars

    for x in range(len(str_buf)):
        for y in range(7):
            p = (str_buf[x] & (1 << y)) > 0
            set_pixel(offset_x + x, offset_y + y, p)

    l = len(str_buf)
    del str_buf
    return l
コード例 #28
0
ファイル: resynthesis.py プロジェクト: davebrent/consyn
    def process(self, samples, unit, target):
        if "f" in self.trim:
            temporary = numpy.empty_like(samples)
            temporary[:] = samples
            temporary[numpy.abs(samples) <= self.cutoff] = 0
            temporary = numpy.trim_zeros(temporary, trim="f")
            samples = samples[samples.shape[0] - temporary.shape[0]:]

        if "b" in self.trim:
            temporary = numpy.empty_like(samples)
            temporary[:] = samples
            temporary[numpy.abs(temporary) <= self.cutoff] = 0
            temporary = numpy.trim_zeros(temporary, trim="b")
            samples = samples[0:temporary.shape[0]]

        return samples, unit
コード例 #29
0
ファイル: rnn_inference.py プロジェクト: rsennrich/nematus
def _reconstruct_hypotheses(ys, parents, cost, beam_size):
    """Converts raw beam search outputs into a more usable form.

    Args:
        ys: NumPy array with shape (max_seq_len, beam_size*batch_size).
        parents: NumPy array with same shape as ys.
        cost: NumPy array with same shape as ys.
        beam_size: integer.

    Returns:
        A list of lists of (translation, score) pairs. The outer list contains
        one list for each input sentence in the batch. The inner lists contain
        k elements (where k is the beam size).
    """
    def reconstruct_single(ys, parents, hypoId, hypo, pos):
        if pos < 0:
            hypo.reverse()
            return hypo
        else:
            hypo.append(ys[pos, hypoId])
            hypoId = parents[pos, hypoId]
            return reconstruct_single(ys, parents, hypoId, hypo, pos - 1)

    hypotheses = []
    batch_size = ys.shape[1] // beam_size
    pos = ys.shape[0] - 1
    for batch in range(batch_size):
        hypotheses.append([])
        for beam in range(beam_size):
            i = batch*beam_size + beam
            hypo = reconstruct_single(ys, parents, i, [], pos)
            hypo = numpy.trim_zeros(hypo, trim='b') # b for back
            hypo.append(0)
            hypotheses[batch].append((hypo, cost[i]))
    return hypotheses
コード例 #30
0
ファイル: elastic_net.py プロジェクト: ceholden/glmnet-python
def enet_path(X, y, alpha=0.5, **kwargs):
    """ Convenience wrapper for running elastic_net that transforms coefs

    Args:
        X (np.ndarray): 2D (n_obs x n_features) design matrix
        y (np.ndarray): 1D independent variable
        alpha (float): ElasticNet mixing parameter (0 <= alpha <= 1.0)
            specifying the mix of Ridge L2 (``alpha=0``) to Lasso L1
            (``alpha=1``) regularization (default: 0.5).
        kwargs: additional arguments provided to GLMNET

    Returns:
        tuple: intercepts, coefficients, rsquareds, and lambdas as np.ndarrays

    """
    n_lambdas, intercepts_, coefs_, ia, nin, rsquareds_, lambdas, _, jerr \
        = elastic_net(X, y, alpha, **kwargs)

    # ia is 1 indexed
    ia = np.trim_zeros(ia, 'b') - 1

    # glmnet.f returns coefficients as 'compressed' array that
    # requires re-indexing using ia and nin
    coefs = np.zeros_like(coefs_)
    coefs[ia, :] = coefs_[:np.max(nin), :n_lambdas]

    return intercepts_, coefs, rsquareds_, lambdas
コード例 #31
0
ファイル: cfinite_sequence.py プロジェクト: ipozdnyakov/sage
    def guess(self, sequence, algorithm='sage'):
        """
        Return the minimal CFiniteSequence that generates the sequence.

        Assume the first value has index 0.

        INPUT:

        - ``sequence`` -- list of integers
        - ``algorithm`` -- string
            - 'sage' - the default is to use Sage's matrix kernel function
            - 'pari' - use Pari's implementation of LLL
            - 'bm' - use Sage's Berlekamp-Massey algorithm

        OUTPUT:

        - a CFiniteSequence, or 0 if none could be found

        With the default kernel method, trailing zeroes are chopped
        off before a guessing attempt. This may reduce the data
        below the accepted length of six values.

        EXAMPLES::

            sage: C.<x> = CFiniteSequences(QQ)
            sage: C.guess([1,2,4,8,16,32])
            C-finite sequence, generated by 1/(-2*x + 1)
            sage: r = C.guess([1,2,3,4,5])
            Traceback (most recent call last):
            ...
            ValueError: Sequence too short for guessing.

        With Berlekamp-Massey, if an odd number of values is given, the last one is dropped.
        So with an odd number of values the result may not generate the last value::

            sage: r = C.guess([1,2,4,8,9], algorithm='bm'); r
            C-finite sequence, generated by 1/(-2*x + 1)
            sage: r[0:5]
            [1, 2, 4, 8, 16]
        """
        S = self.polynomial_ring()
        if algorithm == 'bm':
            from sage.matrix.berlekamp_massey import berlekamp_massey
            if len(sequence) < 2:
                raise ValueError('Sequence too short for guessing.')
            R = PowerSeriesRing(QQ, 'x')
            if len(sequence) % 2 == 1:
                sequence = sequence[:-1]
            l = len(sequence) - 1
            denominator = S(berlekamp_massey(sequence).list()[::-1])
            numerator = R(S(sequence) * denominator, prec=l).truncate()

            return CFiniteSequence(numerator / denominator)
        elif algorithm == 'pari':
            global _gp
            if len(sequence) < 6:
                raise ValueError('Sequence too short for guessing.')
            if _gp is None:
                _gp = Gp()
                _gp("ggf(v)=local(l,m,p,q,B);l=length(v);B=floor(l/2);\
                if(B<3,return(0));m=matrix(B,B,x,y,v[x-y+B+1]);\
                q=qflll(m,4)[1];if(length(q)==0,return(0));\
                p=sum(k=1,B,x^(k-1)*q[k,1]);\
                q=Pol(Pol(vector(l,n,v[l-n+1]))*p+O(x^(B+1)));\
                if(polcoeff(p,0)<0,q=-q;p=-p);q=q/p;p=Ser(q+O(x^(l+1)));\
                for(m=1,l,if(polcoeff(p,m-1)!=v[m],return(0)));q")
            _gp.set('gf', sequence)
            _gp("gf=ggf(gf)")
            num = S(sage_eval(_gp.eval("Vec(numerator(gf))"))[::-1])
            den = S(sage_eval(_gp.eval("Vec(denominator(gf))"))[::-1])
            if num == 0:
                return 0
            else:
                return CFiniteSequence(num / den)
        else:
            from sage.matrix.constructor import matrix
            from sage.functions.other import floor, ceil
            from numpy import trim_zeros
            l = len(sequence)
            while l > 0 and sequence[l - 1] == 0:
                l -= 1
            sequence = sequence[:l]
            if l == 0:
                return 0
            if l < 6:
                raise ValueError('Sequence too short for guessing.')

            hl = ceil(ZZ(l) / 2)
            A = matrix([sequence[k:k + hl] for k in range(hl)])
            K = A.kernel()
            if K.dimension() == 0:
                return 0
            R = PolynomialRing(QQ, 'x')
            den = R(trim_zeros(K.basis()[-1].list()[::-1]))
            if den == 1:
                return 0
            offset = next((i for i, x in enumerate(sequence) if x != 0), None)
            S = PowerSeriesRing(QQ, 'x', default_prec=l - offset)
            num = S(R(sequence) * den).add_bigoh(floor(ZZ(l) / 2 +
                                                       1)).truncate()
            if num == 0 or sequence != S(num / den).list():
                return 0
            else:
                return CFiniteSequence(num / den)
コード例 #32
0
def nodeGraphGeneration():
    nodeMatrix = []
    adjacentNodeDict = {}

    nodeElevationDict, numRows, numCols = generateNodeElevationDict()

    #Creates 2D Matrix of node ids, the shape of which is determined by the number size of the user selected region
    for i in range(numRows):
        nodeRow = []
        for j in range(numCols):
            nodeRow.append(i * numCols + (j + 1))
        nodeMatrix.append(nodeRow)

    #Adds zero array first row
    boundedNodeMatrix = [np.zeros(numCols + 2, dtype=int)]

    #Adds zeros on either end of each row, and appends them to boundedNodeMatrix
    for i in nodeMatrix:
        i = np.insert(i, 0, 0)
        i = np.insert(i, numCols + 1, 0)
        boundedNodeMatrix.append(i)
    #Adds zero array for last row
    boundedNodeMatrix.append(np.zeros(numCols + 2, dtype=int))

    #Iterates through each cell in the table
    for row in range(1, numRows + 1):
        for column in range(1, numCols + 1):
            #Adjacent nodes to a current node are discovered (eight nodes surrounding the current node, similar to the 5 on a standard keyboard numPad)
            adjacentNodeMatrix = [
                boundedNodeMatrix[row - 1][column],
                boundedNodeMatrix[row - 1][column + 1],
                boundedNodeMatrix[row][column + 1],
                boundedNodeMatrix[row + 1][column + 1],
                boundedNodeMatrix[row + 1][column],
                boundedNodeMatrix[row + 1][column - 1],
                boundedNodeMatrix[row][column - 1],
                boundedNodeMatrix[row - 1][column - 1]
            ]
            adjacentNodeMatrix = np.sort(adjacentNodeMatrix, kind='mergesort')
            adjacentNodeMatrix = np.trim_zeros(adjacentNodeMatrix)

            #Dictionary containing respective distances
            adjacentNodeDistanceDict = {}

            #Stores distances of each node to their eight neighbors in a dictionary
            for i in adjacentNodeMatrix:
                adjacentNodeElevation = nodeElevationDict[i]
                currentNodeElevation = nodeElevationDict[boundedNodeMatrix[row]
                                                         [column]]
                peakXY = XYCoordDict[regionHighPt[0]]
                nodeXY = XYCoordDict[i]
                distanceHeuristic = np.sqrt((peakXY[0] - nodeXY[0])**2 +
                                            (peakXY[1] - nodeXY[1])**2)
                #Distance modified to account for human energy conserving behavior
                adjacentNodeDistanceDict.update({
                    i: ((adjacentNodeElevation - currentNodeElevation)**2 *
                        distanceHeuristic)
                })

            adjacentNodeDict.update(
                {boundedNodeMatrix[row][column]: adjacentNodeDistanceDict})

    nodeMap = adjacentNodeDict

    return nodeMap, numRows, numCols
コード例 #33
0
def createPickleFromRawData(params, writeToFile=False):
    data = pd.read_csv('electricity/LD2011_2014.txt',
                       sep=";",
                       index_col=0,
                       parse_dates=True)
    data.index = data.index - pd.DateOffset(minutes=5)
    num_timeseries = data.shape[1]
    data = data.resample('H').mean()
    # xF = ['Hour', 'DayOfWeek']
    clients = data.columns[0:370].values.tolist()
    data['Hour'] = data.index.hour  # - 1
    data['Day'] = data.index.day - 1
    data['Month'] = data.index.month - 1
    data['DayOfWeek'] = data.index.dayofweek
    data['Weekend'] = ((data['DayOfWeek'] == 5) | (data['DayOfWeek'] == 6)) * 1
    xF = ['Hour', 'Day', 'Month', 'DayOfWeek', 'Weekend']
    if params.isTsId: xF += ['tsId']
    XX = list()
    lagXX = list()
    YY = list()
    for i, client in enumerate(clients):
        print(i)
        ts = np.trim_zeros(data.loc[:, client], trim='f')
        ts = ts.to_frame()

        lags = [{'hours': 1}]
        if params.isLagFeat:
            lags += [{'weeks': 1}, {'weeks': 2}, {'years': 1}]
        lagX = list()
        for lag in lags:
            lagX.append(ts.iloc[:, 0].get(
                pd.DatetimeIndex(ts.index) -
                pd.DateOffset(**lag)).fillna(0).reset_index(drop=True))
        lagX = pd.concat(lagX, axis=1, ignore_index=True)
        lagX = lagX.values.tolist()
        lagXX.append(lagX)

        Y = ts.values.tolist()
        YY.append(Y)

        # ts['Hour'] = pd.DatetimeIndex(ts.index).hour
        # ts['Day'] = pd.DatetimeIndex(ts.index).day - 1
        # ts['Month'] = pd.DatetimeIndex(ts.index).month - 1
        # ts['Year'] = pd.DatetimeIndex(ts.index).year - 2011
        # ts['DayOfWeek'] = pd.DatetimeIndex(ts.index).dayofweek
        # ts['Weekend'] = ((ts['DayOfWeek'] == 5) | (ts['DayOfWeek'] == 6)) * 1
        if params.isTsId:
            ts['tsId'] = i
        X = data.loc[ts.index, xF].values.tolist()
        XX.append(X)


#        print(X[:10])
#        print('---------')
#        print(lagX[:10])
#        print('---------')
#        print(Y[:10])
#        print('------------------------')

# Create feature_dict and emb_dict
    features = xF
    feature_dict = OrderedDict([(x, i) for i, x in enumerate(features)])
    emb_list = OrderedDict([('Hour', (24, 9)), ('Day', (31, 10)),
                            ('month', (12, 6)), ('DayOfWeek', (7, 4))])

    if params.isTsId:
        emb_list.append(('tsId', (370, 20)))
    #('minute', (4, 2)),
    #('hour', (24, 9)),
    #('day', (31, 10)),
    #('month', (12, 6)),
    #('year', (5, 3)),
    #('dayofweek', (7, 4)),
    #('tsId', (370, 20)),
    emb_dict = OrderedDict(emb_list)

    if writeToFile:
        with open('datasets/electricity.pkl', 'wb') as f:
            pickle.dump(XX, f)
            pickle.dump(lagXX, f)
            pickle.dump(YY, f)
            pickle.dump(feature_dict, f)
            pickle.dump(emb_dict, f)

    return XX, lagXX, YY, feature_dict, emb_dict
コード例 #34
0
args.directories.sort(
)  #sort directories once, so they will be in the same order when we read images and commands
time_format = '%Y-%m-%d_%H-%M-%S'
trainstart = datetime.datetime.now()
time_string = trainstart.strftime(time_format)

steer = np.array([])  #this is where we store the steering training data
data_lengths = [
]  #this will hold the lengths of each file of steering data after zeros are trimmed.
#this loops through the input directories, and then through the files in the directories to load in the steering data:
for directory in args.directories:
    ctlfiles = glob.glob(os.path.join(directory, 'commands*.npz'))
    for ctlfile in sorted(ctlfiles):
        ctldata = np.load(ctlfile)['arr_0']
        data_to_append = np.trim_zeros(ctldata[:, 0], trim='b')
        data_lengths.append(len(data_to_append))
        steer = np.concatenate(
            (steer, data_to_append[args.delay:len(data_to_append)]),
            axis=0)  #note that we compensate for delay here

#use these values to normalize target data before training
steerSampleMean = steer.mean()
steerSampleSTD = steer.std()
#these values get saved to un-normalize network output during testing
np.savez("steerstats.npz", [steerSampleMean, steerSampleSTD])

#defines image size:
nrows = 78
ncols = 128
コード例 #35
0
    def construct(self):
        

        LineAnim, LineObjects = linInterp(0,N)
        
        triList = []
        labList = []
        triLabTups = np.zeros(N, dtype=object)
        triLabGroups = np.zeros(N, dtype=object)
        
        for n in range(1,N):
           triLabTups[n-1] = [mktri(n),mkLab(n)]
           triLabGroups[n-1] = Group(mktri(n), mkLab(n))
           
           triList.append( ShowCreation( triLabTups[n-1][0] ) )
           labList.append( ShowCreation( triLabTups[n-1][1] ) )
           
        label = TexMobject('f(x)  =  \\sqrt{x}', color=color).move_to([4.5,3,0])
        label.scale(1.25)
        LineAnim.append(FadeIn(label))
        
        self.play(*triList)
        self.wait(1)
        self.play(*labList)
        self.wait(1)
        self.play(*LineAnim)
        self.wait(1)
        

        
        
        flipList = []
        rotList = []
        centList = []
        finRotList = []
        for n in range(1,len(triLabGroups)-1):
            obj2 = triLabGroups[n]
            tri = triLabTups[n]
            
            flipList.append(flipAnim(obj2[0]))
            flipList.append( ApplyMethod(obj2[1].shift, .5*LEFT) )
            rotList.append(rotAnim(obj2,n))
            centList.append(centerAtOrigin(obj2, n))
            
            
        
        tupArray = np.array(triLabTups).flatten()
        tupArray = np.trim_zeros(tupArray)      
        
        
        i = 0
        for ele in tupArray:
            if i==0:
                i+=1
                continue
            for el in ele:
                self.remove(el)
                i+=1
        
        self.play(*flipList, run_time=.8)
        self.wait(1)
        self.play(*rotList, run_time=.8)
        self.wait(1)
        self.play(*centList, run_time=.8)
        self.wait(1)
        
        
        
        fade = [FadeOut(label)]
        for li in LineObjects:
            fade.append(FadeOut(li))
            
            
        
        
        self.play(*fade)
        
        self.wait(.2)
        
        
        
        self.remove(triLabTups[0][0] )
        self.remove(triLabTups[0][1] )
        
        
        toGroup = np.array(triLabGroups).flatten()
        toGroup = np.trim_zeros(toGroup)      
        initGroup = Group(*toGroup)
        
        self.play(ApplyMethod(initGroup.shift, [7,1.5,0]) , run_time=.75)
        self.wait(.5)
        
        for n in range(1,len(triLabGroups)-1):
            obj2 = triLabGroups[n]
            tri = triLabTups[n]    
            finRotList.append(rotToPlace(obj2,n, about=toGroup[0].get_corner(DOWN+LEFT)))
        
        self.play(*finRotList, run_time=.7)
        
        self.wait(4)
        
        
        self.play(FadeOut(initGroup))
コード例 #36
0
def prio_and_senio(parent_theory, f, ndist, do_architecture):
    """Get the arm length prob. distr. and priority vs seniority form C and save it in the
    theory DataTable"""
    tt = parent_theory.tables[f.file_name_short]
    # arm length
    lgmax = np.log10(react_dist[ndist].contents.arch_maxwt * 1.01)
    lgmin = np.log10(react_dist[ndist].contents.monmass / 1.01)
    num_armwt_bin = react_dist[ndist].contents.num_armwt_bin
    lgstep = (lgmax - lgmin) / (1.0 * num_armwt_bin)
    tmp_x = np.power(
        10,
        [lgmin + ibin * lgstep - 0.5 * lgstep for ibin in range(1, num_armwt_bin + 1)],
    )
    tmp_y = [
        react_dist[ndist].contents.numin_armwt_bin[ibin]
        for ibin in range(1, num_armwt_bin + 1)
    ]
    # trim right zeros
    tmp_y = np.trim_zeros(tmp_y, "b")
    new_len = len(tmp_y)
    tmp_x = tmp_x[:new_len]
    # trim left zeros
    tmp_y = np.trim_zeros(tmp_y, "f")
    new_len = len(tmp_y)
    tmp_x = tmp_x[-new_len:]

    tt.extra_tables["proba_arm_wt"] = np.zeros((new_len, 2))
    tt.extra_tables["proba_arm_wt"][:, 0] = tmp_x
    tt.extra_tables["proba_arm_wt"][:, 1] = tmp_y
    # normalize
    try:
        tt.extra_tables["proba_arm_wt"][:, 1] /= tt.extra_tables["proba_arm_wt"][
            :, 1
        ].sum()
    except ZeroDivisionError:
        pass

    # number of branch points branch point
    max_num_br = react_dist[ndist].contents.max_num_br

    # if max_num_br < 100:
    rmax = min(max_num_br + 1, pb_global_const.MAX_NBR)
    tt.extra_tables["proba_br_pt"] = np.zeros((max_num_br + 1, 2))
    tt.extra_tables["proba_br_pt"][:, 0] = np.arange(max_num_br + 1)
    tt.extra_tables["proba_br_pt"][:, 1] = [
        react_dist[ndist].contents.numin_num_br_bin[i] for i in range(max_num_br + 1)
    ]
    try:
        tt.extra_tables["proba_br_pt"][:, 1] /= tt.extra_tables["proba_br_pt"][
            :, 1
        ].sum()
    except ZeroDivisionError:
        pass
    # else:
    #     # bin the data
    #     tmp_x = list(np.arange(max_num_br + 1))
    #     tmp_y = [react_dist[ndist].contents.numin_num_br_bin[i] for i in range(max_num_br + 1)]
    #     hist, bin_edge = np.histogram(tmp_x, bins=20, weights=tmp_y, density=True)
    #     tt.extra_tables['proba_br_pt'] = np.zeros((len(hist), 2))
    #     tt.extra_tables['proba_br_pt'][:, 0] = np.diff(bin_edge) / 2 + bin_edge[:-1]
    #     tt.extra_tables['proba_br_pt'][:, 1] = hist

    if not do_architecture:
        return
    # P&S
    max_prio = return_max_prio()
    max_senio = return_max_senio()

    avarmlen_v_senio = [
        return_avarmlen_v_senio(ct.c_int(s), ct.c_int(ndist))
        for s in range(1, max_senio + 1)
    ]
    avarmlen_v_prio = [
        return_avarmlen_v_prio(ct.c_int(p), ct.c_int(ndist))
        for p in range(1, max_prio + 1)
    ]

    avprio_v_senio = [
        return_avprio_v_senio(ct.c_int(s)) for s in range(1, max_senio + 1)
    ]
    avsenio_v_prio = [
        return_avsenio_v_prio(ct.c_int(p)) for p in range(1, max_prio + 1)
    ]

    proba_senio = [return_proba_senio(ct.c_int(s)) for s in range(1, max_senio + 1)]
    proba_prio = [return_proba_prio(ct.c_int(p)) for p in range(1, max_prio + 1)]

    tt.extra_tables["avarmlen_v_senio"] = np.zeros((max_senio, 2))
    tt.extra_tables["avarmlen_v_senio"][:, 0] = np.arange(1, max_senio + 1)
    tt.extra_tables["avarmlen_v_senio"][:, 1] = avarmlen_v_senio[:]

    tt.extra_tables["avarmlen_v_prio"] = np.zeros((max_prio, 2))
    tt.extra_tables["avarmlen_v_prio"][:, 0] = np.arange(1, max_prio + 1)
    tt.extra_tables["avarmlen_v_prio"][:, 1] = avarmlen_v_prio[:]

    tt.extra_tables["avprio_v_senio"] = np.zeros((max_senio, 2))
    tt.extra_tables["avprio_v_senio"][:, 0] = np.arange(1, max_senio + 1)
    tt.extra_tables["avprio_v_senio"][:, 1] = avprio_v_senio[:]

    tt.extra_tables["avsenio_v_prio"] = np.zeros((max_prio, 2))
    tt.extra_tables["avsenio_v_prio"][:, 0] = np.arange(1, max_prio + 1)
    tt.extra_tables["avsenio_v_prio"][:, 1] = avsenio_v_prio[:]

    tt.extra_tables["proba_senio"] = np.zeros((max_senio, 2))
    tt.extra_tables["proba_senio"][:, 0] = np.arange(1, max_senio + 1)
    tt.extra_tables["proba_senio"][:, 1] = proba_senio[:]

    tt.extra_tables["proba_prio"] = np.zeros((max_prio, 2))
    tt.extra_tables["proba_prio"][:, 0] = np.arange(1, max_prio + 1)
    tt.extra_tables["proba_prio"][:, 1] = proba_prio[:]
コード例 #37
0
def plot_meanCF(data,
                clusters,
                ylim=[0.0, 90],
                centroids=None,
                order=None,
                ylab='',
                xlab='',
                title=None):
    '''
    plot the mean per cluster or the centroid and the percentiles

    data: the dataset N*M as a numpy array
    centroids: centroids as a 2D numpy array N*M*K
    clusters: list of index of assignment (assigned)
    date: the date to ve written on the file
    ylim: limits of the y axis
    name: the name of the output file   
    '''

    maxClusters = max(clusters) + 1

    #days=['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
    nonZeroMax = np.max(np.count_nonzero(data, axis=1))
    x = np.arange(0, nonZeroMax + 2, 1)
    if divmod(maxClusters, 5)[1] != 0:
        line = int(math.ceil(max(clusters) / 5) + 1)
    else:
        line = int(math.ceil(int((maxClusters) / 5)))
    #print('line'+str(line))
    f, axs = plt.subplots(line,
                          5,
                          figsize=(5 * 5, 5 * line),
                          sharex='col',
                          sharey='row')
    f.subplots_adjust(hspace=.2, wspace=.05)
    axs = axs.ravel()

    for i in range(maxClusters):
        #print(i)
        #axs[i].axis('off')
        if order is not None:
            i2 = order[i]
        else:
            i2 = i

        try:
            Y = np.array([
                np.concatenate([[0],
                                np.pad(np.trim_zeros(data[int(j)]),
                                       (0,
                                        int(nonZeroMax + 1 -
                                            len(np.trim_zeros(data[int(j)])))),
                                       'constant')])
                for j in np.where(clusters == i2)[0]
            ],
                         dtype=float)

            if centroids is None:
                centroid = np.nanmedian(Y, axis=0)
            else:
                centroid = centroids[i2]

            if i >= (line - 1) * 5.:
                # print(i)
                axs[i].set_xlabel(xlab, size=20)
                for tick in axs[i].xaxis.get_major_ticks():
                    tick.label.set_fontsize(14)
            if i % 5 == 0:

                axs[i].set_ylabel(ylab, size=20)
                for tick in axs[i].yaxis.get_major_ticks():
                    tick.label.set_fontsize(14)
            axs[i].set_ylim(ylim)

            axs[i].set_title('Cluster: ' + str(i2) + '; Size: ' + str(len(Y)),
                             size=15)

            #print(np.shape(Y))

            if len(Y) > 1:
                for per in zip([2.5, 5, 10, 25], [97.5, 95, 90, 75]):
                    axs[i].fill_between(x,
                                        np.nanpercentile(Y, per[0], axis=0),
                                        np.nanpercentile(Y, per[1], axis=0),
                                        color="#3F5D7D",
                                        alpha=0.3)
                axs[i].plot(x, centroid, lw=1, color="white")
            else:
                axs[i].plot(x, centroid, lw=1, color="#3F5D7D")
        except ValueError:
            print('skip: ' + str(i))
    #day= datetime.strptime(date,'%Y-%m-%d').weekday()
    #f.suptitle('date = '+date+' ('+days[day]+')',size=15)
    if title == None:
        f.savefig('./gmm-K' + str(maxClusters) + '.png', bbox_inches='tight')
    else:
        f.savefig(title + '.png', bbox_inches='tight')
    plt.clf()
    plt.close()
コード例 #38
0
def clean_target_from_padding(target: np.ndarray):
    """Helper function to remove the padding and put the target array in easy to use format"""
    return [np.trim_zeros(x.flatten(), 'b') for x in target]
コード例 #39
0
    def visualize(self, s, v, a, nt, dt):
        """
        Parameters
        ----------
        s : list
            the position of the rocket
        v : list
            the velocity of the rocket
        a : list
            the acceleration of the rocket
        nt : int
            the number of time steps
        dt : int
            the time between each time step
        """
        # Set up subplots
        f, axs = plt.subplots(nrows=3, ncols=1, sharex=True, figsize=(14,9))
        f.tight_layout(pad=3.0)
        f.set_facecolor('w')

        # Set up common axis and title names
        f.text(0.5, 0.02, 'Time During Docking [sec.]', ha='center', fontsize=14)
        f.text(0.5, 0.96, 'Rocket Docking Model', ha='center', fontsize=16)

        # Set up individual axis titles
        axs[0].set_ylabel('Altitude [km]')
        axs[0].set_title('Position vs. Time')

        axs[1].set_ylabel('Velocity [km/s]')
        axs[1].set_title('Velocity vs. Time')

        axs[2].set_ylabel('G Force')
        axs[2].set_title('Acceleration vs. Time')


        # Remove trailing 0s if rocket reached terminal velocity by tmax
        if s[-1] <= 0:
            s, v, a = map(lambda x: np.trim_zeros(x, 'b'), [s, v, a])

        # Calculate common x values and g forces throughout launch
        x = np.arange(0, int(len(s)*dt), dt)
        gs = self.calculate_g(a)

        # Combine data from launch into only a few variables
        axes = [axs[0], axs[1], axs[2]]
        data = [s, v, gs]
        scale_factors = [1000, 1000, 1]
        colors = ['b', 'g', 'r']

        # Set the x and y axis values for each subplot
        for ax, d, sf in zip(axes, data, scale_factors):
            if axes == axs[2]:
                break
            ax.set_ylim(0, 1.3 * np.amax(d) / sf)
            ax.set_xlim(0, np.amax(x))
        axs[1].set_ylim(-15 , 0)
        axs[2].set_ylim(-5, 10)
        axs[2].set_xlim(0, np.amax(x))

        # Pseudo-animate the launch
        for i in range(0, len(x)):
            for ax, d, sf, color in zip(axes, data, [1000, 1000, 1], colors):
                ax.plot(x[:i], d[:i] / sf, color)
            plt.pause(0.0001)
        plt.show()
コード例 #40
0
def main():
    # print command line arguments
    if len(sys.argv) <= 2:
        print('usage:\n./plotHM.py heapProfileHM(R/W).out heatmap_folder')
        exit()
    filename = sys.argv[1]
    outFolder = sys.argv[2]
    try:
        os.mkdir(outFolder)
    except OSError as exc:
        if exc.errno != errno.EEXIST:
            raise
        pass
    ometaF = open(outFolder+"/meta.txt", "a")

    mat = None
    mat_name = None
    with open(filename) as fp:
        line = fp.readline()
        cnt = 1
        while line:
            if line.strip().find("res") != -1 or line.strip().find("size-limit") != -1:
                print(line)
                ometaF.write(line)
            elif line.strip(' ') == '\n':
                # do nothing, filter out
                pass
            elif line.strip().find("fs") != -1:
                if mat is not None:
                    plt.xlabel('mem addr')
                    plt.ylabel('time')
                    plt.imshow(mat, cmap='hot', interpolation='nearest')
                    # estimate locality, is this formula right?
                    avg_byts_per_bucket = np.sum(mat)/np.size(mat, 1)
                    total_0_axis = np.sum(mat, axis=0)
                    locality_0_aixs = np.absolute(total_0_axis - avg_byts_per_bucket)
                    # Or ?
                    # locality_0_aixs = total_0_axis - avg_byts_per_bucket * 0.95
                    # locality_0_aixs[locality_0_aixs < 0] = 0
                    est_locality = np.sum(locality_0_aixs)/np.sum(mat)
                    
                    plt.savefig(outFolder+"/"+mat_name+"_"+format(est_locality, '.3f')+".png", dpi=1000)

                mat_name = line.strip()
                # create new array
                mat = None
            else:
                col = (line.strip('\t\n').replace("\t"," "))
                col = list(map(int, col.split()))
                col = np.trim_zeros(col, 'b')
                col = np.ndarray((1,len(col)), buffer=np.array(col))
                if mat is None:
                    mat = col
                else:
                    if col.shape[1]> mat.shape[1]:
                        zero_mat = np.zeros((mat.shape[0], col.shape[1]-mat.shape[1]))
                        mat = np.concatenate([mat, zero_mat], axis=1)
                    elif col.shape[1] < mat.shape[1]:
                        pad_zeros = np.zeros(mat.shape[1] - col.shape[1])
                        pad_zeros = np.ndarray((1,len(pad_zeros)), buffer=pad_zeros)
                        col = np.concatenate([col, pad_zeros], axis=1)
                    else:
                        pass
                    mat = np.vstack([mat, col])
            line = fp.readline()
            cnt += 1
            if cnt % 1e3 == 0:
                print('[%d] 1k line pass' % (cnt/1e3))

    # handle the end of file
    if mat is not None:
        plt.xlabel('mem addr')
        plt.ylabel('time')
        plt.imshow(mat, cmap='hot', interpolation='nearest')
        # estimate locality, is this formula right?
        avg_byts_per_bucket = np.sum(mat)/np.size(mat, 1)
        total_0_axis = np.sum(mat, axis=0)
        locality_0_aixs = np.absolute(total_0_axis - avg_byts_per_bucket)
        # Or ?
        # locality_0_aixs = total_0_axis - avg_byts_per_bucket * 0.95
        # locality_0_aixs[locality_0_aixs < 0] = 0
        est_locality = np.sum(locality_0_aixs)/np.sum(mat)
        
        plt.savefig(outFolder+"/"+mat_name+"_"+format(est_locality, '.3f')+".png", dpi=1000)
    ometaF.close()
コード例 #41
0
find_rid_targets(local_data_dir_path)

timeseries = []
i = 0

for n in rid_targets:
    j = 0
    for p in rid_targets[n]:

        data = pd.read_csv(p, index_col=0, parse_dates=['Timestamp'])
        data.columns = [n]

        data_kw = data.resample(freq).sum()

        if j == 0:
            timeseries.append(np.trim_zeros(data_kw.iloc[:, 0], trim='f'))
        else:
            timeseries[i] = timeseries[i].append(np.trim_zeros(data_kw.iloc[:, 0], trim='f'))
        j += 1
    i += 1


# Let us plot the resulting time series for the first ten customers for the time period spanning the first two weeks of 2014.

fig, axs = plt.subplots(1, 2, figsize=(30, 20), sharex=True)
axx = axs.ravel()
for i in range(0, 1):
    timeseries[i].loc["2019-11-01":"2019-11-14"].plot(ax=axx[i])
    axx[i].set_xlabel("Timestamp")
    axx[i].set_ylabel("kW consumption")
plt.show()
コード例 #42
0
def calc_flux_variance(data, model, err_binsize):
    kernel = Box1DKernel(err_binsize)
    errors = convolve((data - model)**2, kernel, boundary=None)
    errors = errors
    errors = np.trim_zeros(errors)
    return errors
コード例 #43
0
def adaptive_peak_finder(
    signal,
    vert_offset='unspecified',
    window_length='unspecified',
    min_peak_distance='unspecified',
    min_amount_from_threshold='unspecified'
):  # if threshold isnt specified it defaults it is replaced with the mean of the signal

    # -------------------------------------------  IMPORTED MODULES -----------------------------------------------
    from numpy import array, int_, append, zeros, ceil, trim_zeros

    # -------------------------------------------------------------------------------------------------------------
    # Checking inputs

    if vert_offset == 'unspecified':
        print(
            'No vert_offset was entered, therefore default of 0 will be used making the threshold = moving mean'
        )
        vert_offset = 0

    if window_length == 'unspecified':
        print(
            'ERROR, no window_length was provided, if you dont want to update the threshold use the basic_peak_finder function'
        )
        window_length = int(input('Please re-enter the window length: '))
        return (adaptive_peak_finder(signal, vert_offset, window_length))

    if window_length % 2 == 0:
        print(
            'ERROR, an even number was entered for the window length, the window length should be odd'
        )
        window_length = int(input('Please re-enter the window length: '))
        return (adaptive_peak_finder(signal, vert_offset, window_length))

    # Initializing varibles (NB when initializing, the most peaks possible would come from a sawtooth wave
    # where every point was either a peak or trough, therefore arrays are initialized to half the signal
    # input length)
    peak_index = zeros(
        int(ceil(len(signal) / 2)), dtype=int_
    )  # initializing a peak_index array where the index of the peak in the signal array will be stored as integers
    peak_value = zeros(
        int(ceil(len(signal) / 2))
    )  # initializing a peak_value array where the peak values will be stored
    m = 0  # m is the iterator that will step through peak_ arrays indices filling the array with the peak locations and values

    ignored_peak_index = zeros(
        int(ceil(len(signal) / 2)), dtype=int_
    )  # initializing an ignored_peak_index array where the index of the peak that didnt meet the min distance requirement between it and the previous peak are stored.
    ignored_peak_value = zeros(
        int(ceil(len(signal) / 2))
    )  # initializing a ignored_peak_value array where the ignored peak values that didnt meet the min distance requirement between it and the previous peak will be stored
    k = 0  # k is the iterator that will step through ignored_peak_ arrays indices filling the array with the ignored peak locations and values

    thresholds = zeros(len(signal) - 1)
    thresholds[0] = (
        signal[0:int((window_length - 1) / 2) + 1].mean()
    ) + vert_offset  # creating a list to store the thresholds as these can be graphed or used for debugging
    # NB the first threshold point is also initilized within.

    for i in range(
            1, signal.size - 1
    ):  # NB the reason we start at i=1 is because the threshold list is initilized with the threshold for i=0 in it, see above.
        # Also the reason we stop at signal.size-1 is we need to stop and not go through the loop for the final element in signal
        # because inside the loop we look at signal[i+1] which doesnt exist if i = final element in signal.

        # Calculating threshold at each point
        if i <= (window_length - 1) / 2:
            window_mean = (
                signal[0:i + (int((window_length - 1) / 2)) + 1].mean()
            )  # NB we need to add one to the stop point as we want to include index i+(window-1)/2 in the window mean calc
            thresholds[i] = window_mean + vert_offset

        elif i >= signal.size - (window_length - 1) / 2:
            window_mean = (
                signal[i - int((window_length - 1) / 2):signal.size].mean()
            )  # NB we dont need to add one to the stop point as a stop point of signal.size = len(signal) defines the stop point as the end of signal
            thresholds[
                i] = window_mean + vert_offset  # Incase the threshold is -ve, we need to work out the threshold this way rather than just window_mean*multiple of mean

        else:
            window_mean = (
                signal[i - int((window_length - 1) / 2):i +
                       (int((window_length - 1) / 2)) + 1].mean()
            )  # NB we need to add one to the stop point as we want to include index i+(window-1)/2 in the threshold calc
            thresholds[
                i] = window_mean + vert_offset  # Incase the threshold is -ve, we need to work out the threshold this way rather than just window_mean*multiple of mean

        # With the threshold calculated we now check to see if the current point meets the criteria to be a peak
        if signal[i] >= thresholds[i] and signal[i] > signal[
                i -
                1]:  # checking that the current point of interest is above the threshold and above its previous neighbour
            j = 1

            while signal[i] == signal[
                    i +
                    j]:  # If you have a point in the signal consisting of multiple signals, to see if you have a peak you must see if the point AFTER the flat part of the signal decreases!
                j = j + 1  # to do this we must step along the flat part of the signal until the gradient of the signal changes. then as seen below we check to see if the point at the change
                # in gradient is lower than our flat section. If it is, then the flat section is infact a peak. The index of this peak is considered to be the left most point in the flat section.

            # Second criteria to be a peak is that the neighbouring point must be less than signal[i]
            if signal[i] > signal[
                    i +
                    j]:  # signal[i+j] will be the first point after signal[i] that isnt equal to signal[i]. see earlier notes for while statement

                if min_peak_distance != 'unspecified' and peak_index[
                        0] != 0 and i - peak_index[
                            m -
                            1] < min_peak_distance:  # this condition is only true if min_peak_distance has been specified and and if it is far enough from the previous positively identified peak we have found at least one previous peak needed to compare the min_peak_distance to.
                    ignored_peak_index[
                        k] = i  # we store the ignored peak data incase the user wants to check what was ignored in the signal
                    ignored_peak_value[k] = signal[i]
                    k = k + 1  # k is used to step through ignored_peak arrays as an index is filled

                elif min_amount_from_threshold != 'unspecified' and abs(
                        signal[i] - thresholds[i]
                ) < min_amount_from_threshold:  # we can set an amount a peak must be above the threshold for it to be a true peak, this is handy for when a signal may jump around the threshold a tiny amount and may not be an actual peak.
                    ignored_peak_index[
                        k] = i  # we store the ignored peak data incase the user wants to check what was ignored in the signal
                    ignored_peak_value[k] = signal[i]
                    k = k + 1  # k is used to step through ignored_peak arrays as an index is filled

                else:  # You need the peak_index and peak_value twice, once for if min_peak_distance is specified and once for if it isnt
                    peak_index[m] = i
                    peak_value[m] = signal[i]
                    m = m + 1  # m is used to step through peak_ arrays as an index is filled

    # trimming all unfilled indices of peak_index, peak_value, ignored_peak_index and ignored_peak_values
    peak_index = trim_zeros(
        peak_index
    )  # as we cannot possible have a peak at an index location 0, we can trim peak_index and use its length to define the peak_value array. As a peaks value could be 0, so we cant use the trim_zeros function of the peak_value array.
    peak_value = peak_value[:len(
        peak_index
    )]  # trimming the peak_value array to the length of peak_index, as they must be the same length, each index is information on the same peak.

    ignored_peak_index = trim_zeros(
        ignored_peak_index
    )  # the code is such that we cannot possible have an ignored_peak at an index location 0, so we can trim peak_index and use its length to define the ignored_peak_value array. As an ignored_peak_value could be 0, so we cant use the trim_zeros function of that array.
    ignored_peak_value = ignored_peak_value[:len(
        ignored_peak_index
    )]  # trimming the ignored_peak_value array to the length of ignored_peak_index, as they must be the same length, each index is information on the same peak.

    return (
        peak_index, peak_value, ignored_peak_index, ignored_peak_value,
        thresholds
    )  # a list of the thresholds is stored so it can be plotted or used for debugging purposes
コード例 #44
0
import numpy
import matplotlib.pyplot as pyplot

pyplot.figure()
V_all = []
I_all = []
for name in ['semi-rev-v-ramp-1-11.txt', 'semi-rev-v-ramp-11-12.txt', 'semi-rev-v-ramp-11.8-11.9.txt', 'semi-rev-v-ramp-11.89-11.93.txt' ,'semi-rev-v-ramp-11.93-12.txt', 'semi-rev-v-ramp-11.8-13.txt']:
    (T, V, I, ti, ti_V, voltages) = numpy.loadtxt(name)
    V = -numpy.absolute(V)
    I = -numpy.absolute(I)

    ti_V = numpy.trim_zeros(ti_V, 'b')
    voltages = numpy.trim_zeros(voltages, 'b')

    ramp_index = []
    for ti_ramp_pt in ti_V:
        for i, time in enumerate(ti):
            if time >= ti_ramp_pt or time == ti[-1]:
                ramp_index.append(i)
                break

    av_V = []
    av_I = []
    for i in ramp_index[1:]:
        av_V.append(numpy.average(V[i - 10:i]))
        av_I.append(numpy.average(I[i - 10:i]))

    if name == 'semi-rev-v-ramp-11.8-13.txt':
        av_V = av_V[3:]
        av_I = av_I[3:]
    elif name == 'semi-rev-v-ramp-11-12.txt':
コード例 #45
0
ファイル: strippy.py プロジェクト: henryorton/strippy
 def read_clevels(fileName):
     with open(fileName) as o:
         s = o.read().replace('\n', ' ')
     lvls = re.search(r'\#\#\$LEVELS=\s?\(.*\)(.*?)\#\#\$', s).group(1)
     return np.trim_zeros(np.array(lvls.split(), dtype=float))
コード例 #46
0
import numpy as np

a = np.array((0, 0, 0, 1, 2, 3, 0, 2, 1, 0))
np.trim_zeros(a)
np.trim_zeros(a, 'b')
np.trim_zeros([0, 1, 2, 0])
コード例 #47
0
    def forward(self,
                x,
                im_sizes,
                image_offset,
                gt_boxes=None,
                gt_classes=None,
                gt_rels=None,
                proposals=None,
                train_anchor_inds=None,
                return_fmap=False):
        """
        Forward pass for Relation detection
        Args:
            x: Images@[batch_size, 3, IM_SIZE, IM_SIZE]
            im_sizes: A numpy array of (h, w, scale) for each image.
            image_offset: Offset onto what image we're on for MGPU training (if single GPU this is 0)

            parameters for training:
            gt_boxes: [num_gt, 4] GT boxes over the batch.
            gt_classes: [num_gt, 2] gt boxes where each one is (img_id, class)
            gt_rels:
            proposals:
            train_anchor_inds: a [num_train, 2] array of indices for the anchors that will
                                  be used to compute the training loss. Each (img_ind, fpn_idx)
            return_fmap:

        Returns:
            If train:
                scores, boxdeltas, labels, boxes, boxtargets, rpnscores, rpnboxes, rellabels
            If test:
                prob dists, boxes, img inds, maxscores, classes
        """
        s_t = time.time()
        verbose = False

        def check(sl, een, sst=s_t):
            if verbose:
                print('{}{}'.format(sl, een - sst))

        result = self.detector(x,
                               im_sizes,
                               image_offset,
                               gt_boxes,
                               gt_classes,
                               gt_rels,
                               proposals,
                               train_anchor_inds,
                               return_fmap=True)
        check('detector', tt())

        assert not result.is_none(), 'Empty detection result'

        # image_offset refer to Blob
        # self.batch_size_per_gpu * index
        im_inds = result.im_inds - image_offset
        boxes = result.rm_box_priors
        obj_scores, box_classes = F.softmax(
            result.rm_obj_dists[:, 1:].contiguous(), dim=1).max(1)
        box_classes += 1
        # TODO: predcls implementation obj_scores and box_classes

        num_img = im_inds[-1] + 1

        # embed(header='rel_model.py before rel_assignments')
        if self.training and result.rel_labels is None:
            assert self.mode == 'sgdet'

            # only in sgdet mode

            # shapes:
            # im_inds: (box_num,)
            # boxes: (box_num, 4)
            # rm_obj_labels: (box_num,)
            # gt_boxes: (box_num, 4)
            # gt_classes: (box_num, 2) maybe[im_ind, class_ind]
            # gt_rels: (rel_num, 4)
            # image_offset: integer
            result.rel_labels = rel_assignments(im_inds.data,
                                                boxes.data,
                                                result.rm_obj_labels.data,
                                                gt_boxes.data,
                                                gt_classes.data,
                                                gt_rels.data,
                                                image_offset,
                                                filter_non_overlap=True,
                                                num_sample_per_gt=1)
        rel_inds = self.get_rel_inds(result.rel_labels, im_inds, boxes)
        rois = torch.cat((im_inds[:, None].float(), boxes), 1)
        # union boxes feats (NumOfRels, obj_dim)
        union_box_feats = self.visual_rep(result.fmap.detach(), rois,
                                          rel_inds[:, 1:].contiguous())
        # single box feats (NumOfBoxes, feats)
        box_feats = self.obj_feature_map(result.fmap.detach(), rois)
        # box spatial feats (NumOfBox, 4)

        box_pair_feats = self.fuse_message(union_box_feats, boxes, box_classes,
                                           rel_inds)
        box_pair_score = self.relpn_fc(box_pair_feats)

        if self.training:
            # sampling pos and neg relations here for training
            rel_sample_pos, rel_sample_neg = 0, 0
            pn_rel_label, pn_pair_score = list(), list()
            for i, s, e in enumerate_by_image(
                    result.rel_labels[:, 0].data.contiguous()):
                im_i_rel_label = result.rel_labels[s:e].contiguous()
                im_i_box_pair_score = box_pair_score[s:e].contiguous()

                im_i_rel_fg_inds = torch.nonzero(
                    im_i_rel_label[:, -1].contiguous()).squeeze()
                im_i_rel_fg_inds = im_i_rel_fg_inds.data.cpu().numpy()
                im_i_fg_sample_num = min(RELEVANT_PER_IM,
                                         im_i_rel_fg_inds.shape[0])
                if im_i_rel_fg_inds.size > 0:
                    im_i_rel_fg_inds = np.random.choice(
                        im_i_rel_fg_inds,
                        size=im_i_fg_sample_num,
                        replace=False)

                im_i_rel_bg_inds = torch.nonzero(
                    im_i_rel_label[:, -1].contiguous() == 0).squeeze()
                im_i_rel_bg_inds = im_i_rel_bg_inds.data.cpu().numpy()
                im_i_bg_sample_num = min(EDGES_PER_IM - im_i_fg_sample_num,
                                         im_i_rel_bg_inds.shape[0])
                if im_i_rel_bg_inds.size > 0:
                    im_i_rel_bg_inds = np.random.choice(
                        im_i_rel_bg_inds,
                        size=im_i_bg_sample_num,
                        replace=False)

                #print('{}/{} fg/bg in image {}'.format(im_i_fg_sample_num, im_i_bg_sample_num, i))
                rel_sample_pos += im_i_fg_sample_num
                rel_sample_neg += im_i_bg_sample_num

                im_i_keep_inds = np.append(im_i_rel_fg_inds, im_i_rel_bg_inds)
                im_i_pair_score = im_i_box_pair_score[
                    im_i_keep_inds.tolist()].contiguous()

                im_i_rel_pn_labels = Variable(
                    torch.zeros(im_i_fg_sample_num + im_i_bg_sample_num).type(
                        torch.LongTensor).cuda(x.get_device()))
                im_i_rel_pn_labels[:im_i_fg_sample_num] = 1

                pn_rel_label.append(im_i_rel_pn_labels)
                pn_pair_score.append(im_i_pair_score)

            result.rel_pn_dists = torch.cat(pn_pair_score, 0)
            result.rel_pn_labels = torch.cat(pn_rel_label, 0)
            result.rel_sample_pos = torch.Tensor([rel_sample_pos]).cuda(
                im_i_rel_label.get_device())
            result.rel_sample_neg = torch.Tensor([rel_sample_neg]).cuda(
                im_i_rel_label.get_device())

        box_pair_relevant = F.softmax(box_pair_score, dim=1)
        box_pos_pair_ind = torch.nonzero(box_pair_relevant[:, 1].contiguous(
        ) > box_pair_relevant[:, 0].contiguous()).squeeze()

        if box_pos_pair_ind.data.shape == torch.Size([]):
            return None
        #print('{}/{} trim edges'.format(box_pos_pair_ind.size(0), rel_inds.size(0)))
        result.rel_trim_pos = torch.Tensor([box_pos_pair_ind.size(0)]).cuda(
            box_pos_pair_ind.get_device())
        result.rel_trim_total = torch.Tensor([rel_inds.size(0)
                                              ]).cuda(rel_inds.get_device())

        if self.trim_graph:
            # filtering relations
            filter_rel_inds = rel_inds[box_pos_pair_ind.data]
            filter_box_pair_feats = box_pair_feats[box_pos_pair_ind.data]
        else:
            filter_rel_inds = rel_inds
            filter_box_pair_feats = box_pair_feats
        if self.training:
            if self.trim_graph:
                filter_rel_labels = result.rel_labels[box_pos_pair_ind.data]
            else:
                filter_rel_labels = result.rel_labels
            num_gt_filtered = torch.nonzero(filter_rel_labels[:, -1])
            if num_gt_filtered.shape == torch.Size([]):
                num_gt_filtered = 0
            else:
                num_gt_filtered = num_gt_filtered.size(0)
            num_gt_orignial = torch.nonzero(result.rel_labels[:, -1]).size(0)
            result.rel_pn_recall = torch.Tensor(
                [num_gt_filtered / num_gt_orignial]).cuda(x.get_device())
            result.rel_labels = filter_rel_labels
        check('trim', tt())

        # message passing between boxes and relations
        if self.mode in ('sgcls', 'sgdet'):
            for _ in range(self.mp_iter_num):
                box_feats = self.message_passing(box_feats,
                                                 filter_box_pair_feats,
                                                 filter_rel_inds)
            box_cls_scores = self.cls_fc(box_feats)
            result.rm_obj_dists = box_cls_scores
            obj_scores, box_classes = F.softmax(
                box_cls_scores[:, 1:].contiguous(), dim=1).max(1)
            box_classes += 1  # skip background
        check('mp', tt())

        # RelationCNN
        filter_box_pair_feats_fc1 = self.relcnn_fc1(filter_box_pair_feats)
        filter_box_pair_score = self.relcnn_fc2(filter_box_pair_feats_fc1)

        result.rel_dists = filter_box_pair_score
        pred_scores_stage_one = F.softmax(result.rel_dists, dim=1).data

        # filter_box_pair_feats is to be added to memory
        if self.training:
            padded_filter_feats, pack_lengths, re_filter_rel_inds, padded_rel_labels = \
                self.pad_sequence(
                    filter_rel_inds,
                    filter_box_pair_feats_fc1,
                    rel_labels=result.rel_labels
                )
        else:
            padded_filter_feats, pack_lengths, re_filter_rel_inds, padded_rel_inds = \
                self.pad_sequence(
                    filter_rel_inds,
                    filter_box_pair_feats_fc1
                )

        # trimming zeros to avoid no rel in image
        trim_pack_lengths = np.trim_zeros(pack_lengths)
        trim_padded_filter_feats = padded_filter_feats[:trim_pack_lengths.
                                                       shape[0]]
        packed_filter_feats = pack_padded_sequence(trim_padded_filter_feats,
                                                   trim_pack_lengths,
                                                   batch_first=True)
        if self.training:
            trim_padded_rel_labels = padded_rel_labels[:trim_pack_lengths.
                                                       shape[0]]
            packed_rel_labels = pack_padded_sequence(trim_padded_rel_labels,
                                                     trim_pack_lengths,
                                                     batch_first=True)
            rel_mem_dists = self.mem_module(inputs=packed_filter_feats,
                                            rel_labels=packed_rel_labels)
            rel_mem_dists = self.re_order_packed_seq(rel_mem_dists,
                                                     filter_rel_inds,
                                                     re_filter_rel_inds)
            result.rel_mem_dists = rel_mem_dists
        else:
            trim_padded_rel_inds = padded_rel_inds[:trim_pack_lengths.shape[0]]
            packed_rel_inds = pack_padded_sequence(trim_padded_rel_inds,
                                                   trim_pack_lengths,
                                                   batch_first=True)
            rel_mem_dists = self.mem_module(inputs=packed_filter_feats,
                                            rel_inds=packed_rel_inds,
                                            obj_classes=box_classes)
            rel_mem_probs = self.re_order_packed_seq(rel_mem_dists,
                                                     filter_rel_inds,
                                                     re_filter_rel_inds)
            rel_mem_probs = rel_mem_probs.data

        check('mem', tt())
        if self.training:
            return result

        # pad stage one output in rel_mem_probs if it sums zero
        for rel_i in range(rel_mem_probs.size(0)):
            rel_i_probs = rel_mem_probs[rel_i]
            if rel_i_probs.sum() == 0:
                rel_mem_probs[rel_i] = pred_scores_stage_one[rel_i]
        """
        filter_dets
        boxes: bbox regression else [num_box, 4]
        obj_scores: [num_box] probabilities for the scores
        obj_classes: [num_box] class labels integer
        rel_inds: [num_rel, 2] TENSOR consisting of (im_ind0, im_ind1)
        pred_scores: [num_rel, num_predicates] including irrelevant class(#relclass + 1)
        """
        check('mem processing', tt())
        return filter_dets(boxes, obj_scores, box_classes,
                           filter_rel_inds[:, 1:].contiguous(), rel_mem_probs)
コード例 #48
0
def _trim(data, val):
    data_out = np.copy(data)
    return np.trim_zeros(data_out - val) + val
コード例 #49
0
    def compute(self):
        for i in range(self.partition):
            if i >= 2:
                continue
            temptrain_X, temptrain_Y, tempval_X, tempval_Y = self.create_data(
                i)
            #print((temptrain_X[0]), (temptrain_Y[0]))
            self.train_X, self.train_Y = training_set_factorize(
                temptrain_X, temptrain_Y, augment_data=self.augment_data_flag)
            self.val_X, self.val_Y = validation_set_factorize(
                tempval_X, tempval_Y)
            self.val_Y = self.val_Y.long()
            print(i, "phase 1 completed")
            #create dataset
            self.train_X = torch.tensor(self.train_X)
            self.train_Y = torch.tensor(self.train_Y)
            train_set = data_utils.TensorDataset(self.train_X, self.train_Y)
            train_loader = data_utils.DataLoader(dataset=train_set,
                                                 batch_size=BATCH_SIZE,
                                                 drop_last=True,
                                                 shuffle=True)
            cur_model = deepcopy(self.model).to(device)

            ij_recall_history = []
            ij_precision_history = []
            for j in range(self.epochs):
                if j >= 5:
                    continue
                print("epoch %i" % j)
                path = 'cnncrf_train_epoch' + str(j) + '_part' + str(i) + '.pt'
                cur_model.load_state_dict(torch.load(path))
                #validation
                p0t1 = 0
                p0t0 = 0
                p1t1 = 0
                p1t0 = 0
                for j in range(self.val_X.size(0)):
                    print(j, self.val_X.size())
                    val_X = torch.tensor(self.val_X[j])
                    val_Y = torch.tensor(self.val_Y[j])
                    val_X = val_X.reshape(VAL_SEQ_LEN, VAL_BATCH_SIZE,
                                          -1).float().contiguous().to(device)
                    val_Y = val_Y.reshape(
                        VAL_SEQ_LEN, ).long().contiguous().to(device)
                    scores, path = cur_model(val_X)
                    prediction = torch.from_numpy(np.array(path)).reshape(
                        VAL_SEQ_LEN, )
                    prediction = prediction.numpy()
                    val_X = val_X.cpu().numpy()
                    val_Y = val_Y.cpu().numpy()
                    last_index = np.trim_zeros(prediction, 'b').shape[
                        0]  #This line might be buggy!!! might want to use the true size
                    output_Y = prediction[0:last_index]
                    target_Y = val_Y[0:last_index]
                    output_Y[output_Y > 1] = 0
                    target_Y[target_Y > 1] = 0
                    countp0t1, countp0t0, countp1t1, countp1t0 = self.compute_acc(
                        output_Y, target_Y)
                    p0t1 += countp0t1
                    p0t0 += countp0t0
                    p1t1 += countp1t1
                    p1t0 += countp1t0

                if ((p1t1 + p1t0) == 0):
                    ij_precision_history.append(-1)
                else:
                    ij_precision_history.append(p1t1 * 1.0 / (p1t1 + p1t0))

                if ((p1t1 + p0t1) == 0):
                    ij_recall_history.append(-1)
                else:
                    ij_recall_history.append(p1t1 * 1.0 / (p1t1 + p0t1))
            self.precision_history.append(ij_precision_history)
            self.recall_history.append(ij_recall_history)
            print(self.precision_history)
            print(self.recall_history)
        return self.precision_history, self.recall_history, self.loss_history
コード例 #50
0
ファイル: VAD_Proposed.py プロジェクト: aditthapron/VAD
 def np_trim_zeros(x):
     return np.trim_zeros(x)
コード例 #51
0
def inspect(time_leng, pattern_leng, top_print):
    #ファイル1のデータカウント
    pattern_dict1 = {}
    sumpsth = 0
    for i, name in enumerate(sheet_names1):
        sheet_df1[i] = file1.parse(name)
        if(sheet_df1[i] == NaN) :
            break
        #print(sheet_df1[i][1][1])
        start_number = (np.where(sheet_df1[i]['INFORMATION']=="CHANNEL")[0][0]) +cannel_start
        end_number = (np.where(sheet_df1[i]['INFORMATION']=="CHANNEL")[0][1]) - cannel_end
        sig1 = (sheet_df1[i]['Unnamed: 3'][start_number : end_number]).values
        sig1 = sig1.astype("int")
        #print(len(sig1))
        sig1 = np.trim_zeros(sig1)
#        print()
        leng = len(sig1)
        psth = np.zeros(int((leng/time_leng)+1), dtype=np.int)
        l = 0
        for k in range(0, leng, time_leng) :
            psth[l] = sig1[k : k+time_leng].sum()
            l += 1
#        print(sig1[len(psth)-1])
        if(len(psth) > pattern_leng) : 
            for k in range(len(psth) - pattern_leng +1) :   # PSTHデータからはパターンを重ねて検索している
                if (str(psth[k : k + pattern_leng]) in pattern_dict1) : 
                    pattern_dict1[str(psth[k : k + pattern_leng])] += 1
                else : 
                    pattern_dict1[str(psth[k : k + pattern_leng])] = 1
            sumpsth += (len(psth) -pattern_leng+1)
#    print()
#    print(sumpsth)

    #ファイル2のデータカウント
    pattern_dict2 = {}
    sum_dict = pattern_dict1.copy()
    for i, name in enumerate(sheet_names2):
        sheet_df2[i] = file2.parse(name)
        start_number = (np.where(sheet_df2[i]['INFORMATION']=="CHANNEL")[0][0]) + cannel_start
        end_number = (np.where(sheet_df2[i]['INFORMATION']=="CHANNEL")[0][1]) - cannel_end
        sig1 = (sheet_df2[i]['Unnamed: 3'][start_number : end_number]).values
        sig1 = sig1.astype("int")
        sig1 = np.trim_zeros(sig1)
        leng = len(sig1)
#        print(sig1[0])
        psth = np.zeros(int((leng/time_leng)+1), dtype=np.int)
        l = 0
        for k in range(j, leng, time_leng) :
            psth[l] = sig1[k+j : k+j+time_leng].sum()
            l += 1
        if(len(psth) > pattern_leng) : 
            for k in range(len(psth) - pattern_leng +1) :
                if (str(psth[k : k+ pattern_leng]) in pattern_dict2) : 
                    pattern_dict2[str(psth[k : k+ pattern_leng])] += 1
                    sum_dict[str(psth[k : k+ pattern_leng])] += 1
                else : 
                    pattern_dict2[str(psth[k : k+ pattern_leng])] = 1
                    if (str(psth[k : k+ pattern_leng]) in sum_dict) : 
                        sum_dict[str(psth[k : k+ pattern_leng])] += 1
                    else :
                        sum_dict[str(psth[k : k+ pattern_leng])] = 1

            
    if(len(pattern_dict1.keys()) == 0 ):
        return 0
                            
    #情報量の計算準備
    pattern_information =  0.0
#    print()
#    print()
#    print(sum(pattern_dict1.values()))
#    print(sum(sum_dict.values()))
#    print()
    sum_pattern1 = sum(pattern_dict1.values()) + len(sum_dict.keys())
    sum_pattern2 = sum(pattern_dict2.values()) + len(sum_dict.keys())
#    print(sum_pattern1)
#    print(sum_pattern2)
#    sum_pattern = sum(sum_dict.values()) 
    pattern1 = len(pattern_dict1.keys())
    probability1 = np.zeros(pattern1, float)
    probability2 = np.zeros(pattern1, float)
    top_dict = {}
    k = 0
    for i in (pattern_dict1.keys()) :
        probability1[k] = ((pattern_dict1[i]+1) / sum_pattern1)
        if(i in pattern_dict2) : 
            probability2[k] = ((pattern_dict2[i]+1) / sum_pattern2)
        else :
            probability2[k] = (1 / sum_pattern2)
        info = probability1[k] * math.log2(probability1[k]/probability2[k])
        pattern_information += info
        top_dict[i] = info
#        print(i)
        k += 1



    if (len(pattern_dict1.keys()) < top_print) :
        max_print = len(pattern_dict1.keys())
    else :
        max_print =top_print
    
    #グラフ出力の準備
    print_probability1 = np.zeros(max_print, float)
    print_probability2 = np.zeros(max_print, float)
    print_kullback = np.zeros(max_print, float)
    print_pattern = []
    k = 0
    for i, v in sorted(top_dict.items(), key=lambda x:-x[1])[0:max_print] :
        print_probability1[k] = ((pattern_dict1[i]+1) / sum_pattern1)
        if(i in pattern_dict2) : 
            print_probability2[k] = ((pattern_dict2[i]+1) / sum_pattern2)
        else :
            print_probability2[k] = (1 / sum_pattern2)
        print_kullback[k] = print_probability1[k] * math.log2(print_probability1[k]/print_probability2[k]) #kullback項の保存
        print_pattern.append(i)
        k += 1
#        print(i)
#        print_probability2[k] = (pattern_dict2[i]+1 / sum_pattern2)


        
 #   print(print_pattern)
    #グラフ出力
    #パターンの確率比較
    fig = plt.figure(dpi=600)
    ax = fig.gca()
    x = np.arange(len(print_pattern))
    w = 0.4
    plt.bar(x+w, print_probability1, width=w, color="red", label="after")
    plt.bar(x, print_probability2, width=w, color="blue", label="before")
    plt.xlim(-(w/2), max_print-(w/2))
    plt.xlabel("pattern")
    plt.ylabel("probability")
    #plt.xticks(x + w/2, print_pattern)
    plt.xticks(color="None")
    #ax.set_xticklabels(print_pattern, rotation=90)
    plt.legend()
    plt.savefig(str(time_leng) + "-" + str(pattern_leng) + ".png", bbox_inches='tight')
    #パターンのカルバックライブラー項比較
    fig = plt.figure(dpi=600)
    ax = fig.gca()
    plt.bar(print_pattern, print_kullback, color="red", label="after")
    plt.xlim(-0.5, max_print-0.5)
    plt.xlabel("pattern")
    plt.ylabel("probability")
    plt.xticks(color="None")
    #ax.set_xticklabels(print_pattern, rotation=90)
    plt.legend()
    #print(print_kullback)
    #plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', borderaxespad=0, ncol=2)
    #fig.subplots_adjust(bottom=10)
    plt.savefig(str(time_leng) + "-" + str(pattern_leng) + "-kullback.png", bbox_inches='tight')
    plt.close()
    del pattern_dict1, pattern_dict2, sum_dict, probability1, probability2, top_dict, print_pattern
    
    return pattern_information, sum_pattern1, sum_pattern2
コード例 #52
0
            (savings, alpha, gamma, agents))

plt.figure()
plt.plot(cycle, sigma2)
plt.ylabel(r'$\sigma^2_m$')
plt.xlabel('Monte Carlo Cycle')
plt.margins(x=0.1, y=0.1)
plt.tight_layout()
plt.savefig('../fig/sigma_savings_%s_alpha_%s_gamma_%s_agents_%s.pdf' %
            (savings, alpha, gamma, agents))

filename = '../output/wealth_pdf_savings_%s_alpha_%s_gamma_%s_agents_%s.txt' % (
    savings, alpha, gamma, agents)

m, counts = np.genfromtxt(filename, unpack=True)
counts = np.trim_zeros(counts, 'b')
m = m[:len(counts)]
n = 1 + 3 * float(savings) / (1 - float(savings))
pdf = n**n / scipy.special.gamma(n) * m**(n - 1) * np.exp(-n * m)

plt.figure()
plt.plot(m, pdf)
plt.hist(m, weights=counts, bins=m, normed=True)
plt.ylabel('Normalized distribution of agents')
plt.xlabel('Money')
plt.tight_layout()
plt.savefig('../fig/histogram_savings_%s_alpha_%s_gamma_%s_agents_%s.pdf' %
            (savings, alpha, gamma, agents))

m = m + 0.025
コード例 #53
0
def kernelDetection(fluxProfile, kernel, kernels, num):
    """ Detects dimming using Mexican Hat kernel for dip detection and set of Fresnel kernels for kernel matching """
    """ Prunes profiles"""
    light_curve = np.trim_zeros(fluxProfile[1:])
    if len(light_curve) == 0:
        return -2  # reject empty profiles
    med = np.median(light_curve)
    indices = np.where(
        light_curve > min(med * 2, med + 5 * np.std(light_curve)))
    light_curve = np.delete(light_curve, indices)
    trunc_profile = np.where(light_curve < 0, 0, light_curve)
    if len(trunc_profile) < 1100:
        return -2  # reject objects that go out of frame rapidly, ensuring adequate evaluation of median flux
    if abs(np.mean(trunc_profile[:1000]) -
           np.mean(trunc_profile[-1000:])) > np.std(trunc_profile[:1000]):
        return -2  # reject tracking failures
    if np.median(trunc_profile) < 5000:
        return -2  # reject stars that are very dim, as SNR is too poor
    m = np.std(trunc_profile[900:1100]) / np.median(trunc_profile[900:1100])
    """ Dip detection"""
    # astropy v2.0+ changed convolve_fft quite a bit... see documentation, for now normalize_kernel=False
    conv = convolve_fft(trunc_profile, kernel, normalize_kernel=False)
    minPeak = np.argmin(conv)
    minVal = min(conv)

    # if geometric dip (greater than 40%), flag as candidate without template matching
    norm_trunc_profile = trunc_profile / np.median(trunc_profile)
    if norm_trunc_profile[minPeak] < 0.6:
        critTime = np.where(fluxProfile == light_curve[minPeak])[0]
        print datetime.datetime.now(
        ), "Detected >40% dip: frame", str(critTime) + ", star", num
        return critTime[0]

    if 30 <= minPeak < len(trunc_profile) - 30:
        med = np.median(
            np.concatenate((trunc_profile[minPeak - 100:minPeak - 30],
                            trunc_profile[minPeak + 30:minPeak + 100])))
        trunc_profile = trunc_profile[(minPeak - 30):(minPeak + 30)]
        unc = (
            (np.sqrt(abs(trunc_profile) / 100.) / np.median(trunc_profile)) *
            100)
        unc = np.where(unc == 0, 0.01, unc)
        trunc_profile /= med
    else:
        return -2  # reject events that are cut off at the start/end of time series
    bkgZone = conv[10:-10]

    if minVal < np.mean(
            bkgZone) - 3.75 * np.std(bkgZone):  # dip detection threshold
        """ Kernel matching"""
        old_min = np.inf
        for ind in range(0, len(kernels)):
            if min(kernels[ind]) > 0.8:
                continue
            new_min, loc = diffMatch(kernels[ind], trunc_profile, unc)
            if new_min < old_min:
                active_kernel = ind
                min_loc = loc
                old_min = new_min
        unc_l = unc[min_loc:min_loc + len(kernels[active_kernel])]
        thresh = 0
        for u in unc_l:
            thresh += (abs(m)**1) / (abs(u)**1)  # kernel match threshold
        if old_min < thresh:
            critTime = np.where(fluxProfile == light_curve[minPeak])[0]
            print datetime.datetime.now(
            ), "Detected candidate: frame", str(critTime) + ", star", num
            if len(critTime) > 1:
                raise ValueError
            return critTime[
                0]  # returns location in original time series where dip occurs
        else:
            return -1  # reject events that do not pass kernel matching
    else:
        return -1  # reject events that do not pass dip detection
コード例 #54
0
#the sample rate is the number of bits of infomration recorded per second
print(kernelRate)
print(kernelAudData)

#wav bit type the amount of information recorded in each bit often 8, 16 or 32 bit
kernelAudData.dtype

print("Data Shape:")
print(np.shape(kernelAudData))

#wav length
kernel_len_PRI = kernelAudData.shape[0] / rate

#Final kernel
kernel = np.trim_zeros(kernelAudData, 'fb')
plt.plot(kernel)

# In[7]:

# Padding
print('Channel 1 Length:'), print(len(channel1))
print('NSAMPS Length:'), print(NSAMPS)

while np.mod(NSAMPS, RECORD_SECONDS) != 0:
    NSAMPS += 1
    print(NSAMPS)

if len(channel1) > NSAMPS:
    mod_channel1 = channel1[NSAMPS - len(channel1)]
    print("* Shortened *")
コード例 #55
0
ファイル: files-data2.0.py プロジェクト: kitada4010/sbc
def inspect(time_leng, pattern_leng, top_print, count_data):
    #ファイル1のデータカウント
    pattern_dict1 = {}
    sumpsth = 0
    for i, name in enumerate(sheet_names1):
        sheet_df1[i] = file1.parse(name)
        try:
            #print(sheet_df1[i][1][1])
            start_number = (np.where(sheet_df1[i]['INFORMATION']=="CHANNEL")[0][0]) +cannel_start
            end_number = (np.where(sheet_df1[i]['INFORMATION']=="CHANNEL")[0][1]) - cannel_end
            sig1 = (sheet_df1[i]['Unnamed: 3'][start_number : end_number]).values
        except KeyError :
            print("not max data number")
            break
        sig1 = sig1.astype("int")
        #print(len(sig1))
        sig1 = np.trim_zeros(sig1)
        #        print()
        leng = len(sig1)
        psth = np.zeros(int((leng/time_leng)+1), dtype=np.int)
        l = 0
        for k in range(0, leng, time_leng) :
            psth[l] = sig1[k : k+time_leng].sum()
            l += 1
            #        print(sig1[len(psth)-1])
        if(len(psth) > pattern_leng) : 
            for k in range(len(psth) - pattern_leng +1) :   # PSTHデータからはパターンを重ねて検索している
                if (str(psth[k : k + pattern_leng]) in pattern_dict1) : 
                    pattern_dict1[str(psth[k : k + pattern_leng])] += 1
                else : 
                    pattern_dict1[str(psth[k : k + pattern_leng])] = 1
                    sumpsth += (len(psth) -pattern_leng+1)

#    print()
#    print(sumpsth)

    #ファイル2のデータカウント
    pattern_dict2 = {}
    sum_dict = pattern_dict1.copy()
    for i, name in enumerate(sheet_names2):
        sheet_df2[i] = file2.parse(name)
        try :
            start_number = (np.where(sheet_df2[i]['INFORMATION']=="CHANNEL")[0][0]) + cannel_start
            end_number = (np.where(sheet_df2[i]['INFORMATION']=="CHANNEL")[0][1]) - cannel_end
            sig1 = (sheet_df2[i]['Unnamed: 3'][start_number : end_number]).values
        except KeyError :
            print("not max data number")
            break
        sig1 = sig1.astype("int")
        sig1 = np.trim_zeros(sig1)
        leng = len(sig1)
#        print(sig1[0])
        psth = np.zeros(int((leng/time_leng)+1), dtype=np.int)
        l = 0
        for k in range(j, leng, time_leng) :
            psth[l] = sig1[k+j : k+j+time_leng].sum()
            l += 1
        if(len(psth) > pattern_leng) : 
            for k in range(len(psth) - pattern_leng +1) :
                if (str(psth[k : k+ pattern_leng]) in pattern_dict2) : 
                    pattern_dict2[str(psth[k : k+ pattern_leng])] += 1
                    sum_dict[str(psth[k : k+ pattern_leng])] += 1
                else : 
                    pattern_dict2[str(psth[k : k+ pattern_leng])] = 1
                    if (str(psth[k : k+ pattern_leng]) in sum_dict) : 
                        sum_dict[str(psth[k : k+ pattern_leng])] += 1
                    else :
                        sum_dict[str(psth[k : k+ pattern_leng])] = 1

            
    sum_pattern1 = sum(pattern_dict1.values()) #+ len(sum_dict.keys())
    sum_pattern2 = sum(pattern_dict2.values()) #+ len(sum_dict.keys())            
    if(len(pattern_dict1.keys()) == 0) :
        del pattern_dict1, pattern_dict2, sum_dict, leng, psth, start_number, end_number, sig1, i, l, k, sum_pattern1
        if(len(pattern_dict2.keys()) == 0 ):
            gc.collect()
            return 0, 0, 0
        else :
            gc.collect()
            return 0, 0, sum_pattern2
    elif(len(pattern_dict2.keys()) == 0 ):
        del pattern_dict1, pattern_dict2, sum_dict, leng, psth, start_number, end_number, sig1, i, l, k, sum_pattern2
        gc.collect()
        return 0, sum_pattern1, 0
                            
    #情報量の計算準備
    pattern_information =  0.0
#    print()
#    print()
#    print(sum(pattern_dict1.values()))
#    print(sum(sum_dict.values()))
#    print()
    sum_pattern1 = sum(pattern_dict1.values()) + len(sum_dict.keys())
    sum_pattern2 = sum(pattern_dict2.values()) + len(sum_dict.keys())
#    print(sum_pattern1)
#    print(sum_pattern2)
#    sum_pattern = sum(sum_dict.values()) 
    pattern1 = len(pattern_dict1.keys())
    probability1 = np.zeros(pattern1, float)
    probability2 = np.zeros(pattern1, float)
    top_dict = {}
    k = 0
    for i in (pattern_dict1.keys()) :
        probability1[k] = ((pattern_dict1[i]) / sum_pattern1)
        if(i in pattern_dict2) : 
            probability2[k] = ((pattern_dict2[i]+1) / sum_pattern2)
        else :
            probability2[k] = (1 / sum_pattern2)
        info = probability1[k] * math.log2(probability1[k]/probability2[k])
        pattern_information += info
        top_dict[i] = info
#        print(i)
        k += 1



    if (len(pattern_dict1.keys()) < top_print) :
        max_print = len(pattern_dict1.keys())
    else :
        max_print =top_print
    
    #グラフ出力の準備
#    print_probability1 = np.zeros(max_print, float)
#    print_probability2 = np.zeros(max_print, float)
#    print_count1 = np.zeros(max_print, float)
#    print_count2 = np.zeros(max_print, float)
#    print_kullback = np.zeros(max_print, float)
#    print_pattern = []
    k = 0
    #print(time_leng, pattern_leng, sum_pattern1, sum_pattern2, file=count_data)
    for i, v in sorted(top_dict.items(), key=lambda x:-x[1]) :[0:max_print] :
コード例 #56
0
def sample2str(sample):
  trimmed = np.trim_zeros(sample)
  if len(trimmed) and trimmed[-1] == EOS_ID:
    trimmed = trimmed[:-1]
  return " ".join(map(str, trimmed))
コード例 #57
0
def jointSolve(d, orders):
    import pylab
    path = __path__[0]

    lines = {}
    lines['cuar'] = numpy.loadtxt(path + "/data/cuar.lines")
    lines['hgne'] = numpy.loadtxt(path + "/data/hgne.lines")
    lines['xe'] = numpy.loadtxt(path + "/data/xe.lines")

    startsoln = numpy.load(path + "/data/esi_wavesolution.dat")
    startsoln = numpy.load(path + '/data/shao_wave.dat')
    startsoln = [i for i, j in startsoln]
    alldata = d['arc']
    arclist = d.keys()
    arclist.remove('arc')
    soln = []
    if alldata.shape[1] > 3000:
        xvals = numpy.arange(4096.)
        resolve = False
        cuslice = slice(3860, 3940)
        fw1 = 75.
        fw2 = 9
        WIDTH = 4
    else:
        xvals = numpy.arange(2048.)
        resolve = True
        cuslice = slice(1930, 1970)
        fw1 = 37.
        fw2 = 7
        WIDTH = 3
    for i in range(10):
        solution = startsoln[i]
        start, end = orders[i]
        if resolve == True:
            tmp = numpy.arange(0.5, 4096., 2.)
            w = sf.genfunc(tmp, 0., solution)
            solution = sf.lsqfit(numpy.array([xvals, w]).T, 'chebyshev', 3)

        data = numpy.nanmedian(alldata[start:end], axis=0)
        data[numpy.isnan(data)] = 0.
        if i == 0:
            data[cuslice] = numpy.median(data)
        bak = ndimage.percentile_filter(data, 50., int(fw1))
        data -= bak

        peaks = []
        p = ndimage.maximum_filter(data, fw2)
        std = clip(numpy.trim_zeros(data), 3.)[1]
        peak = numpy.where((p > 30. * std) & (p == data))[0]
        for p in peak:
            if p - WIDTH < 0 or p + WIDTH + 1 > xvals.size:
                continue
            x = xvals[p - WIDTH:p + WIDTH + 1].copy()
            f = data[p - WIDTH:p + WIDTH + 1].copy()
            fitdata = numpy.array([x, f]).T
            fit = numpy.array([0., f.max(), xvals[p], 1.])
            fit, chi = sf.ngaussfit(fitdata, fit, weight=1)
            peaks.append(fit[2])

        for converge in range(10):
            wave = 10**sf.genfunc(xvals, 0., solution)

            refit = []
            corr = []
            err = wave[wave.size / 2] - wave[wave.size / 2 - 1]
            p = 10.**sf.genfunc(peaks, 0., solution)
            for arc in arclist:
                for k in range(p.size):
                    if i == 0 and p[k] > 4344.:
                        continue
                    cent = p[k]
                    diff = cent - lines[arc]
                    corr.append(diff[abs(diff).argmin()])
            corr = numpy.array(corr)
            corr = corr[abs(corr) < 5 * err]
            m, s = clip(corr)
            corr = numpy.median(corr[abs(corr - m) < 5. * s])
            for arc in arclist:
                for k in range(p.size):
                    if i == 0 and p[k] > 4344.:
                        continue
                    pos = peaks[k]
                    cent = p[k]
                    diff = abs(cent - lines[arc] - corr)
                    if diff.min() < 2. * err:
                        refit.append([pos, lines[arc][diff.argmin()]])
            refit = numpy.asarray(refit)
            solution = sf.lsqfit(refit, 'polynomial', 3)
            refit = []
            err = solution['coeff'][1]
            p = sf.genfunc(peaks, 0., solution)
            for k in range(p.size):
                delta = 1e9
                match = None
                for arc in arclist:
                    for j in lines[arc]:
                        if i == 0 and j > 4344.:
                            continue
                        diff = abs(p[k] - j)
                        if diff < delta and diff < 1. * err:
                            delta = diff
                            match = j
                if match is not None:
                    refit.append([peaks[k], match])
            refit = numpy.asarray(refit)
            refit[:, 1] = numpy.log10(refit[:, 1])
            solution = sf.lsqfit(refit, 'chebyshev', 3)

            solution2 = sf.lsqfit(refit[:, ::-1], 'chebyshev', 3)

            g = 10**sf.genfunc(peaks, 0., solution)
            g2 = 10**sf.genfunc(peak, 0., solution)
            w = 10**sf.genfunc(xvals, 0., solution)
            if (w == wave).all():
                print("Order %d converged in %d iterations" % (i, converge))
                soln.append([solution, solution2])
                break
                pylab.plot(w, data)
                for arc in arclist:
                    for j in lines[arc]:
                        if j > w[0] and j < w[-1]:
                            pylab.axvline(j, c='b')
                for j in 10**refit[:, 1]:
                    if j > w[0] and j < w[-1]:
                        pylab.axvline(j, c='r')
                for j in g:
                    pylab.axvline(j, c='g')
                for j in g2:
                    pylab.axvline(j, c='c')
                pylab.show()
                break

    return soln
コード例 #58
0
def max_peak_finder(
    signal,
    threshold='unspecified',
    initial_thres='unspecified',
    vert_offset='unspecified',
    window_length='unspecified',
    min_peak_distance='unspecified',
    min_amount_from_threshold='unspecified',
    peak_value_consistancy_fraction='unspecified'
):  # if thershold isnt specified it defaults it is replaced with the mean of the signal

    # -------------------------------------------  IMPORTED MODULES -----------------------------------------------
    from numpy import array, int_, append, argmax, argsort, zeros, ceil, trim_zeros
    # -------------------------------------------------------------------------------------------------------------

    # Checking inputs
    if threshold == 'unspecified':

        if vert_offset == 'unspecified':
            # No vert_offset was entered, therefore default of 0 will be used making the threshold = moving mean
            vert_offset = 0

        if window_length == 'unspecified':
            print(
                'ERROR, no window_length was provided, if you dont want to update the threshold use the basic_peak_finder function'
            )
            window_length = int(input('Please re-enter the window length: '))
            return (max_peak_finder(signal, threshold, vert_offset,
                                    window_length, min_peak_distance,
                                    min_amount_from_threshold,
                                    peak_value_consistancy_fraction))

        if window_length % 2 == 0:
            print(
                'ERROR, an even number was entered for the window length, the window length should be odd'
            )
            window_length = int(input('Please re-enter the window length: '))
            return (max_peak_finder(signal, threshold, vert_offset,
                                    window_length, min_peak_distance,
                                    min_amount_from_threshold,
                                    peak_value_consistancy_fraction))

        if peak_value_consistancy_fraction != 'unspecified' and peak_value_consistancy_fraction <= 0 and peak_value_consistancy_fraction >= 1:
            print(
                'ERROR, peak_value_consistancy_persentage received an input that was not a fraction between 0 and 1'
            )
            peak_value_consistancy_fraction = int(
                input('Please re-enter the fraction: '))
            return (max_peak_finder(signal, threshold, vert_offset,
                                    window_length, min_peak_distance,
                                    min_amount_from_threshold,
                                    peak_value_consistancy_fraction))

    elif threshold != 'unspecified' and vert_offset != 'unspecified' or window_length != 'unspecified':
        print(
            'ERROR! Threshold has been defined as a constant, but so too has one or more of the adaptive arguments: vert_offset or window_length'
        )
        print(
            'If the threshold is fixed as a constant the adaptive arguments are not needed and should be left undefined'
        )
        return ()

    # Initilzing varibles
    ignored_peak_index = zeros(
        int(ceil(len(signal) / 2)), dtype=int_
    )  # initializing an ignored_peak_index array where the index of the peak that didnt meet the min distance requirement between it and the previous peak are stored
    ignored_peak_value = zeros(
        int(ceil(len(signal) / 2))
    )  # initializing a ignored_peak_value array where the ignored peak values that didnt meet the min distance requirement between it and the previous peak will be stored
    m = 0  # m is an iterator that ensures once an index is filled in the ignored_peak_ arrays, the next ignored_peak_ is placed in the next empty index location.

    ignored_max_index = zeros(
        int(ceil(len(signal) / 2)), dtype=int_
    )  # initializing an ignored_max_index array where indexs of max values that werent actually local peaks are stored for debugging
    k = 0  # k is an iterator that moves through ignored_max_index array. see later

    max_peak_index = zeros(
        int(ceil(len(signal) / 2)), dtype=int_
    )  # initializing a max_peak_index array where the index of the peak in the signal array will be stored as integers
    max_peak_value = zeros(
        int(ceil(len(signal) / 2))
    )  # initializing a max_peak_value array where the peak values will be stored
    n = 0  # n is an iterator that moves through max_peak_ arrays. see later

    crossing_index = zeros(
        3, dtype=int_
    )  # creating a list to store the indices of the crossing points
    j = 0  # j is the iterate with a range of 0-2 (ie the 3 indices of crossing_index) after j = 2 it will be reset to 0. See later.

    outer_crossings = zeros(
        len(signal),
        dtype=int_)  # Storing the outer_crossings for debugging purposes
    p = 0  # p is an iterator that moves through outer_crossings arrays. see later

    inner_crossings = zeros(
        len(signal),
        dtype=int_)  # Storing the inner_crossings for debugging purposes
    q = 0  # q is an iterator that moves through inner_crossings arrays. see later

    thresholds = zeros(
        len(signal)
    )  # creating an array to store the thresholds as these are used to check when crossings have occured and can be graphed

    # Determining how the first point in the signal is treated w.r.t a threshold
    if initial_thres != 'unspecified':
        thresholds[0] = initial_thres
        start = 1  # with the 0th thershold determine, we want our for loop to start at the second point (1th iteration)

    else:  # a seperate threshold for the first point in signal was not entered, therefore it is treated like every other point and the function starts at the zeroth point in signal
        start = 0

    end_check = False  # setting an end_check variable which is set to true if the signal ends with an inner crossing and was above the threshold before it.

    # Peforming function
    for i in range(
            start, len(signal)
    ):  # NB the reason we start at i=1 is because the threshold list is initilized with the threshold for i=0 in it, see above.

        if threshold == 'unspecified':  # this insures if threshold = 'unspecified' then an adaptive threshold is calculated at each iteration

            # Calculating threshold at each point using a central smooth algorithm like matlabs smooth function
            # if confused by this expression see the example in my Smooth Functions description AND remember that the stop condition value isnt used!
            if i <= (window_length - 1) / 2:
                window_mean = (
                    signal[0:window_length].mean()
                )  # using the mean of a full window length to approximate the mean of the first few points until i > window length
                thresholds[i] = window_mean + vert_offset

            elif i >= len(signal) - (window_length - 1) / 2:
                window_mean = (
                    signal[-window_length:].mean()
                )  # using the mean of a full window length to approximate the mean of the last few points where a moving mean would need fewer forward points than avalible
                thresholds[
                    i] = window_mean + vert_offset  # Incase the threshold is -ve, we need to work out the threshold this way rather than just window_mean*multiple of mean

            else:
                window_mean = (
                    signal[i - int((window_length - 1) / 2):i +
                           (int((window_length - 1) / 2)) + 1].mean()
                )  # NB we need to add one to the stop point as we want to include index i+(window-1)/2 in the threshold calc
                thresholds[
                    i] = window_mean + vert_offset  # Incase the threshold is -ve, we need to work out the threshold this way rather than just window_mean*multiple of mean

        else:
            thresholds[
                i] = threshold  # if thershold was specified, the thresholds array, for every point is just the specified threshold.

        # With the threshold calculated we now check to see if the current point meets the criteria to be a peak or trough
        # checking to see if point is a positive gradient crossing of the threshold
        if signal[i] >= thresholds[i] and signal[i - 1] <= thresholds[
                i -
                1] and i != 0:  # NB if i = 0, signal[i-1]>=thresholds[i-1] --> signal[-1]>=thresholds[-1] which is looking at the end of the signal
            crossing_index[j] = i
            j = j + 1  # j steps forward one to fill the next index of crossing_index next time a crossing is found.

        # checking to see if point is a negetive gradient crossing of the threshold
        elif signal[i] <= thresholds[i] and signal[i - 1] >= thresholds[
                i -
                1] and i != 0:  # NB if i = 0, signal[i-1]>=thresholds[i-1] --> signal[-1]>=thresholds[-1] which is looking at the end of the signal
            crossing_index[j] = i
            j = j + 1  # j steps forward one to fill the next index of crossing_index next time a crossing is found.

        # Once 3 crossing have occured we can know there is a peak and trough between the two outer crossings and we can
        # go about finding them.

        # END PEAK CHECKER (I have the end_check variable just for improved readability)
        if i == len(signal) - 1 and j == 2 and signal[
                crossing_index[1] - 1] > thresholds[crossing_index[1] - 1]:
            end_check = True  # the signal was above the threshold between the last outer crossing and final inner crossing which means we need to check for a end peak between these crossings

        if j == 3 or end_check == True:

            # Argsort returns the INDICES of the sorted VALUES of an array
            # I.E. sorted data is the indices of the maximum to minimum values of the data between the crossings, the [::-1] ensures it is max to min values
            sorted_data_indices = (argsort(
                signal[crossing_index[0]:crossing_index[j - 1]])[::-1]
                                   ) + crossing_index[0]

            # Checking if the max value between two crossings is infact a local peak or if it is a false positive.
            for index in sorted_data_indices:
                # the condition below is: if the suspected_index_of_max_peak is not the first element or last element in the total signal and it is a local peak and its above its threshold then it is our max local peak.
                if index != 0 and index != len(signal) - 1 and signal[
                        index -
                        1] < signal[index] and signal[index + 1] <= signal[
                            index] and signal[index] >= thresholds[index]:
                    index_of_max_peak = index  # if the above condition is true, the suspected_index_of_max_peak has been shown to be a local peak! And hence it is
                    # the true index of the max peak between the current outer crossings.
                    break  # once the max peak has been found we want to break.

                else:  # there was no max peak (which would be weird if we have crossings!) or the index was the first or last point in the signal
                    ignored_max_index[k] = i
                    k = k + 1  # k is an iterator that moves through ignored_max_index array.
                    index_of_max_peak = 'none found'

            if index_of_max_peak != 'none found':
                if min_peak_distance != 'unspecified' and n > 0 and index_of_max_peak - max_peak_index[
                        n -
                        1] < min_peak_distance:  # Checking if the max peak in possible peaks is far enough from the previous positively identified max peak
                    ignored_peak_index[
                        m] = index_of_max_peak  # we store the ignored peak data incase the user wants to check what was ignored in the signal
                    ignored_peak_value[m] = signal[index_of_max_peak]
                    m = m + 1  # m is an iterator that ensures once an index is filled in the ignored_peak_ arrays, the next ignored_peak_ is placed in the next empty index location.

                elif min_amount_from_threshold != 'unspecified' and abs(
                        signal[index_of_max_peak] -
                        thresholds[index_of_max_peak]
                ) < min_amount_from_threshold:  # we can set an amount a peak must be above the threshold for it to be a true peak, this is handy for when a signal may jump around the threshold a tiny amount and may not be an actual peak.
                    ignored_peak_index[
                        m] = index_of_max_peak  # we store the ignored peak data incase the user wants to check what was ignored in the signal
                    ignored_peak_value[m] = signal[index_of_max_peak]
                    m = m + 1  # m is an iterator that ensures once an index is filled in the ignored_peak_ arrays, the next ignored_peak_ is placed in the next empty index location.

                elif peak_value_consistancy_fraction != 'unspecified' and n > 0 and signal[
                        index_of_max_peak] <= (
                            1 + peak_value_consistancy_fraction
                        ) * signal[max_peak_index[
                            n - 1]] and signal[index_of_max_peak] >= (
                                1 - peak_value_consistancy_fraction
                            ) * signal[max_peak_index[
                                n -
                                1]]:  # checking to see that the value of the peak found is within + or -5% of the previous peaks value, if it is, it is deemed too different to be a correct peak identification. Hence some foward knowledge of the signal is required before using this test.
                    ignored_peak_index[
                        m] = index_of_max_peak  # we store the ignored peak data incase the user wants to check what was ignored in the signal
                    ignored_peak_value[m] = signal[index_of_max_peak]
                    m = m + 1  # m is an iterator that ensures once an index is filled in the ignored_peak_ arrays, the next ignored_peak_ is placed in the next empty index location.

                else:
                    max_peak_index[
                        n] = index_of_max_peak  # adding the index of the current maximum peak to a list of all the peaks
                    max_peak_value[n] = signal[
                        index_of_max_peak]  # adding the value of the current max peak to a list of all the peaks
                    n = n + 1  # n is an iterator that ensures once an index is filled in the max_peak_ arrays, the next max_peak_ is placed in the next empty index location.

            # STORING AND RESETTING THE CROSSING ARRAY DATA
            # Also NB that when finding the argmin or argmax of signal[crossing_index[0]: crossing_index[2]+1], this is just a view of the array signal meaning it will treat this view first
            # as its own array with the first element of the array indexed back at 0. To work out the peaks index in the whole signal array we need to add crossing_index[0].
            if p == 0:
                outer_crossings[p] = crossing_index[0]
                outer_crossings[p + 1] = crossing_index[2]
                p = p + 2  # p is an iterator that ensures once an index is filled in the outer_crossings, the outer crossing is placed in the next empty index location.

            elif j == 3:  # if end_check is true, the signal ends on an inner crossing and the for loop ends with j == 2, so we dont want to add crossing_index[2] if j == 2
                outer_crossings[p] = crossing_index[
                    2]  # since crossing_index[2] becomes crossing_index[0] (see below) for successive peak finding, if we store crossing_index[0], it will just be the crossing_index[2] of the last peak
                p = p + 1  # p is an iterator that ensures once an index is filled in the outer_crossings, the outer crossing is placed in the next empty index location.

            inner_crossings[q] = crossing_index[1]
            q = q + 1  # q is an iterator that ensures once an index is filled in the inner_crossings arrays, the next inner crossing is placed in the next empty index location.

            crossing_index[0] = crossing_index[
                2]  # Setting the last crossing crossing to be the first crossings in the crossing_index, so we start looking for another two crossings from our current last crossing.
            # I.E. our final crossing now becomes our first crossing to continue searching from

            j = 1  # resetting j to one so we find our next 2 crossing. NB our first crossing will be the last crossing from the prevous set of 3 crossings as outlined above.

    # trimming all unfilled indices of max_peak_ arrays, ignored_peak_ arrays, and crossing arrays
    max_peak_index = trim_zeros(
        max_peak_index
    )  # as we cannot possible have a peak at an index location 0, we can trim peak_index and use its length to define the peak_value array. As a peaks value could be 0, so we cant use the trim_zeros function of the peak_value array.
    max_peak_value = max_peak_value[:len(
        max_peak_index
    )]  # trimming the peak_value array to the length of peak_index, as they must be the same length, each index is information on the same peak.

    ignored_peak_index = trim_zeros(
        ignored_peak_index
    )  # the code is such that we cannot possible have an ignored_peak at an index location 0, so we can trim peak_index and use its length to define the ignored_peak_value array. As an ignored_peak_value could be 0, so we cant use the trim_zeros function of that array.
    ignored_peak_value = ignored_peak_value[:len(
        ignored_peak_index
    )]  # trimming the ignored_peak_value array to the length of ignored_peak_index, as they must be the same length, each index is information on the same peak.

    ignored_max_index = trim_zeros(ignored_max_index, 'b')

    outer_crossings = trim_zeros(outer_crossings, 'b')
    inner_crossings = trim_zeros(inner_crossings)

    return (max_peak_index, max_peak_value, thresholds, outer_crossings,
            inner_crossings, ignored_peak_index)
コード例 #59
0
ファイル: Encoder.py プロジェクト: gbarland/QRcrypt
def encoderfunc(textin, EC):
    # Version Selector(partial) #

    mode = '0010'
    l = len(textin)
    version = 0
    datacap = 0
    terminator = 0
    groups = 0
    gpoly = 0
    g1 = 0
    g2 = 0
    charcount = '{0:09b}'.format(l)
    if EC == 'Low':
        bits = [0, 1]
        if l < 25:
            version = 1
            datacap = 152
            groups = 1
            cwcount = 19
            eccw = 7
            gpoly = [0, 87, 229, 146, 149, 238, 102, 21]
        elif 25 <= l < 47:
            version = 2
            datacap = 272
            groups = 1
            cwcount = 34
            eccw = 10
            gpoly = [0, 251, 67, 46, 61, 118, 70, 64, 94, 32, 45]
        elif 47 <= l < 77:
            version = 3
            datacap = 440
            groups = 1
            cwcount = 55
            eccw = 15
            gpoly = [
                0, 8, 183, 61, 91, 202, 37, 51, 58, 58, 237, 140, 124, 5, 99,
                105
            ]
        elif 77 <= l < 114:
            version = 4
            datacap = 640
            groups = 1
            cwcount = 80
            eccw = 20
            gpoly = [
                0, 17, 60, 79, 50, 61, 163, 26, 187, 202, 180, 221, 225, 83,
                239, 156, 164, 212, 212, 188, 190
            ]
        elif 114 <= l < 154:
            version = 5
            datacap = 864
            groups = 1
            cwcount = 108
            eccw = 26
            gpoly = [
                0, 173, 125, 158, 2, 103, 182, 118, 17, 145, 201, 111, 28, 165,
                53, 161, 21, 245, 142, 13, 102, 48, 227, 153, 145, 218, 70
            ]
    elif EC == 'Medium':
        bits = [0, 0]
        if l < 20:
            version = 1
            datacap = 128
            groups = 1
            cwcount = 16
            eccw = 10
            gpoly = [0, 251, 67, 46, 61, 118, 70, 64, 94, 32, 45]
        elif 20 <= l < 38:
            version = 2
            datacap = 224
            groups = 1
            cwcount = 28
            eccw = 16
            gpoly = [
                0, 120, 104, 107, 109, 102, 161, 76, 3, 91, 191, 147, 169, 182,
                194, 225, 120
            ]
        elif 38 <= l < 61:
            version = 3
            datacap = 352
            groups = 1
            cwcount = 44
            eccw = 26
            gpoly = [
                0, 173, 125, 158, 2, 103, 182, 118, 17, 145, 201, 111, 28, 165,
                53, 161, 21, 245, 142, 13, 102, 48, 227, 153, 145, 218, 70
            ]
        elif 61 <= l < 90:
            version = 4
            datacap = 512
            groups = 1
            cwcount = 64
            eccw = 18
            gpoly = [
                0, 215, 234, 158, 94, 184, 97, 118, 170, 79, 187, 152, 148,
                252, 179, 5, 98, 96, 153
            ]
        elif 90 <= l < 122:
            version = 5
            datacap = 688
            groups = 1
            cwcount = 86
            eccw = 24
            gpoly = [
                24, 229, 121, 135, 48, 211, 117, 251, 126, 159, 180, 169, 152,
                192, 226, 228, 218, 111, 0, 117, 232, 87, 96, 227, 21
            ]
    elif EC == 'Quartile':
        bits = [1, 1]
        if l < 16:
            version = 1
            datacap = 104
            groups = 1
            cwcount = 13
            eccw = 13
            gpoly = [
                13, 74, 152, 176, 100, 86, 100, 106, 104, 130, 218, 206, 140,
                78
            ]
        elif 16 <= l < 29:
            version = 2
            datacap = 176
            groups = 1
            cwcount = 22
            eccw = 22
            gpoly = [
                0, 210, 171, 247, 242, 93, 230, 14, 109, 221, 53, 200, 74, 8,
                172, 98, 80, 219, 134, 160, 105, 165, 231
            ]
        elif 29 <= l < 47:
            version = 3
            datacap = 272
            groups = 1
            cwcount = 34
            eccw = 18
            gpoly = [
                0, 215, 234, 158, 94, 184, 97, 118, 170, 79, 187, 152, 148,
                252, 179, 5, 98, 96, 153
            ]
        elif 47 <= l < 67:
            version = 4
            datacap = 384
            groups = 1
            cwcount = 48
            eccw = 26
            gpoly = [
                0, 173, 125, 158, 2, 103, 182, 118, 17, 145, 201, 111, 28, 165,
                53, 161, 21, 245, 142, 13, 102, 48, 227, 153, 145, 218, 70
            ]
        elif 67 <= l < 87:
            version = 5
            datacap = 496
            groups = 2
            g1 = 15
            g2 = 16
            cwcount = 62
            eccw = 18
            gpoly = [
                0, 215, 234, 158, 94, 184, 97, 118, 170, 79, 187, 152, 148,
                252, 179, 5, 98, 96, 153
            ]
    elif EC == 'High':
        bits = [1, 0]
        if l < 10:
            version = 1
            datacap = 72
            groups = 1
            cwcount = 9
            eccw = 17
        elif 10 <= l < 20:
            version = 2
            datacap = 128
            groups = 1
            cwcount = 16
            eccw = 28
        elif 20 <= l < 35:
            version = 3
            datacap = 208
            groups = 1
            cwcount = 26
            eccw = 22
            gpoly = [
                0, 210, 171, 247, 242, 93, 230, 14, 109, 221, 53, 200, 74, 8,
                172, 98, 80, 219, 134, 160, 105, 165, 231
            ]
        elif 35 <= l < 50:
            version = 4
            datacap = 288
            groups = 1
            cwcount = 36
            eccw = 16
            gpoly = [
                0, 120, 104, 107, 109, 102, 161, 76, 3, 91, 191, 147, 169, 182,
                194, 225, 120
            ]
        elif 50 <= l < 64:
            version = 5
            datacap = 368
            groups = 2
            g1 = 11
            g2 = 12
            cwcount = 46
            eccw = 22
            gpoly = [
                0, 210, 171, 247, 242, 93, 230, 14, 109, 221, 53, 200, 74, 8,
                172, 98, 80, 219, 134, 160, 105, 165, 231
            ]

    if version == 1:
        remainder = 0
    else:
        remainder = 7

    print("version=" + str(version))
    print("datacap=" + str(datacap))
    print("groups=", groups)

    # String Split Function #

    def split(fn):
        while fn:
            yield fn[:2]
            fn = fn[2:]

    # Loop to convert characters to binary encoded data #

    a = list(split(textin))
    length = len(a)
    encoded = ''

    for i in range(0, length):
        a[i] = a[i].replace('0', '00').replace('1', '01').replace(
            '2', '02').replace('3', '03').replace('4', '04').replace(
                '5', '05').replace('6', '06').replace('7', '07').replace(
                    '8', '08').replace('9', '09').replace('A', '10')
        a[i] = a[i].replace('B', '11').replace('C', '12').replace(
            'D', '13').replace('E', '14').replace('F', '15').replace(
                'G', '16').replace('H', '17').replace('I', '18').replace(
                    'J', '19').replace('K', '20').replace('L', '21')
        a[i] = a[i].replace('M', '22').replace('N', '23').replace(
            'O', '24').replace('P', '25').replace('Q', '26').replace(
                'R', '27').replace('S', '28').replace('T', '29').replace(
                    'U', '30').replace('V', '31').replace('W', '32')
        a[i] = a[i].replace('X', '33').replace('Y', '34').replace(
            'Z', '35').replace(' ', '36').replace('$', '37').replace(
                '%', '38').replace('*', '39').replace('+', '40').replace(
                    '-', '41').replace('.',
                                       '42').replace('/',
                                                     '43').replace(':', '44')
        a[i] = a[i].replace('b', '11').replace('c', '12').replace(
            'd', '13').replace('e', '14').replace('f', '15').replace(
                'g', '16').replace('h', '17').replace('i', '18').replace(
                    'j', '19').replace('k', '20').replace('l', '21')
        a[i] = a[i].replace('m', '22').replace('n', '23').replace(
            'o', '24').replace('p', '25').replace('q', '26').replace(
                'r', '27').replace('s', '28').replace('t', '29').replace(
                    'u', '30').replace('v', '31').replace('w', '32')
        a[i] = a[i].replace('x', '33').replace('y', '34').replace(
            'z', '35').replace('a', '10')

        if len(a[i]) == 4:
            a[i] = (int((a[i])[:2]) * 45) + (int((a[i])[2:5]))
            a[i] = '{0:011b}'.format(a[i])
        elif len(a[i]) == 2:
            a[i] = '{0:06b}'.format(int(a[i]))

        encoded = encoded + a[i]

    print("encoded=", encoded)
    datastring2 = (mode + charcount + encoded)
    dist = (datacap - len(datastring2))

    # Pad bits for short string #

    if dist >= 4:
        terminator = "0000"
    elif dist == 3:
        terminator = "000"
    elif dist == 2:
        terminator = "00"
    elif dist == 1:
        terminator = "0"
    elif dist == 0:
        terminator = ""

    datastring = (mode + charcount + encoded + terminator)
    dist2 = (datacap - len(datastring))
    l2 = len(datastring)

    pad = (8 - (l2 % 8))
    pad = "0" * pad
    datastring = datastring + pad
    extrapad = int((datacap - len(datastring)) / 8)

    # POTENTIAL PROBLEM HERE ##### LOOK AT INT ABOVE #############

    for i in range(1, extrapad + 1):
        if i % 2 == 1:
            datastring = datastring + "11101100"
        elif i % 2 == 0:
            datastring = datastring + "00010001"

    print("datastring=", datastring)

    # Creating a log antilog table #

    exp2int = [1]
    int2exp = [i for i in range(256)]

    for i in range(1, 256):
        b = exp2int[i - 1] * 2
        if b >= 256:  # XOR if result is larger than or equal to 256. Use ^^ to XOR two integers
            b = b ^ 285  # USED TO BE b^^285, bitwise xor#
        exp2int.append(b)
        int2exp[b] = i

    int2exp[0] = []  # setting zero blank
    int2exp[1] = 0  # 1 comes up at 255 again

    # Message Polynomial #

    line = datastring
    mpoly = [int(line[i:i + 8], 2) for i in range(0, len(line), 8)]
    print("mpoly=", mpoly)

    # Long Division #

    print("generator polynomial=", gpoly)
    print("message polynomial=", mpoly)

    def longdivision(steps, mpoly, gpoly):
        for i in range(0, steps):
            gap = len(mpoly) - len(gpoly)
            m = mpoly[0]
            m = int2exp[m]
            if gap > 0:
                newgpoly = [exp2int[(g + m) % 255] for g in gpoly] + [0] * gap
            else:
                newgpoly = [exp2int[(g + m) % 255] for g in gpoly]
            blank = []
            if gap < 0:
                mpoly = mpoly + [0] * abs(gap)
            for i in range(0, len(newgpoly)):
                b = [mpoly[i] ^ newgpoly[i]]  # More xor problems?
                blank = blank + b
            mpoly = numpy.trim_zeros(blank, trim='f')
        return mpoly

    # interleaving #

    if groups == 1:
        steps = len(mpoly)
        ecwords = longdivision(steps, mpoly, gpoly)
        print("ecwords=", ecwords)
        message = mpoly + ecwords
        message = ['{0:08b}'.format(i) for i in message] + [0] * remainder
        blank = ''
        for i in message:
            blank = blank + str(i)
        message = blank
        print("full message=", message)

    elif groups == 2:
        b1 = mpoly[0:g1]
        b2 = mpoly[g1:(2 * g1)]
        b3 = mpoly[(2 * g1):(2 * g1 + g2)]
        b4 = mpoly[(2 * g1 + g2):(2 * g1 + 2 * g2)]
        ec1 = longdivision(g1, b1, gpoly)
        ec2 = longdivision(g1, b2, gpoly)
        ec3 = longdivision(g2, b3, gpoly)
        ec4 = longdivision(g2, b4, gpoly)
        print("b1=", b1)
        print("b2=", b2)
        print("b3=", b3)
        print("b4=", b4)
        print("ec1=", ec1)
        print("ec2=", ec2)
        print("ec3=", ec3)
        print("ec4=", ec4)

        blank = []
        for i in range(g1):
            blank = blank + [b1[i]] + [b2[i]] + [b3[i]] + [b4[i]]
        itldata = blank + [b3[-1]] + [b4[-1]]

        blank = []
        for i in range(len(ec1)):
            blank = blank + [ec1[i]] + [ec2[i]] + [ec3[i]] + [ec4[i]]
        itlecw = blank

        message = itldata + itlecw
        message = ['{0:08b}'.format(i) for i in message] + [0] * remainder
        blank = ''
        for i in message:
            blank = blank + str(i)
        message = blank
        print("full message=", message)

    # Graphical Array #

    dimension = ((version - 1) * 4) + 21
    print("dimension  = ", dimension, "x", dimension)
    m = numpy.zeros([dimension, dimension])

    # Finder & Timing Pattern #

    for i in range(7):
        m[i][0:7] = 3
        m[i][-7:dimension] = 3
    for i in range(-1, -8, -1):
        m[i][0:7] = 3

    for i in range(1, 6):
        m[i][1:6] = 0
        m[i][-6:dimension - 1] = 0
    for i in range(-2, -7, -1):
        m[i][1:6] = 0

    for i in range(2, 5):
        m[i][2:5] = 3
        m[i][-5:dimension - 2] = 3
    for i in range(-3, -6, -1):
        m[i][2:5] = 3

    for i in range(8, dimension - 8, 2):
        m[i][6] = 3
        m[6][i] = 3

    # Data graphic #

    shift = 1
    direction = 1
    l = 0
    u = 0
    g = 1
    cap1 = (dimension - 9) * 2
    cap2 = dimension * 2
    count = 0
    counter = 0
    count3 = 0
    maxcount = ((dimension - 17) / 2) + 4
    maxcount3 = (dimension - 17) * 2
    maxcol = ((dimension - 7) / 2) - 1

    for pos, i in enumerate(message):
        if pos > ((dimension - 9) * 8):
            counter = counter + 1
        if pos > 0 and pos % cap1 == 0 and count < 4:
            count = count + 1
            shift = shift + 2
            direction = -direction
            g = -g
            u = u + g
        elif 4 <= count < maxcount and counter % cap2 == 0:
            count = count + 1
            shift = shift + 2
            direction = -direction
            g = -g
            u = u + g
        if (dimension - 1 - u) == 6:
            u = u + (1 * direction)
            counter = counter + 2
        if count == maxcol and counter % cap2 == 0:
            u = u + 8
            counter = counter + 14
            count = count + 1
        if count3 == maxcount3:
            shift = shift + 3
            direction = -direction
            g = -g
            u = u + g
        elif count3 > maxcount3 and count3 % maxcount3 == 0:
            shift = shift + 2
            direction = -direction
            g = -g
            u = u + g
        if count >= maxcol:
            count3 = count3 + 1
        m[dimension - 1 - u][dimension - shift - l] = int(i) + 1
        l = (l + 1) % 2
        u = u + (pos % 2) * direction

    # Masking #

    masknum = 0
    maskbin = [0, 0, 0]

    # maskbin=[0,0,1]
    def mask0(m):
        for r in range(dimension):
            for c in range(dimension):
                if m[r][c] == 1 or 2:
                    if (r + c) % 2 == 0:
                        if m[r][c] == 2:
                            m[r][c] = 1
                        elif m[r][c] == 1:
                            m[r][c] = 2

    def mask1(m):
        for r in range(dimension):
            for c in range(dimension):
                if m[r][c] == 1 or 2:
                    if r % 2 == 0:
                        if m[r][c] == 2:
                            m[r][c] = 1
                        elif m[r][c] == 1:
                            m[r][c] = 2

    mask0(m)
    # mask1(m)

    # Information Bits #

    gpoly = [1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1]
    bitstring = bits + maskbin
    infostring = bitstring + [0] * 10
    infostring = numpy.trim_zeros(infostring[:5], trim='f') + infostring[5:]
    print("infostring=", infostring)

    if len(infostring) == 10:
        ecstring = infostring + [0]

    while len(infostring) >= 11:
        gap = len(infostring) - len(gpoly)
        if gap != 0:
            gpoly = gpoly + [0] * gap
        pos = 0
        for i, j in zip(infostring, gpoly):
            infostring[pos] = bool(i) ^ bool(j)
            pos = pos + 1
        infostring = numpy.trim_zeros(infostring, trim='f')
        ecstring = infostring
        print("ecstring=", ecstring)

    if len(ecstring) < 10:
        ecstring = (10 - len(ecstring)) * [0] + ecstring
    print("final ecstring=", ecstring)

    ecstring = bitstring + ecstring
    qrspec = [1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0]
    pos = 0
    for i, j in zip(ecstring, qrspec):
        qrspec[pos] = (bool(i) ^ bool(j)) * 3
        pos = pos + 1
    formatstring = qrspec
    print("format string=", formatstring)

    if EC == 'L':
        if masknum == 0:
            formatstring = [3, 3, 3, 0, 3, 3, 3, 3, 3, 0, 0, 0, 3, 0, 0]
        elif masknum == 1:
            formatstring = [3, 3, 3, 0, 0, 3, 0, 3, 3, 3, 3, 0, 0, 3, 3]
    elif EC == 'M':
        if masknum == 0:
            formatstring = [3, 0, 3, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0]
        elif masknum == 1:
            formatstring = [3, 0, 3, 0, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 3]
    elif EC == 'Q':
        if masknum == 0:
            formatstring = [0, 3, 3, 0, 3, 0, 3, 0, 3, 0, 3, 3, 3, 3, 3]
        elif masknum == 1:
            formatstring = [0, 3, 3, 0, 0, 0, 0, 0, 3, 3, 0, 3, 0, 0, 0]
    elif EC == 'H':
        if masknum == 0:
            formatstring = [0, 0, 3, 0, 3, 3, 0, 3, 0, 0, 0, 3, 0, 0, 3]
        elif masknum == 1:
            formatstring = [0, 0, 3, 0, 0, 3, 3, 3, 0, 3, 3, 3, 3, 3, 0]
    print("formatstring2=", formatstring)

    for pos, i in enumerate(range(-8, 0, 1)):
        m[8][i] = formatstring[(pos + 7)]
    for pos, i in enumerate(range(-1, -8, -1)):
        m[i][8] = formatstring[pos]
    for pos, i in enumerate([0, 1, 2, 3, 4, 5, 7, 8]):
        m[8][i] = formatstring[pos]
    for pos, i in enumerate([8, 7, 5, 4, 3, 2, 1, 0]):
        m[i][8] = formatstring[(pos + 7)]
    m[-8][8] = 3

    for i in range(dimension):
        for j in range(dimension):
            if m[i][j] == 1:
                m[i][j] = 0
            if m[i][j] == 2:
                m[i][j] = 3

    plt.matshow(m, fignum=None, cmap='Greys')
    plt.savefig("qrcode.png", format='png')
    plt.show()
コード例 #60
0
                  delimiter=',',
                  usecols=(6, ),
                  unpack=True)
vale_returns = np.diff(vale) / vale[:-1]
smooth_vale = np.convolve(weights / weights.sum(), vale_returns)[N - 1:-N + 1]

K = 3
t = np.arange(N - 1, len(bhp_returns))
poly_bhp = np.polyfit(t, smooth_bhp, K)
poly_vale = np.polyfit(t, smooth_vale, K)

poly_sub = np.polysub(poly_bhp, poly_vale)
xpoints = np.roots(poly_sub)
print "Intersection points", xpoints

reals = np.isreal(xpoints)
print "Real number?", reals

xpoints = np.select([reals], [xpoints])
xpoints = xpoints.real
print "Real intersection points", xpoints

print "Sans 0s", np.trim_zeros(xpoints)

plot(t, bhp_returns[N - 1:], lw=1.0)
plot(t, smooth_bhp, lw=2.0)

plot(t, vale_returns[N - 1:], lw=1.0)
plot(t, smooth_vale, lw=2.0)
show()