Beispiel #1
0
 def sample(self, batchsize):
     if batchsize > len(self.queue):
         self.logger.debug("Batchsize is to big for queue size: {}".format(
             len(self.queue)))
         return None
     batch = randsample(self.queue, batchsize)
     return Transition(*zip(*batch))
Beispiel #2
0
	def ramp_reversal_detect(self, N=3):
		'''Detect horizontally concatenated (i.e. ramp-reversed) data
			
			This method will sample N random spectra from the bundle and look
			for a large jump in the data at the halfway point as a marker of
			concatenated forward and backward spectrum sweeps.
			
			Args:
				N = 3 (int): Number of spectra to sample for detection
			Returns:
				(bool)
		'''
		
		if N > self.N:
			someY = self.allY
		else:
			someY = randsample(self.allY, 3)
		# END if
		
		dratio = 0
		L = self.curve_len
		if L%2 == 0:
			for Y in someY:
				dmid = Y[L/2-1] - Y[L/2]
				dwhole = Y[-1] - Y[0]
				dratio += abs(dmid/dwhole)/N
		else:
			dratio = 0
		# END if
		
		if dratio > 0.5:
			return True
		else:
			return False
Beispiel #3
0
    def ramp_reversal_detect(self, N=3):
        '''Detect horizontally concatenated (i.e. ramp-reversed) data
			
			This method will sample N random spectra from the bundle and look
			for a large jump in the data at the halfway point as a marker of
			concatenated forward and backward spectrum sweeps.
			
			Args:
				N = 3 (int): Number of spectra to sample for detection
			Returns:
				(bool)
		'''

        if N > self.N:
            someY = self.allY
        else:
            someY = randsample(self.allY, 3)
        # END if

        dratio = 0
        L = self.curve_len
        if L % 2 == 0:
            for Y in someY:
                dmid = Y[L / 2 - 1] - Y[L / 2]
                dwhole = Y[-1] - Y[0]
                dratio += abs(dmid / dwhole) / N
        else:
            dratio = 0
        # END if

        if dratio > 0.5:
            return True
        else:
            return False
Beispiel #4
0
 def get(self):
     while True:
         proxies = randsample(self.proxies, len(self.proxies))
         for proxy in proxies:
             host_port = 'http://%s:%d' % (proxy['host'], proxy['port'])
             proxy_set = {'http': host_port, 'https': host_port}
             try:
                 self.check_proxy(proxy_set, proxy.get('out_host', proxy['host']))
                 return proxy_set
             except ProxyError:
                 pass
         print('No proxies online, sleeping for %d seconds...' % self.recheck_interval, flush=True)
         sleep(self.recheck_interval)
 def sample(self, batchsize):
     """ Return sample of transitions uniformly
     from the buffer if buffer is large enough
     for the given batch size. Sample is a named
     tuple of transition where the elements are
     torch tensor.
     """
     # randomly sample transitions
     random_sample = randsample(self.queue, batchsize)
     # zip
     zipped = [
         torch.from_numpy(np.asarray(arr).astype(np.float32)).float()
         for arr in zip(*random_sample)
     ]
     sample = Transition(zipped[0], zipped[1].unsqueeze_(-1).long(),
                         zipped[2].unsqueeze_(-1), zipped[3],
                         zipped[4].unsqueeze_(-1).byte())
     return sample
Beispiel #6
0
 def sample(self, batch_size):
     if batch_size > len(self.queue):
         return None
     batch = randsample(self.queue, batch_size)
     return self.tuple_type(*zip(*batch))
Beispiel #7
0
 def sample(self, batchsize, *args, **kwargs):
     """Sample transition from the buffer"""
     return randsample(self.queue, batchsize)
Beispiel #8
0
#!/usr/bin/env python2.7
#coding=utf8

import os
import string
from random import sample as randsample
from random import randint

if __name__ == "__main__":
    dictnames = []
    dict_num = randint(0, 100) # configure dict num
    query_num = randint(1, 100000) # configure query num
    print "%d %d" % (dict_num, query_num)
    for i_ in range(dict_num):
        dict_name = 'null'
        while True:
            dict_name = ''.join(randsample(string.letters + string.digits, randint(1, 5)))
            if dict_name not in dictnames:
                break
        words_num = randint(0, 1000) # configure words num in one dict
        print "%s %d" % (dict_name, words_num)
        for j_ in range(words_num):
            print ''.join(map(lambda x: string.ascii_lowercase[ord(x)%26], os.urandom(randint(randint(1, 32), 32))))
    for i_ in range(query_num):
        print ''.join(map(lambda x: string.ascii_lowercase[ord(x)%26], os.urandom(randint(randint(1, 128), 128))))
Beispiel #9
0
 def sample(self, batchsize):
     if batchsize > len(self.queue):
         return None
     batch = randsample(self.queue, batchsize)
     return self.Transition(*zip(*batch))