def sample(self, batchsize):
if batchsize > len(self.queue):
self.logger.debug("Batchsize is to big for queue size: {}".format(
len(self.queue)))
return None
batch = randsample(self.queue, batchsize)
return Transition(*zip(*batch))
def ramp_reversal_detect(self, N=3): '''Detect horizontally concatenated (i.e. ramp-reversed) data This method will sample N random spectra from the bundle and look for a large jump in the data at the halfway point as a marker of concatenated forward and backward spectrum sweeps. Args: N = 3 (int): Number of spectra to sample for detection Returns: (bool) ''' if N > self.N: someY = self.allY else: someY = randsample(self.allY, 3) # END if dratio = 0 L = self.curve_len if L%2 == 0: for Y in someY: dmid = Y[L/2-1] - Y[L/2] dwhole = Y[-1] - Y[0] dratio += abs(dmid/dwhole)/N else: dratio = 0 # END if if dratio > 0.5: return True else: return False
def ramp_reversal_detect(self, N=3):
'''Detect horizontally concatenated (i.e. ramp-reversed) data
This method will sample N random spectra from the bundle and look
for a large jump in the data at the halfway point as a marker of
concatenated forward and backward spectrum sweeps.
Args:
N = 3 (int): Number of spectra to sample for detection
Returns:
(bool)
'''
if N > self.N:
someY = self.allY
else:
someY = randsample(self.allY, 3)
# END if
dratio = 0
L = self.curve_len
if L % 2 == 0:
for Y in someY:
dmid = Y[L / 2 - 1] - Y[L / 2]
dwhole = Y[-1] - Y[0]
dratio += abs(dmid / dwhole) / N
else:
dratio = 0
# END if
if dratio > 0.5:
return True
else:
return False
def get(self):
while True:
proxies = randsample(self.proxies, len(self.proxies))
for proxy in proxies:
host_port = 'http://%s:%d' % (proxy['host'], proxy['port'])
proxy_set = {'http': host_port, 'https': host_port}
try:
self.check_proxy(proxy_set, proxy.get('out_host', proxy['host']))
return proxy_set
except ProxyError:
pass
print('No proxies online, sleeping for %d seconds...' % self.recheck_interval, flush=True)
sleep(self.recheck_interval)
def sample(self, batchsize):
""" Return sample of transitions uniformly
from the buffer if buffer is large enough
for the given batch size. Sample is a named
tuple of transition where the elements are
torch tensor.
"""
# randomly sample transitions
random_sample = randsample(self.queue, batchsize)
# zip
zipped = [
torch.from_numpy(np.asarray(arr).astype(np.float32)).float()
for arr in zip(*random_sample)
]
sample = Transition(zipped[0], zipped[1].unsqueeze_(-1).long(),
zipped[2].unsqueeze_(-1), zipped[3],
zipped[4].unsqueeze_(-1).byte())
return sample
def sample(self, batch_size):
if batch_size > len(self.queue):
return None
batch = randsample(self.queue, batch_size)
return self.tuple_type(*zip(*batch))
def sample(self, batchsize, *args, **kwargs):
"""Sample transition from the buffer"""
return randsample(self.queue, batchsize)
#!/usr/bin/env python2.7
#coding=utf8
import os
import string
from random import sample as randsample
from random import randint
if __name__ == "__main__":
dictnames = []
dict_num = randint(0, 100) # configure dict num
query_num = randint(1, 100000) # configure query num
print "%d %d" % (dict_num, query_num)
for i_ in range(dict_num):
dict_name = 'null'
while True:
dict_name = ''.join(randsample(string.letters + string.digits, randint(1, 5)))
if dict_name not in dictnames:
break
words_num = randint(0, 1000) # configure words num in one dict
print "%s %d" % (dict_name, words_num)
for j_ in range(words_num):
print ''.join(map(lambda x: string.ascii_lowercase[ord(x)%26], os.urandom(randint(randint(1, 32), 32))))
for i_ in range(query_num):
print ''.join(map(lambda x: string.ascii_lowercase[ord(x)%26], os.urandom(randint(randint(1, 128), 128))))
def sample(self, batchsize):
if batchsize > len(self.queue):
return None
batch = randsample(self.queue, batchsize)
return self.Transition(*zip(*batch))