def __init__(self, att, params=None):
IntGenerator.__init__(self, att, params)
# get list of files
import os, glob
self.files = []
assert 'file' in self.params, "{0}: no directive 'file'".format(self)
if some_tmp_files_to_unlink:
# ignore files
self.files = some_tmp_files_to_unlink
else:
# normal operation, process directive files
for f in self.params['file'].split(os.pathsep):
self.files += glob.glob(f)
# handle conversion
mode = self.params.get('mode', 'blob')
assert mode == 'blob' or mode == 'text', \
"{0}: mode must be 'blob' or 'text'".format(self)
self.conv = bytes if mode == 'blob' else \
lambda s: s.decode(opts.encoding)
# set size & offset
assert len(self.files) > 0, "{0}: non empty set of files".format(self)
if self.size == None:
self.setSize(len(self.files))
if self.size > len(self.files):
self.size = len(self.files)
self.offset = 0
self.cleanParams(FileGenerator.DIRS)
def __init__(self, att, params=None, words=None):
# keep explicit size for later
self.__size = params['size'] if params and 'size' in params else \
att.params['size'] if att and 'size' in att.params else \
None
# NOT att.size: this is computed and does not supersede len(words)
self.words = None # temporary
IntGenerator.__init__(self, att, params)
assert not (words and 'word' in self.params), \
"internal constructor issue, two word list specification!"
if words:
self.words = words
else:
spec = self.params['word']
assert len(spec) > 0, "{0}: empty word specification".format(self)
if spec[0] == ':':
self.words = spec[1:].split(',')
elif opts.self_test_hack:
# self tests cannot depend from an external file
# print("-- use Hobbit list for testing...")
self.words = WordGenerator.HOBBITS
else:
# load word list from file
f = open(spec, encoding='utf-8')
assert f, "{0}: cannnot open '{1}'".format(self, spec)
self.words = [u(l.rstrip()) for l in f]
f.close()
# TODO: should check that UNIQUE is ok?
# overwrite default size from IntGenerator
self.setSize(self.__size if self.__size else len(self.words))
assert self.size <= len(self.words), \
"{0}: 'size' {1} > number of words {2}". \
format(self, self.size, len(self.words))
self.cleanParams(WordGenerator.DIRS)
def __init__(self, att, params=None):
IntGenerator.__init__(self, att, params)
# get network target and mask
self.network = self.params.get('inet', ';0.0.0.0/0')
if len(self.network):
beg, end = self.network[0] in ',;', self.network[0] in '.;'
else:
beg, end, self.network = True, True, ';0.0.0.0/0'
if beg or end:
self.network = self.network[1:]
if self.network.find(':') != -1:
# IPv6: rely on netaddr, will fail if not available
from netaddr import IPNetwork, IPAddress
self.tonet = IPAddress
if self.network.find('/') == -1:
self.network += '/64'
try:
ip = IPNetwork(self.network)
except:
assert False, "{0}: invalid ipv6 {1}".format(self, self.network)
self.net = ip.first
self.mask = ip.hostmask.value
else:
# IPv4: local implementation, ipaddress is not always available
self.tonet = lambda i: \
'.'.join(str(int((i & (255 << n * 8)) / (1 << n * 8))) \
for n in range(3, -1, -1))
if self.network.find('/') == -1:
self.network += '/24'
ipv4, m = re.match(r'(.*)/(.*)', self.network).group(1, 2)
assert re.match(r'(\d+\.){3}\d+$', ipv4), \
"{0}: invalid ipv4 {1}".format(self, self.network)
try:
n, m = 0, int(m)
assert 0 <= m and m <= 32, \
"{0}: ipv4 mask not in 0..32".format(self, m)
# extract network number
for i in map(int, re.split('\.', ipv4)):
assert 0 <= i and i <= 255, \
"{0}: ipv4 byte {1} not in 0..255".format(self, i)
n = n * 256 + i
except:
assert False, "{0}: invalid ipv4 {1}".format(self, self.network)
assert 0 <= n and n <= 0xffffffff, \
"{0}: ipv4 address {1} not on 4 bytes".format(self, n)
self.mask = (1 << (32 - m)) - 1
self.net = n & (0xffffffff ^ self.mask)
# override size & offset if was not set explicitely
size = self.mask - 1 + int(beg) + int(end)
if self.size == None:
assert size > 0, \
"{0}: empty address range {1}".format(self, self.network)
self.setSize(size)
else:
# ??? fix default size in some cases...
if self.size > size:
self.setSize(size)
if not 'offset' in self.params:
self.offset = int(not beg)
self.cleanParams(InetGenerator.DIRS)
def __init__(self, att, params=None):
IntGenerator.__init__(self, att, params)
self.offset = 0
self.tz = self.params.get('tz')
start, end = 'start' in self.params, 'end' in self.params
ref = self.params['start'] if start else \
self.params['end'] if end else \
None
if ref != None:
self.ref = TimestampGenerator.parse(ref)
self.dir = 2 * ('start' in self.params) - 1
else:
self.ref = datetime.today()
self.dir = -1
# precision, defaults to 60 seconds
self.prec = self.params.get('prec', 60)
# set size
if start and end:
dend = TimestampGenerator.parse(self.params['end'])
assert self.ref <= dend, \
"{0}: 'end' must be after 'start'".format(self)
d = (dend - self.ref) / self.prec
# t = d.total_seconds() # requires 2.7
t = (d.microseconds + (d.seconds + d.days * 86400) * 10**6) / 10**6
self.setSize(int(t + 1))
self.cleanParams(TimestampGenerator.DIRS)
def __init__(self, att, params=None):
IntGenerator.__init__(self, att, params)
self.offset = 0
start, end = 'start' in self.params, 'end' in self.params
ref = self.params['start'] if start else \
self.params['end'] if end else \
None
if ref != None:
self.ref = DateGenerator.parse(ref)
self.dir = 2 * ('start' in self.params) - 1
else:
self.ref = date.today()
self.dir = -1
# precision, defaults to 1 day
self.prec = self.params.get('prec', 1)
assert self.prec > 0, \
"{0}: 'prec' {1} not > 0".format(self, self.prec)
# adjust size of both start & end are specified
if start and end:
dend = DateGenerator.parse(self.params['end'])
assert self.ref <= dend, \
"{0}: 'end' must be after 'start'".format(self)
delta = (dend - self.ref) / self.prec
self.setSize(delta.days + 1)
self.cleanParams(DateGenerator.DIRS)
def __init__(self, att, params=None):
IntGenerator.__init__(self, att, params)
WithLength.__init__(self, 8, 16)
# set defaults from attributes
self.prefix = self.params.get('prefix', att.name if att else 'str')
if self.size == None:
self.setSize(opts.size if opts.size else 1000) # ???
self.cleanParams(StringGenerator.DIRS)
def setSize(self, size):
if self.words == None:
# hack: too early!
return
IntGenerator.setSize(self, size)
# ??? fix anyway...
if self.size > len(self.words):
self.size = len(self.words)
# do not access the list out of bounds
if self.offset + self.size > len(self.words):
self.offset = self.size - len(self.words)
def __init__(self, name, params):
assert name != None and params != None, "mandatory parameters"
size = params['size'] if 'size' in params else None
mult = params['mult'] if 'mult' in params else None
IntGenerator.__init__(self, None, params)
self.nullp = 0.0
# set size, possibly overriding super constructor
if size:
self.setSize(size)
elif mult and opts.size:
self.setSize(mult * opts.size)
elif opts.size:
self.setSize(opts.size)
assert self.size != None, \
"{0}: size not set in macro {1}".format(self, name)
# persistent
WithPersistent.__init__(self)
self.cleanParams(SharedGenerator.DIRS)
def __init__(self, att, params):
IntGenerator.__init__(self, att, params)
self._rand2 = random.Random()
self.cleanParams(SeedGenerator.DIRS)
def genData(self):
d = self.ref + self.dir * \
timedelta(days=self.prec * IntGenerator.genData(self))
return db.dateValue(d)
def __init__(self, att, params=None):
IntGenerator.__init__(self, att, params)
self.unit = self.params.get('unit', 's')
self.cleanParams(IntervalGenerator.DIRS)