def test_datamodel_typefns(self):
self.eq(s_datamodel.getTypeRepr('str', 'haha'), 'haha')
self.eq(s_datamodel.getTypeRepr('inet:ipv4', 0x01020304), '1.2.3.4')
self.eq(s_datamodel.getTypeNorm('str', 'haha'), ('haha', {}))
self.eq(s_datamodel.getTypeNorm('inet:ipv4', 0x01020304), (16909060, {}))
self.eq(s_datamodel.getTypeNorm('inet:ipv4', '1.2.3.4'), (16909060, {}))
self.raises(BadTypeValu, s_datamodel.getTypeNorm, 'inet:ipv4', 'hahaha')
self.eq(s_datamodel.getTypeParse('str', 'haha'), ('haha', {}))
self.eq(s_datamodel.getTypeParse('inet:ipv4', '1.2.3.4'), (16909060, {}))
def setConfOpt(self, name, valu):
'''
Set a single config option for the object.
'''
oldval = self._conf_opts.get(name)
cdef = self.getConfDef(name)
ctype = cdef[1].get('type')
if ctype is not None:
valu, _ = s_datamodel.getTypeNorm(ctype, valu)
if valu == oldval:
return False
asloc = cdef[1].get('asloc')
if asloc is not None:
setattr(self, asloc, valu)
self._conf_opts[name] = valu
self._conf_defs[name][1]['valu'] = valu
self.fire('syn:conf:set', name=name, valu=valu, oldval=oldval)
self.fire('syn:conf:set:%s' % name,
name=name,
valu=valu,
oldval=oldval)
def _initDbConn(self):
import psycopg2
self._psycopg2 = psycopg2
retry = self._link[1].get('retry', 0)
dbinfo = self._initDbInfo()
db = None
tries = 0
while db is None:
try:
db = psycopg2.connect(**dbinfo)
except Exception as e: # pragma: no cover
tries += 1
if tries > retry:
raise
time.sleep(1)
seqscan = self._link[1].get('pg:seqscan', 0)
seqscan, _ = s_datamodel.getTypeNorm('bool', seqscan)
c = db.cursor()
c.execute('SET enable_seqscan=%s', (seqscan, ))
c.close()
return db
def test_datamodel_typefns(self):
self.eq(s_datamodel.getTypeRepr('str', 'haha'), 'haha')
self.eq(s_datamodel.getTypeRepr('inet:ipv4', 0x01020304), '1.2.3.4')
self.eq(s_datamodel.getTypeNorm('str', 'haha'), ('haha', {}))
self.eq(s_datamodel.getTypeNorm('inet:ipv4', 0x01020304),
(16909060, {}))
self.eq(s_datamodel.getTypeFrob('str', 'haha'), ('haha', {}))
self.eq(s_datamodel.getTypeFrob('inet:ipv4', '1.2.3.4'),
(16909060, {}))
self.eq(s_datamodel.getTypeFrob('inet:ipv4', 0x01020304),
(16909060, {}))
self.eq(s_datamodel.getTypeFrob('inet:ipv4', 'haha'), (None, {}))
self.eq(s_datamodel.getTypeParse('str', 'haha'), ('haha', {}))
self.eq(s_datamodel.getTypeParse('inet:ipv4', '1.2.3.4'),
(16909060, {}))
def getConfNorm(self, name, valu):
'''
Return a normalized version of valu based on type knowledge for name.
Args:
name (str): The name of the config option
valu (obj): The valu of the config option
Returns:
(obj): The normalized form for valu
'''
cdef = self.getConfDef(name)
ctype = cdef[1].get('type')
if ctype is None:
return valu, {}
return s_datamodel.getTypeNorm(ctype, valu)
def setConfOpt(self, name, valu):
'''
Set a single config option for the object.
This will perform type normalization if the configration option has a 'type' value set.
Args:
name (str): Configuration name
valu: Value to set to the configuration option.
Notes:
This fires the following events, so that the EventBus can react to configuration changes. Each event
includes the name, new valu and oldvalu.
- ``syn:conf:set``
- ``syn:conf:set:<name>``
Returns:
None
'''
oldval = self._conf_opts.get(name)
cdef = self.getConfDef(name)
ctype = cdef[1].get('type')
if ctype is not None:
valu, _ = s_datamodel.getTypeNorm(ctype, valu)
if valu == oldval:
return False
asloc = cdef[1].get('asloc')
if asloc is not None:
setattr(self, asloc, valu)
self._conf_opts[name] = valu
self._conf_defs[name][1]['valu'] = valu
self.fire('syn:conf:set', name=name, valu=valu, oldval=oldval)
self.fire('syn:conf:set:%s' % name,
name=name,
valu=valu,
oldval=oldval)
def _initDbConn(self):
dbinfo = self._initDbInfo()
dbname = dbinfo.get('name')
# Initial DB Size. Must be < 2 GiB for 32-bit. Can be big for 64-bit systems. Will create
# a file of that size. On Windows, will actually immediately take up that much
# disk space.
DEFAULT_MAP_SIZE = 512 * 1024 * 1024
# _write_lock exists solely to hold off other threads' write transactions long enough to
# potentially increase the map size.
self._write_lock = Lock()
map_size = self._link[1].get('lmdb:mapsize', DEFAULT_MAP_SIZE)
self._map_size, _ = s_datamodel.getTypeNorm('int', map_size)
self._max_map_size = 2**46 if sys.maxsize > 2**32 else 2**30
map_slack = self._link[1].get('lmdb:mapslack', 2 ** 30)
self._map_slack, _ = s_datamodel.getTypeNorm('int', map_slack)
# Maximum number of 'databases', really tables. We use 5 different tables (1 main plus
# 3 indices and a blob store), + 10 tables for possible migration use cases.
MAX_DBS = 5 + 10
# flush system buffers to disk only once per transaction. Set to False can lead to last
# transaction loss, but not corruption
metasync_val = self._link[1].get('lmdb:metasync', False)
metasync, _ = s_datamodel.getTypeNorm('bool', metasync_val)
metasync = (metasync == 1)
# If sync is False, could lead to database corruption on power loss
sync_val = self._link[1].get('lmdb:sync', True)
sync, _ = s_datamodel.getTypeNorm('bool', sync_val)
sync = (sync == 1)
# Write data directly to mapped memory
WRITEMAP = True
# Doesn't create a subdirectory for storage files
SUBDIR = False
# We can disable locking, but bad things might happen if we have multiple threads
DEFAULT_LOCK = True
lock_val = self._link[1].get('lmdb:lock', DEFAULT_LOCK)
lock, _ = s_datamodel.getTypeNorm('bool', lock_val)
lock = (lock == 1)
# Maximum simultaneous readers.
MAX_READERS = 4
max_readers = self._link[1].get('lmdb:maxreaders', MAX_READERS)
max_readers, _ = s_datamodel.getTypeNorm('int', max_readers)
if max_readers == 1:
lock = False
self.dbenv = lmdb.Environment(dbname,
map_size=self._map_size,
subdir=SUBDIR,
metasync=metasync,
writemap=WRITEMAP,
max_readers=max_readers,
max_dbs=MAX_DBS,
sync=sync,
lock=lock)
# Check we're not running a weird version of LMDB
if self.dbenv.stat()['psize'] != 4096:
raise s_common.BadCoreStore(store='lmdb', mesg='Unknown version of lmdb configured')
# Ensure we have enough room in the map for expansion
self._ensure_map_slack()
def onfini():
self.dbenv.close()
self.onfini(onfini)
