def do_statspollute(dbfile):
# source
gl_database = create_database("sqlite:%s" % dbfile)
source_store = Store(gl_database)
stats = source_store.find(models.Stats)
counter = 0
for s in stats:
source_store.remove(s)
counter += 1
print "removed %d entry in stats" % counter
counter = 0
# 21 days in the past
for past_hours in xrange(24 * 7 * 3):
past_hours += 4
when = utc_past_date(hours=past_hours)
newstat = models.Stats()
newstat.freemb = randint(1000, 1050)
newstat.year = when.isocalendar()[0]
newstat.week = when.isocalendar()[1]
level = round((randint(0, 1000) / 240.0), 1) - 2
def random_pollution():
return int(randint(0,11) + (5 * level))
activity_fake = {
'successfull_logins': random_pollution(),
'failed_logins': random_pollution(),
'started_submissions': random_pollution(),
'completed_submissions': random_pollution(),
'uploaded_files': int(randint(0,11) + (5 * level)),
'appended_files': random_pollution(),
'wb_comments': random_pollution(),
'wb_messages': random_pollution(),
'receiver_comments': random_pollution(),
'receiver_messages': random_pollution()
}
for k, v in activity_fake.iteritems():
if v < 0:
activity_fake[k] = 0
newstat.start = when
newstat.summary = activity_fake
counter += 1
source_store.add(newstat)
print "Committing %d stats" % counter
source_store.commit()
class StormORM(ORM):
"""
Storm implementation of ORM super class.
"""
def __init__(self, uri=None, store=None):
'''
@param uri: Database URI following storm rules.
@param store: Storm store.
If uri is given a new store is instanciated and it is used
to execute the statements.
If both parameters are given the early created store overrides
the store given.
'''
from storm.locals import create_database, Store
self.uri = uri
self.store = store
if self.uri:
database = create_database(self.uri)
self.store = Store(database)
if not self.store:
raise Exception('None storm store')
self.attrParser = StormAttributeParser()
def _getObject(self, csvType, csvStatement):
"""
Retrieves the object to be used at statement execution.
@param csvType: The CSVType
@param csvStatement: The CSVStatement
@return: The object early instanciated (for insert statement) or
retrieved from database (for update or delete statements).
"""
typo = csvType.type
keys = csvType.keys
attributes = csvStatement.attributes
if csvStatement.action in [DELETE, UPDATE]:
if csvType.hasPrimaryKey:
return self.store.get(typo, attributes[ csvType.primaryKey[0] ])
else:
pred = And([Eq(typo, key, attributes[i]) for i,key in keys.iteritems()])
result = self.store.find(typo, pred)
if result.count() == 0:
return None
elif result.count() == 1:
return result.one()
else:
return [r for r in result]
elif csvStatement.action is INSERT:
return typo()
def executeStatement(self, csvType, csvStatement):
"""
Executes csv statements matched by the pair csvType, csvStatement.
@param csvType: The CSVType
@param csvStatement: The CSVStatement
@return: Total statements executed or raises a ValueError if the object retrieved with
the pair csvType, csvStatement is None.
"""
obj = self._getObject(csvType, csvStatement)
if not obj:
msg = 'Statement return None in line %d: %s' % (csvStatement.lineNumber, csvStatement.lineContent)
raise ValueError(msg)
objs = []
if type(obj) is list:
objs += obj
else:
objs.append(obj)
i = 0
for _obj in objs:
self._executeStatement(_obj, csvType, csvStatement)
i += 1
return i
def _executeStatement(self, obj, csvType, csvStatement):
"""
Executes a single csv statement
@param csvType: The CSVType
@param csvStatement: The CSVStatement
"""
keys = csvType.keys
attributes = csvType.attributes
values = csvStatement.attributes
if csvStatement.action is INSERT:
pairs = [(key, values[i]) for i,key in keys.iteritems()]
pairs += [(key, values[i]) for i,key in attributes.iteritems()]
for key, value in pairs:
setattr(obj, key, value)
self.store.add(obj)
elif csvStatement.action is UPDATE:
pairs = [(key, values[i]) for i,key in attributes.iteritems()]
for key, value in pairs:
setattr(obj, key, value)
elif csvStatement.action is DELETE:
self.store.remove(obj)
self.store.commit()
class StormManager(Singleton):
log = logging.getLogger('{}.StormManager'.format(__name__))
def __init__(self):
pass
@loggingInfo
def init(self, *args):
self.dbOK = False
self.openDB()
@loggingInfo
def reset(self):
self.closeDB()
self.openDB()
@loggingInfo
def openDB(self):
try:
self._config = ConfigManager()
self.db = self._config.config[self._config.database]["database"]
create_db = False
if self.db == self._config.Sqlite:
folder = self._config.config[self._config.database]["folder"]
loc = folder + '/icepapcms.db'
print("Using Sqlite database at %s" % loc)
create_db = not os.path.exists(loc)
if create_db:
print("No database file found, creating it")
if not os.path.exists(folder):
os.mkdir(folder)
self._database = create_database("%s:%s" % (self.db, loc))
else:
server = self._config.config[self._config.database]["server"]
user = self._config.config[self._config.database]["user"]
pwd = self._config.config[self._config.database]["password"]
scheme = "{}://{}:{}@{}/icepapcms".format(
self.db, user, pwd, server)
if self.db == 'mysql':
self._database = MySQL(scheme)
else:
self._database = create_database(scheme)
self._store = Store(self._database)
if create_db:
self.dbOK = self.createSqliteDB()
else:
self.dbOK = True
except Exception as e:
self.log.error("Unexpected error on openDB: %s", e)
self.dbOK = False
@loggingInfo
def createSqliteDB(self):
try:
sql_file = resource_filename('icepapcms.db', 'creates_sqlite.sql')
with open(sql_file) as f:
sql_script = f.read()
statements = re.compile(r";[ \t]*$", re.M)
for statement in statements.split(sql_script):
# Remove any comments from the file
statement = re.sub(r"--.*[\n\\Z]", "", statement)
if statement.strip():
create = statement + ";"
self._store.execute(create)
self._store.commit()
return True
except Exception as e:
self.log.error("Unexpected error on createSqliteDB: %s", e)
return False
@loggingInfo
def closeDB(self):
try:
if self.dbOK:
self._store.close()
return True
except Exception as e:
self.log.error("Unexpected error on closeDB:", e)
self.dbOK = False
return False
@loggingInfo
def store(self, obj):
self._store.add(obj)
@loggingInfo
def remove(self, obj):
self._store.remove(obj)
@loggingInfo
def addIcepapSystem(self, icepap_system):
try:
self._store.add(icepap_system)
self.commitTransaction()
return True
except Exception as e:
self.log.error(
"some exception trying to store the icepap system "
"%s: %s", icepap_system, e)
return False
@loggingInfo
def deleteLocation(self, location):
if self.db == self._config.Sqlite:
for system in location.systems:
self.deleteIcepapSystem(system)
self._store.remove(location)
self.commitTransaction()
@loggingInfo
def deleteIcepapSystem(self, icepap_system):
if self.db == self._config.Sqlite:
for driver in icepap_system.drivers:
self.deleteDriver(driver)
self._store.remove(icepap_system)
self.commitTransaction()
@loggingInfo
def deleteDriver(self, driver):
for cfg in driver.historic_cfgs:
for par in cfg.parameters:
self._store.remove(par)
self._store.remove(cfg)
self._store.remove(driver)
self.commitTransaction()
@loggingInfo
def getAllLocations(self):
try:
locations = self._store.find(Location)
location_dict = {}
for location in locations:
location_dict[location.name] = location
return location_dict
except Exception as e:
self.log.error("Unexpected error on getAllLocations: %s", e)
return {}
@loggingInfo
def getLocation(self, name):
return self._store.get(Location, name)
@loggingInfo
def getIcepapSystem(self, icepap_name):
return self._store.get(IcepapSystem, icepap_name)
@loggingInfo
def existsDriver(self, mydriver, id):
drivers = self._store.find(
IcepapDriver, IcepapDriver.addr == IcepapDriverCfg.driver_addr,
IcepapDriverCfg.id == CfgParameter.cfg_id,
CfgParameter.name == str("ID"), CfgParameter.value == id)
if drivers:
for driver in drivers:
if driver.addr != mydriver.addr:
return driver
return None
else:
return None
@loggingInfo
def getLocationIcepapSystem(self, location):
try:
icepaps = self._store.find(IcepapSystem,
IcepapSystem.location_name == location)
icepaps.order_by(IcepapSystem.name)
ipapdict = {}
for ipap_sys in icepaps:
ipapdict[ipap_sys.name] = ipap_sys
return ipapdict
except Exception as e:
self.log.error(
"Unexpected error on getLocationIcepapSystem: "
"%s", e)
return {}
@loggingInfo
def rollback(self):
self._store.rollback()
@loggingInfo
def commitTransaction(self):
try:
self._store.commit()
return True
except Exception:
return False
