def init_db(drop=False, silence=False):
# import all modules here that might define models so that
# they will be registered properly on the metadata. Otherwise
# you will have to import them first before calling init_db()
from labmanager.models import LabManagerUser
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'])
if drop:
print "Droping Database"
Base.metadata.drop_all(bind=engine)
meta = MetaData(engine)
meta.reflect()
if 'alembic_version' in meta.tables:
meta.drop_all()
alembic_config = create_alembic_config(silence)
alembic_config.set_section_option('logger_alembic', 'level', 'WARN')
with app.app_context():
db.create_all()
command.stamp(alembic_config, "head")
with app.app_context():
password = unicode(hashlib.new(ALGORITHM, 'password').hexdigest())
admin_user = LabManagerUser(u'admin', u'Administrator', password)
db.session.add(admin_user)
db.session.commit()
def init_db(drop = False, silence = False):
# import all modules here that might define models so that
# they will be registered properly on the metadata. Otherwise
# you will have to import them first before calling init_db()
from labmanager.models import LabManagerUser
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'])
if drop:
print "Droping Database"
Base.metadata.drop_all(bind=engine)
meta = MetaData(engine)
meta.reflect()
if 'alembic_version' in meta.tables:
meta.drop_all()
alembic_config = create_alembic_config(silence)
alembic_config.set_section_option('logger_alembic', 'level', 'WARN')
with app.app_context():
db.create_all()
command.stamp(alembic_config, "head")
with app.app_context():
password = unicode(hashlib.new(ALGORITHM, 'password').hexdigest())
admin_user = LabManagerUser(u'admin', u'Administrator', password)
db.session.add(admin_user)
db.session.commit()
def fill_geoip():
try:
from geoip2.database import Reader as GeoIP2Reader
except ImportError:
pass
else:
COUNTRY_FILENAME = 'GeoLite2-Country.mmdb'
CITY_FILENAME = 'GeoLite2-City.mmdb'
for filename in [COUNTRY_FILENAME, CITY_FILENAME]:
if not os.path.exists(filename):
r = requests.get("http://geolite.maxmind.com/download/geoip/database/%s.gz" % filename)
open('%s.gz' % filename,'wb').write(r.content)
uncompressed = gzip.open('%s.gz' % filename).read()
open(filename, 'wb').write(uncompressed)
from labmanager import app
with app.app_context():
country_reader = GeoIP2Reader(COUNTRY_FILENAME)
city_reader = GeoIP2Reader(CITY_FILENAME)
countries = 0
errors = 0
cities = 0
print db.session.query(UseLog).filter(UseLog.country.is_(None)).count()
for log in db.session.query(UseLog).filter(UseLog.country == None).all():
ip_address = log.ip_address
try:
country_results = country_reader.country(ip_address)
except:
errors += 1
continue
if country_results:
if country_results.country and country_results.country.iso_code:
log.country = country_results.country.iso_code
countries += 1
db.session.commit()
db.session.remove()
for log in db.session.query(UseLog).filter(UseLog.city == None).all():
ip_address = log.ip_address
try:
city_results = city_reader.city(ip_address)
except:
errors += 1
continue
if city_results and city_results.city and city_results.city.name:
log.city = city_results.city.name
cities += 1
db.session.commit()
print "geoip run {} countries, {} cities, {} errors".format(countries, cities, errors)
def wrapper(*args, **kwargs):
try:
getattr(g, 'testing_if_running_in_context', None)
except RuntimeError:
running_inside_context = False
else:
running_inside_context = True
if running_inside_context:
return func(*args, **kwargs)
with app.app_context():
return func(*args, **kwargs)
def wrapper(*args, **kwargs):
try:
getattr(g, 'testing_if_running_in_context', None)
except RuntimeError:
running_inside_context = False
else:
running_inside_context = True
if running_inside_context:
return func(*args, **kwargs)
with app.app_context():
return func(*args, **kwargs)
def _run_all(self):
# Run global tasks
cache_disabler = CacheDisabler()
for task in _GLOBAL_PERIODIC_TASKS:
now = self._now()
if self._must_be_run(now, task):
for version in task['versions']:
_debug("Running task %r for rlms %s %s..." %
(task['name'], task['rlms'], version))
if task['disable_cache']:
cache_disabler.disable()
try:
task['func']()
except Exception:
traceback.print_exc()
finally:
cache_disabler.reenable()
_debug(u"Finished")
self.latest_executions[task['id']] = now
# Same for regular
for task in _LOCAL_PERIODIC_TASKS:
now = self._now()
if self._must_be_run(now, task):
with app.app_context():
for version in task['versions']:
rlmss = db.session.query(dbRLMS).filter_by(
kind=task['rlms'], version=version).all()
for db_rlms in rlmss:
ManagerClass = get_manager_class(
db_rlms.kind, db_rlms.version, db_rlms.id)
rlms = ManagerClass(db_rlms.configuration)
_debug(u"Running task %r for rlms %s %s (%s)..." %
(repr(task['name']), repr(task['rlms']),
repr(version), repr(db_rlms.location)))
if task['disable_cache']:
cache_disabler.disable()
try:
task['func'](rlms)
except Exception:
traceback.print_exc()
finally:
cache_disabler.reenable()
_debug(u"Finished")
self.latest_executions[task['id']] = now
def _run_all(self):
# Run global tasks
cache_disabler = CacheDisabler()
for task in _GLOBAL_PERIODIC_TASKS:
now = self._now()
if self._must_be_run(now, task):
for version in task['versions']:
_debug("Running task %r for rlms %s %s..." % (task['name'], task['rlms'], version))
if task['disable_cache']:
cache_disabler.disable()
try:
task['func']()
except Exception:
traceback.print_exc()
finally:
cache_disabler.reenable()
_debug(u"Finished")
self.latest_executions[task['id']] = now
# Same for regular
for task in _LOCAL_PERIODIC_TASKS:
now = self._now()
if self._must_be_run(now, task):
with app.app_context():
for version in task['versions']:
rlmss = db.session.query(dbRLMS).filter_by(kind = task['rlms'], version = version).all()
for db_rlms in rlmss:
ManagerClass = get_manager_class(db_rlms.kind, db_rlms.version, db_rlms.id)
rlms = ManagerClass(db_rlms.configuration)
_debug(u"Running task %r for rlms %s %s (%s)..." % (repr(task['name']), repr(task['rlms']), repr(version), repr(db_rlms.location)))
if task['disable_cache']:
cache_disabler.disable()
try:
task['func'](rlms)
except Exception:
traceback.print_exc()
finally:
cache_disabler.reenable()
_debug(u"Finished")
self.latest_executions[task['id']] = now
def upgrade():
with app.app_context():
for id, name, kind, version, configuration in db.session.execute(
sql.select([rlms.c.id, rlms.c.name, rlms.c.kind, rlms.c.version, rlms.c.configuration])
):
kwargs = {}
if not name:
kwargs["name"] = u"%s - %s" % (kind, version)
if kind == "Virtual labs":
config = json.loads(configuration)
if "name" in config and "web_name" not in config:
config["web_name"] = config.pop("name")
kwargs["configuration"] = json.dumps(config)
if kwargs:
update_stmt = rlms.update().where(rlms.c.id == id).values(**kwargs)
db.session.execute(update_stmt)
db.session.commit()
def upgrade():
with app.app_context():
for id, name, kind, version, configuration in db.session.execute(
sql.select([
rlms.c.id, rlms.c.name, rlms.c.kind, rlms.c.version,
rlms.c.configuration
])):
kwargs = {}
if not name:
kwargs['name'] = u'%s - %s' % (kind, version)
if kind == 'Virtual labs':
config = json.loads(configuration)
if 'name' in config and 'web_name' not in config:
config['web_name'] = config.pop('name')
kwargs['configuration'] = json.dumps(config)
if kwargs:
update_stmt = rlms.update().where(rlms.c.id == id).values(
**kwargs)
db.session.execute(update_stmt)
db.session.commit()
def fill_geoip():
try:
from geoip2.database import Reader as GeoIP2Reader
except ImportError:
pass
else:
COUNTRY_FILENAME = 'GeoLite2-Country.mmdb'
CITY_FILENAME = 'GeoLite2-City.mmdb'
for filename in [COUNTRY_FILENAME, CITY_FILENAME]:
if not os.path.exists(filename):
r = requests.get(
"http://geolite.maxmind.com/download/geoip/database/%s.gz"
% filename)
open('%s.gz' % filename, 'wb').write(r.content)
uncompressed = gzip.open('%s.gz' % filename).read()
open(filename, 'wb').write(uncompressed)
from labmanager import app
with app.app_context():
country_reader = GeoIP2Reader(COUNTRY_FILENAME)
city_reader = GeoIP2Reader(CITY_FILENAME)
countries = 0
errors = 0
cities = 0
print db.session.query(UseLog).filter(
UseLog.country.is_(None)).count()
for log in db.session.query(UseLog).filter(
UseLog.country == None).all():
ip_address = log.ip_address
try:
country_results = country_reader.country(ip_address)
except:
errors += 1
continue
if country_results:
if country_results.country and country_results.country.iso_code:
log.country = country_results.country.iso_code
countries += 1
db.session.commit()
db.session.remove()
for log in db.session.query(UseLog).filter(
UseLog.city == None).all():
ip_address = log.ip_address
try:
city_results = city_reader.city(ip_address)
except:
errors += 1
continue
if city_results and city_results.city and city_results.city.name:
log.city = city_results.city.name
cities += 1
db.session.commit()
print "geoip run {} countries, {} cities, {} errors".format(
countries, cities, errors)