def clone_model(self, model, dbsource, dbdest, num_instances,
appname, models, num_models, stats, verbosity):
q = model.objects.using(dbsource).all()
if verbosity > 0:
print("- Model", model.__name__, ":", q.count(), "objects found")
num_model_instances = 0
if inspect.has_m2m(model) is False:
numi = q.count()
if numi > 0:
print("Bulk creating", str(numi), "instances for model",
model.__name__, "...")
try:
qs = QuerySet(model=model, query=q, using=dbdest)
self.bulk_create(q, using=dbdest, qs=qs)
num_instances += numi
num_model_instances += numi
except Exception as e:
err.new(e, self.clone_model,
"Can not bulk create model " + model.__name__)
return
else:
for instance in q:
print(num_instances, appname, "-",
model.__name__, num_model_instances)
num_instances += 1
num_model_instances += 1
try:
self._save_instance(model, instance, dbdest)
except Exception as e:
err.new(e, self.clone_model,
"Can not clone model " + model.__name__)
return
stats[appname]["num_models"] = len(models[appname])
stats[appname][model.__name__] = num_model_instances
return stats, num_instances
def create(self, slug, dataset, dashboard=None):
ds = cf.load_data_(dataset, "x", "y")
if ds is None:
err.new(self.new,
"No dataframe set: please provide one in parameter")
err.throw()
html = self._html(slug, ds.df)
_write_file(slug, html, "datatable", dashboard)
def _save_instance(self, model, instance, dbdest):
try:
self._disable_auto_now_fields(model)
instance.save(using=dbdest, force_insert=True)
except IntegrityError as e:
msg = "ERROR inserting", instance, "- ", model.__name__
err.new(e, self._save_instance, msg)
err.throw()
return
def run(self, request, cmd_args=[]):
try:
self.runfunc(request, cmd_args)
self.end()
return
except Exception as e:
exc = str(e)
if err.exists:
err.new(Command.run, "Can not run command")
cmderr(exc)
def rprint(*args):
msg = ""
for output in args:
msg = msg + " " + str(output)
if settings.DEBUG is True:
print("[Remote terminal]", strip_tags(msg))
try:
publish(msg, event_class="__command__", channel=COMMAND_CHANNEL)
except Exception as e:
err.new(e, rprint, "Can not publish message for remote print")
err.throw()
def clone(self, dbsource, dbdest, applist=None, verbosity=1):
global EXCLUDE
st = time.time()
if applist is None:
applist = settings.INSTALLED_APPS
source = self._get_django_db(dbsource)
if source is None:
msg = "Database", dbsource, "not found"
err.new(self.clone, msg)
return
dest = self._get_django_db(dbdest)
if dest is None:
msg = "Database", dbdest, "not found"
err.new(self.clone, msg)
return
if verbosity > 0:
print("Cloning database", dbsource, "in database", dbdest)
models = {}
for appname in applist:
appstr = appname.split('.')[-1]
if appstr in EXCLUDE or appstr.startswith("django.") \
or appstr == "django.contrib.contenttypes":
continue
# remove all remaining dots to keep just the app name
try:
s = appname.split(".")
appname = s[len(s) - 1]
except Exception:
pass
models[appname] = inspect.models(appname)
num_models = 0
num_instances = 0
stats = {}
appname = "contenttypes"
if verbosity > 0:
print("Found", str(len(applist)), "applications")
for appname in models:
if appname == "contenttypes" or appname == "sessions":
continue
else:
if len(models[appname]) > 0:
print("# Processing app", appname)
stats[appname] = {}
stats[appname]["num_models"] = 0
for model in models[appname]:
if model == Permission or model == Site:
continue
num_models += 1
stats, num_instances = self.clone_model(model, dbsource, dbdest,
num_instances, appname,
models, num_models, stats, verbosity)
err.fatal()
self.stats(models, stats, num_models, num_instances, st)
def load_generator(modname, subgenerator=None):
try:
path = modname + ".chartflo"
if subgenerator is not None:
path = path + "." + subgenerator
mod = importlib.import_module(path)
generator = getattr(mod, "run")
return generator
except ImportError as e:
if "No module named" not in str(e):
err.new(e)
return None
except Exception as e:
err.new(e, load_generator, "Error loading module")
def _write_json(slug, json):
"""
Writes a chart to Vega Lite json format to a file
"""
# check paths
folderpath = safe_join(settings.BASE_DIR, "templates/chartflo")
if not os.path.isdir(folderpath):
try:
os.makedirs(folderpath)
except Exception as e:
err.new(e)
folderpath = safe_join(settings.BASE_DIR, "templates/chartflo/json")
if not os.path.isdir(folderpath):
try:
os.makedirs(folderpath)
except Exception as e:
err.new(e)
folderpath = safe_join(settings.BASE_DIR, "templates/chartflo/json/charts")
if not os.path.isdir(folderpath):
try:
os.makedirs(folderpath)
except Exception as e:
err.new(e)
# gen json
endpath = "charts"
chartsdir_path = safe_join(settings.BASE_DIR,
"templates/chartflo/json/" + endpath)
filepath = chartsdir_path + "/" + slug + ".json"
#~ write the file
try:
filex = open(filepath, "w")
filex.write(json)
filex.close()
except Exception as e:
err.new(e)
def ready(self):
"""
Load generators and initialize class instance
"""
global GENERATORS, cf
from django.conf import settings
apps = settings.INSTALLED_APPS
generators = {}
for app in apps:
try:
res = load_generator(app)
if res is not None:
generators[app] = res
except Exception as e:
err.new(e, self.ready,
"Can not initialize Chartflo generators")
GENERATORS = generators
if err.exists:
err.trace()
def _progress_html(self, number, progress, legend, unit, thresholds, icon,
color):
"""
Generate html for progress bar number widget
"""
if unit is None:
unit = ""
if thresholds:
if progress is None:
err.new(self._progress_html,
"Please provide a progress number to use thresholds")
err.throw()
color = "green"
if progress[0] <= thresholds["low"]:
color = "red"
elif progress[0] < thresholds["high"] and progress[0] > thresholds[
"low"]:
color = "orange"
ibg = " bg-" + color
if icon is None:
icon = '<span class="info-box-icon' + ibg + \
'"><i class="fa fa-thumbs-o-up"></i></span>'
else:
icon = '<span class="info-box-icon' + ibg + \
'"><i class="fa fa-' + icon + '"></i></span>'
if unit != "":
unit = '<span class="unit"> ' + unit + '</span>'
css_class = ""
wrapper = '<div class="info-box bg-' + color + '">'
res = wrapper + icon + '\n<div class="info-box-content">'
if legend is not None:
res = res + '\n<span class="info-box-text ' + \
css_class + '">' + legend + '</span>'
res = res + '\n<span class="info-box-number">' + \
str(number) + ' ' + unit + '</span>'
res += '<div class="progress">'
res += '<div class="progress-bar" style="width:' + \
str(progress[0]) + '%"></div>'
res += '</div>'
res += '<span class="progress-description">' + \
progress[1] + '</span>'
res = res + "</div></div>"
return res
def _write_file(slug, html):
"""
Writes a table's html to a file
"""
# check directories
folderpath = safe_join(settings.BASE_DIR, "templates/tabular")
if not os.path.isdir(folderpath):
try:
os.makedirs(folderpath)
except Exception as e:
err.new(e)
endpath = "tables"
tablesdir_path = safe_join(settings.BASE_DIR,
"templates/tabular/" + endpath)
if not os.path.isdir(tablesdir_path):
try:
os.makedirs(tablesdir_path)
except Exception as e:
err.new(e)
# check file
filepath = tablesdir_path + "/" + slug + ".html"
#~ write the file
try:
filex = open(filepath, "w")
filex.write(html)
filex.close()
except Exception as e:
err.new(e)
if err.exists:
err.throw()
def handle(self, *args, **options):
try:
ex = Exporter()
except Exception as e:
err.new(e, self.handle, "Can not initialize exporter")
if options["archive"] is True:
ex.archive_replicas()
source = options["source"]
dest = options["dest"]
applist = options["applist"]
if options["migrate"] is not False:
try:
call_command("migrate", "--database=" + dest)
except Exception as e:
err.new(e, self.handle,
"Can not migrate destination database " + dest)
if applist is not None:
applist = str.split(options["applist"], ",")
try:
ex.set_local()
ex.clone(source,
dest,
applist=applist,
verbosity=int(options["verbosity"]))
except Exception as e:
err.new(e, self.handle, "Can not clone database")
if err.exists:
err.throw()
def handle(self, *args, **options):
"""
Run a generator
"""
app = options["app"]
quiet = int(options["quiet"])
#runall = int(options["all"])
subgenerator = None
if "." in app:
l = app.split(".")
app = l[0]
subgenerator = l[1]
try:
generator = GENERATORS[app]
if subgenerator is not None:
generator = load_generator(app, subgenerator)
if generator is None:
err.new("Subgenerator " + subgenerator + " not found")
err.trace()
return
except Exception as e:
err.new(e, "Generator not found")
err.report()
err.throw()
return
if quiet > 0:
print("Running generator", app)
try:
generator()
except Exception as e:
err.new(e)
"""
try:
last_run_q = get_last_run_q()
except Exception:
pass
if runall == 0:
try:
events_q = get_events_q()
except Exception as e:
err.new(e)
try:
events_q = get_changes_events(events_q, last_run_q)
except Exception as e:
err.new(e)
else:
try:
events_q = MEvent.objects.all()
except Exception as e:
err.new(e)
try:
generator(events=events_q)
except Exception as e:
err.new(e)
"""
if err.exists:
if settings.DEBUG is True:
err.throw()
else:
err.report()
def generate(self, slug, query, fields):
html = '<table id="table_' + slug + '" class="tabular-table">'
html += "<tr>"
for field in fields:
name = field
try:
name = query[0]._meta.get_field(field).verbose_name.title()
except:
pass
html += '<th>' + name + '</th>'
html += "</tr>"
for row in query:
html += "<tr>"
for field in fields:
try:
val = getattr(row, field)
html += '<td>' + str(val) + '</td>'
except Exception as e:
err.new(e)
html += "</tr>"
html += '</table>'
return html
def handle(self, *args, **options):
app = options["app"]
# check if module exists
mod = importlib.util.find_spec(app)
if mod is None:
err.new("No module named " + app)
err.throw()
return
# get paths
origin = os.path.dirname(chartflo.__file__) + "/templates/dashboards"
dest = os.getcwd() + "/" + app + "/templates/dashboards"
# check directories
if not os.path.exists(dest):
print("Creating directories")
os.makedirs(dest, exist_ok=True)
# copy
print("Copying base templates", "=>", dest)
try:
copytree(origin, dest)
except FileExistsError:
err.new(
"The directory",
app + "/templates/dashboards/base already exists, aborting")
err.throw()
return
# rename stuff
print("Configuring templates")
src = dest + "/base"
des = dest + "/" + app
shutil.move(src, des)
# fic imports in files
config_template(des + "/index.html", app)
# create dashboard object in the database
print("Creating dashboard object in the database")
Dashboard.objects.get_or_create(slug=app, title=app)
# end msg
print("[" + colors.green("Ok") + "] Dashboard base template is in",
app + "/templates/dashboards")
def _write_file(slug, html, ctype="chart", dashboard=None):
"""
Writes html to a file
"""
# check directories
folderpath = safe_join(settings.BASE_DIR, "templates/")
if not os.path.isdir(folderpath):
try:
os.makedirs(folderpath)
except Exception as e:
err.new(e)
endpath = "charts"
if ctype == "number":
endpath = "numbers"
elif ctype == "sparkline":
endpath = "sparklines"
elif ctype == "datatable":
endpath = "datatables"
elif ctype == "sequence":
endpath = "sequences"
if dashboard is not None:
midpath = "dashboards/" + dashboard + "/"
else:
midpath = "/"
chartsdir_path = safe_join(settings.BASE_DIR,
"templates/" + midpath + endpath)
if not os.path.isdir(chartsdir_path):
try:
os.makedirs(chartsdir_path)
except Exception as e:
err.new(e)
# check file
filepath = chartsdir_path + "/" + slug + ".html"
# write the file
try:
filex = open(filepath, "w")
filex.write(html)
filex.close()
except Exception as e:
err.new(e)