def getHTML(request):
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
scenarios = Scenario.objects.using(DEFAULT_DB_ALIAS)
if scenarios.count() <= 1:
return None
release_perm = []
copy_perm = []
promote_perm = []
active_scenarios = []
free_scenarios = []
in_use_scenarios = []
for scenario in scenarios:
try:
user = User.objects.using(scenario.name).get(
username=request.user.username
)
if scenario.status != "Free":
in_use_scenarios.append(scenario.name)
else:
free_scenarios.append(scenario.name)
if user.has_perm("common.release_scenario"):
release_perm.append(scenario.name)
if user.has_perm("common.promote_scenario"):
promote_perm.append(scenario.name)
if user.has_perm("common.copy_scenario"):
copy_perm.append(scenario.name)
if user.is_active:
active_scenarios.append(scenario.name)
except Exception:
# database schema is not properly created, scenario is free
free_scenarios.append(scenario.name)
active_scenarios.append(scenario.name)
# If all scenarios are in use and user is inactive in all of them then he won't see the scenario management menu
if len(free_scenarios) == 0 and len(active_scenarios) == 1:
return None
return render_to_string(
"commands/scenario_copy.html",
{
"scenarios": scenarios,
"DEFAULT_DB_ALIAS": DEFAULT_DB_ALIAS,
"current_database": request.database,
"release_perm": release_perm,
"copy_perm": copy_perm,
"promote_perm": promote_perm,
"active_scenarios": active_scenarios,
"free_scenarios": free_scenarios,
},
request=request,
)
def main():
# Environment settings (which are used in the Django settings file and need
# to be updated BEFORE importing the settings)
os.environ.setdefault('FREPPLE_HOME', sys.path[0])
os.environ.setdefault('DJANGO_SETTINGS_MODULE', "freppledb.settings")
os.environ.setdefault('FREPPLE_APP', os.path.join(sys.path[0], 'custom'))
os.environ.setdefault(
'PYTHONPATH',
os.path.join(sys.path[0], 'lib', 'library.zip') +
os.pathsep +
os.path.join(sys.path[0], 'lib')
)
# Add the custom directory to the Python path.
sys.path += [ os.environ['FREPPLE_APP'], ]
# Initialize django
import django
django.setup()
# Import django
from django.core.management import execute_from_command_line
from django.conf import settings
if os.path.exists(os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin', 'pg_ctl.exe')):
# Using the included postgres database
# Check if the database is running. If not, start it.
os.environ['PATH'] = os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin') + os.pathsep + os.environ['PATH']
status = call([
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin', 'pg_ctl.exe'),
"--pgdata", os.path.join(settings.FREPPLE_LOGDIR, 'database'),
"--silent",
"status"
],
stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW
)
if status:
print("Starting the PostgreSQL database now", settings.FREPPLE_LOGDIR)
call([
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin', 'pg_ctl.exe'),
"--pgdata", os.path.join(settings.FREPPLE_LOGDIR, 'database'),
"--log", os.path.join(settings.FREPPLE_LOGDIR, 'database', 'server.log'),
"-w", # Wait till it's up
"start"
],
stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW
)
# Synchronize the scenario table with the settings
from freppledb.common.models import Scenario
Scenario.syncWithSettings()
# Execute the command
execute_from_command_line(sys.argv)
def getHTML(request):
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
scenarios = Scenario.objects.using(DEFAULT_DB_ALIAS)
if scenarios.count() <= 1:
return None
return render_to_string(
"commands/scenario_copy.html",
{
"scenarios": scenarios,
"DEFAULT_DB_ALIAS": DEFAULT_DB_ALIAS,
"current_database": request.database,
},
request=request,
)
def __call__(self, request):
if not hasattr(request, "user"):
request.user = auth.get_user(request)
if not hasattr(request.user, "scenarios"):
# A scenario list is not available on the request
for i in settings.DATABASES:
try:
if settings.DATABASES[i]["regexp"].match(request.path):
scenario = Scenario.objects.using(
DEFAULT_DB_ALIAS).get(name=i)
if scenario.status != "In use":
return HttpResponseNotFound("Scenario not in use")
request.prefix = "/%s" % i
request.path_info = request.path_info[len(request.
prefix):]
request.path = request.path[len(request.prefix):]
request.database = i
if hasattr(request.user, "_state"):
request.user._state.db = i.name
return self.get_response(request)
except Exception:
pass
request.prefix = ""
request.database = DEFAULT_DB_ALIAS
if hasattr(request.user, "_state"):
request.user._state.db = DEFAULT_DB_ALIAS
else:
# A list of scenarios is already available
if request.user.is_anonymous:
return self.get_response(request)
default_scenario = None
for i in request.user.scenarios:
if i.name == DEFAULT_DB_ALIAS:
default_scenario = i
try:
if settings.DATABASES[i.name]["regexp"].match(
request.path):
request.prefix = "/%s" % i.name
request.path_info = request.path_info[len(request.
prefix):]
request.path = request.path[len(request.prefix):]
request.database = i.name
request.scenario = i
if hasattr(request.user, "_state"):
request.user._state.db = i.name
request.user.is_superuser = i.is_superuser
return self.get_response(request)
except Exception:
pass
request.prefix = ""
request.database = DEFAULT_DB_ALIAS
if hasattr(request.user, "_state"):
request.user._state.db = DEFAULT_DB_ALIAS
if default_scenario:
request.scenario = default_scenario
else:
request.scenario = Scenario(name=DEFAULT_DB_ALIAS)
return self.get_response(request)
def extra_context(reportclass, request, *args, **kwargs):
try:
constraint = int(request.session['constraint'])
except:
constraint = 15
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Loop over all fixtures of all apps and directories
fixtures = set()
folders = list(settings.FIXTURE_DIRS)
for app in get_apps():
if app.__name__.startswith('django'):
continue
folders.append(
os.path.join(os.path.dirname(app.__file__), 'fixtures'))
for f in folders:
try:
for root, dirs, files in os.walk(f):
for i in files:
if i.endswith('.json'):
fixtures.add(i.split('.')[0])
except:
pass # Silently ignore failures
fixtures = sorted(fixtures)
# Send to template
odoo = 'freppledb.odoo' in settings.INSTALLED_APPS
return {
'capacityconstrained': constraint & 4,
'materialconstrained': constraint & 2,
'leadtimeconstrained': constraint & 1,
'fenceconstrained': constraint & 8,
'scenarios': Scenario.objects.all(),
'fixtures': fixtures,
'openbravo': 'freppledb.openbravo' in settings.INSTALLED_APPS,
'odoo': odoo,
'odoo_read': odoo and request.session.get('odoo_read', False),
'odoo_write': odoo and request.session.get('odoo_write', False)
}
def extra_context(reportclass, request, *args, **kwargs):
try:
constraint = int(request.session['constraint'])
except:
constraint = 15
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Loop over all fixtures of all apps and directories
fixtures = set()
folders = list(settings.FIXTURE_DIRS)
for app in get_apps():
if app.__name__.startswith('django'):
continue
folders.append(os.path.join(os.path.dirname(app.__file__), 'fixtures'))
for f in folders:
try:
for root, dirs, files in os.walk(f):
for i in files:
if i.endswith('.json'):
fixtures.add(i.split('.')[0])
except:
pass # Silently ignore failures
fixtures = sorted(fixtures)
# Send to template
odoo = 'freppledb.odoo' in settings.INSTALLED_APPS
return {'capacityconstrained': constraint & 4,
'materialconstrained': constraint & 2,
'leadtimeconstrained': constraint & 1,
'fenceconstrained': constraint & 8,
'scenarios': Scenario.objects.all(),
'fixtures': fixtures,
'openbravo': 'freppledb.openbravo' in settings.INSTALLED_APPS,
'odoo': odoo,
'odoo_read': odoo and request.session.get('odoo_read', False),
'odoo_write': odoo and request.session.get('odoo_write', False)
}
def process_request(self, request):
request.user = auth.get_user(request)
if not hasattr(request.user, 'scenarios'):
# A scenario list is not available on the request
for i in settings.DATABASES:
try:
if settings.DATABASES[i]['regexp'].match(request.path):
scenario = Scenario.objects.get(name=i)
if scenario.status != 'In use':
print('boom')
return HttpResponseNotFound('Scenario not in use')
request.prefix = '/%s' % i
request.path_info = request.path_info[len(request.
prefix):]
request.path = request.path[len(request.prefix):]
request.database = i
return
except Exception as e:
print(e)
pass
request.prefix = ''
request.database = DEFAULT_DB_ALIAS
else:
# A list of scenarios is already available
if not request.user or request.user.is_anonymous():
return
default_scenario = None
for i in request.user.scenarios:
if i.name == DEFAULT_DB_ALIAS:
default_scenario = i
try:
if settings.DATABASES[i.name]['regexp'].match(
request.path):
request.prefix = '/%s' % i.name
request.path_info = request.path_info[len(request.
prefix):]
request.path = request.path[len(request.prefix):]
request.database = i.name
request.scenario = i
request.user._state.db = i.name
request.user.is_superuser = i.is_superuser
return
except:
pass
request.prefix = ''
request.database = DEFAULT_DB_ALIAS
if default_scenario:
request.scenario = default_scenario
else:
request.scenario = Scenario(name=DEFAULT_DB_ALIAS)
def extra_context(reportclass, request, *args, **kwargs):
try:
constraint = int(request.session['constraint'])
except:
constraint = 15
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Collect optional tasks
PlanTaskRegistry.autodiscover()
planning_options = PlanTaskRegistry.getLabels()
# Loop over all accordion of all apps and directories
accordions = set()
accord = ''
for commandname, appname in get_commands().items():
try:
accord = getattr(
import_module('%s.management.commands.%s' %
(appname, commandname)), 'Command')
if accord.index >= 0:
accordions.add(accord)
except Exception as e:
pass # Silently ignore failures
accordions = sorted(accordions, key=operator.attrgetter('index'))
# Send to template
return {
'commandlist':
accordions,
'logfileslist':
json.dumps(
list(x for x in os.listdir(settings.FREPPLE_LOGDIR)
if x.endswith('.log')))
}
def extra_context(reportclass, request, *args, **kwargs):
try:
constraint = int(request.session['constraint'])
except:
constraint = 15
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Collect optional tasks
PlanTaskRegistry.autodiscover()
planning_options = PlanTaskRegistry.getLabels()
# Loop over all fixtures of all apps and directories
fixtures = set()
folders = list(settings.FIXTURE_DIRS)
for app in get_apps():
if app.__name__.startswith('django'):
continue
folders.append(
os.path.join(os.path.dirname(app.__file__), 'fixtures'))
for f in folders:
try:
for root, dirs, files in os.walk(f):
for i in files:
if i.endswith('.json'):
fixtures.add(i.split('.')[0])
except:
pass # Silently ignore failures
fixtures = sorted(fixtures)
# Function to convert from bytes to human readabl format
def sizeof_fmt(num):
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs(num) < 1024.0:
return "%3.1f%sB" % (num, unit)
num /= 1024.0
return "%.1f%sB" % (num, 'Yi')
# List available data files
filestoupload = []
if 'FILEUPLOADFOLDER' in settings.DATABASES[request.database]:
uploadfolder = settings.DATABASES[
request.database]['FILEUPLOADFOLDER']
if os.path.isdir(uploadfolder):
for file in os.listdir(uploadfolder):
if file.endswith(('.csv', '.csv.gz', '.log')):
filestoupload.append([
file,
strftime(
"%Y-%m-%d %H:%M:%S",
localtime(
os.stat(os.path.join(uploadfolder,
file)).st_mtime)),
sizeof_fmt(
os.stat(os.path.join(uploadfolder,
file)).st_size)
])
# Send to template
return {
'capacityconstrained':
constraint & 4,
'materialconstrained':
constraint & 2,
'leadtimeconstrained':
constraint & 1,
'fenceconstrained':
constraint & 8,
'scenarios':
Scenario.objects.all(),
'fixtures':
fixtures,
'openbravo':
'freppledb.openbravo' in settings.INSTALLED_APPS,
'planning_options':
planning_options,
'current_options':
request.session.get('env', [i[0] for i in planning_options]),
'filestoupload':
filestoupload,
'datafolderconfigured':
'FILEUPLOADFOLDER' in settings.DATABASES[request.database]
}
def handle(self, **options):
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
# Pick up options
force = options["force"]
promote = options["promote"]
test = "FREPPLE_TEST" in os.environ
if options["user"]:
try:
user = User.objects.all().get(username=options["user"])
except Exception:
raise CommandError("User '%s' not found" % options["user"])
else:
user = None
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Initialize the task
source = options["source"]
try:
sourcescenario = Scenario.objects.using(DEFAULT_DB_ALIAS).get(
pk=source)
except Exception:
raise CommandError("No source database defined with name '%s'" %
source)
now = datetime.now()
task = None
if "task" in options and options["task"]:
try:
task = Task.objects.all().using(source).get(pk=options["task"])
except Exception:
raise CommandError("Task identifier not found")
if (task.started or task.finished or task.status != "Waiting"
or task.name not in ("frepple_copy", "scenario_copy")):
raise CommandError("Invalid task identifier")
task.status = "0%"
task.started = now
else:
task = Task(name="scenario_copy",
submitted=now,
started=now,
status="0%",
user=user)
task.processid = os.getpid()
task.save(using=source)
# Validate the arguments
destination = options["destination"]
destinationscenario = None
try:
task.arguments = "%s %s" % (source, destination)
if options["description"]:
task.arguments += '--description="%s"' % options[
"description"].replace('"', '\\"')
if force:
task.arguments += " --force"
task.save(using=source)
try:
destinationscenario = Scenario.objects.using(
DEFAULT_DB_ALIAS).get(pk=destination)
except Exception:
raise CommandError(
"No destination database defined with name '%s'" %
destination)
if source == destination:
raise CommandError("Can't copy a schema on itself")
if sourcescenario.status != "In use":
raise CommandError("Source scenario is not in use")
if destinationscenario.status != "Free" and not force and not promote:
raise CommandError("Destination scenario is not free")
if promote and (destination != DEFAULT_DB_ALIAS
or source == DEFAULT_DB_ALIAS):
raise CommandError(
"Incorrect source or destination database with promote flag"
)
# Logging message - always logging in the default database
destinationscenario.status = "Busy"
destinationscenario.save(using=DEFAULT_DB_ALIAS)
# Copying the data
# Commenting the next line is a little more secure, but requires you to create a .pgpass file.
if settings.DATABASES[source]["PASSWORD"]:
os.environ["PGPASSWORD"] = settings.DATABASES[source][
"PASSWORD"]
if os.name == "nt":
# On windows restoring with pg_restore over a pipe is broken :-(
cmd = "pg_dump -c -Fp %s%s%s%s%s | psql %s%s%s%s"
else:
cmd = "pg_dump -Fc %s%s%s%s%s | pg_restore -n public -Fc -c --if-exists %s%s%s -d %s"
commandline = cmd % (
settings.DATABASES[source]["USER"] and
("-U %s " % settings.DATABASES[source]["USER"]) or "",
settings.DATABASES[source]["HOST"] and
("-h %s " % settings.DATABASES[source]["HOST"]) or "",
settings.DATABASES[source]["PORT"] and
("-p %s " % settings.DATABASES[source]["PORT"]) or "",
"""
-T common_user
-T common_scenario
-T auth_group
-T auth_group_permission
-T auth_permission
-T common_user_groups
-T common_user_user_permissions
-T common_preferences
-T reportmanager_report
""" if destination == DEFAULT_DB_ALIAS else "",
test and settings.DATABASES[source]["TEST"]["NAME"]
or settings.DATABASES[source]["NAME"],
settings.DATABASES[destination]["USER"] and
("-U %s " % settings.DATABASES[destination]["USER"]) or "",
settings.DATABASES[destination]["HOST"] and
("-h %s " % settings.DATABASES[destination]["HOST"]) or "",
settings.DATABASES[destination]["PORT"] and
("-p %s " % settings.DATABASES[destination]["PORT"]) or "",
test and settings.DATABASES[destination]["TEST"]["NAME"]
or settings.DATABASES[destination]["NAME"],
)
with subprocess.Popen(
commandline,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
) as p:
try:
task.processid = p.pid
task.save(using=source)
p.wait()
except Exception:
p.kill()
p.wait()
# Consider the destination database free again
destinationscenario.status = "Free"
destinationscenario.lastrefresh = datetime.today()
destinationscenario.save(using=DEFAULT_DB_ALIAS)
raise Exception("Database copy failed")
# Update the scenario table
destinationscenario.status = "In use"
destinationscenario.lastrefresh = datetime.today()
if options["description"]:
destinationscenario.description = options["description"]
destinationscenario.save(using=DEFAULT_DB_ALIAS)
# Give access to the destination scenario to:
# a) the user doing the copy
# b) all superusers from the source schema
# unless it's a promotion
if destination != DEFAULT_DB_ALIAS:
User.objects.using(destination).filter(
is_superuser=True).update(is_active=True)
User.objects.using(destination).filter(
is_superuser=False).update(is_active=False)
if user:
User.objects.using(destination).filter(
username=user.username).update(is_active=True)
# Logging message
task.processid = None
task.status = "Done"
task.finished = datetime.now()
# Update the task in the destination database
task.message = "Scenario %s from %s" % (
"promoted" if promote else "copied",
source,
)
task.save(using=destination)
task.message = "Scenario copied to %s" % destination
# Delete any waiting tasks in the new copy.
# This is needed for situations where the same source is copied to
# multiple destinations at the same moment.
Task.objects.all().using(destination).filter(
id__gt=task.id).delete()
except Exception as e:
if task:
task.status = "Failed"
task.message = "%s" % e
task.finished = datetime.now()
if destinationscenario and destinationscenario.status == "Busy":
destinationscenario.status = "Free"
destinationscenario.save(using=DEFAULT_DB_ALIAS)
raise e
finally:
if task:
task.processid = None
task.save(using=source)
settings.DEBUG = tmp_debug
def getHTML(request):
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
scenarios = Scenario.objects.using(DEFAULT_DB_ALIAS)
if scenarios.count() > 1:
javascript = '''
$("#sourceul li a").click(function(){
$("#source").html($(this).text() + ' <span class="caret"></span>');
$("#sourcescenario").val($(this).text());
});
'''
context = RequestContext(request, {'javascript': javascript, 'scenarios': scenarios})
template = Template('''
{% load i18n %}
<form role="form" method="post" action="{{request.prefix}}/execute/launch/scenario_copy/">{% csrf_token %}
<table id="scenarios">
<tr>
{% comment %}Translators: Translation included with Django {% endcomment %}
<th style="padding: 0px 15px;">{% trans 'scenario'|capfirst %}</th>
<th style="padding: 0px 15px;">{% trans 'status'|capfirst %}</th>
<th>{% trans 'description'|capfirst %}</th>
<th>{% trans 'last modified'|capfirst %}</th>
</tr>
{% for j in scenarios %}{% ifnotequal j.name 'default' %}
<tr>
<td style="padding: 0px 15px;"><input type=checkbox name="{{j.name}}" id="sc{{j.name}}"/>
<label for="sc{{j.name}}"> <strong>{{j.name|capfirst}}</strong>
</label>
</td>
{% with mystatus=j.status|lower %}
<td style="padding: 0px 15px;">{% trans mystatus|capfirst %}</td>
{% endwith %}
<td>{% if j.description %}{{j.description}}{% endif %}</td>
<td>{{j.lastrefresh|date:"DATETIME_FORMAT"}}</td>
</tr>
{% endifnotequal %}{% endfor %}
{% if perms.auth.copy_scenario %}
<tr>
<td><button class="btn btn-primary" name="copy" type="submit" value="{% trans "copy"|capfirst %}" style="width:100%">{% trans "copy"|capfirst %}</button>
</td>
<td style="padding: 0px 15px;" colspan="3">
{% trans "copy"|capfirst %}
<div class="dropdown dropdown-submit-input" style="display: inline-block;">
<button class="btn btn-default dropdown-toggle" id="source" value="" type="button" data-toggle="dropdown" style="min-width: 160px">- <span class="caret"></span></button>
<ul class="dropdown-menu" aria-labelledby="source" id="sourceul" style="top: auto">
{% for j in scenarios %}
{% ifequal j.status 'In use' %}
<li><a name="{{j.name}}">{{j.name}}</a></li>
{% endifequal %}
{% endfor %}
</ul>
</div>
{% trans "into selected scenarios" %}
</td>
</tr>
{% endif %}
{% if perms.auth.release_scenario %}
<tr>
<td><button class="btn btn-primary" name="release" type="submit" value="{% trans "release"|capfirst %}" style="width:100%">{% trans "release"|capfirst %}</button></td>
<td style="padding: 0px 15px;" colspan="3">{% trans "release selected scenarios"|capfirst %}</td>
</tr>
<tr>
<td><button class="btn btn-primary" name="update" type="submit" value="{% trans "update"|capfirst %}" style="width:100%">{% trans "update"|capfirst %}</button></td>
<td style="padding: 0px 15px;" colspan="3"><input class="form-control" name="description" type="text" size="40" placeholder="{% trans "Update description of selected scenarios" %}"/></td>
</tr>
{% endif %}
</table>
<input type="hidden" name="source" id="sourcescenario" value="">
</form>
<script>{{ javascript|safe }}</script>
''')
return template.render(context)
# A list of translation strings from the above
translated = (
_("copy"), _("release"), _("release selected scenarios"), _("into selected scenarios"),
_("update"), _("Update description of selected scenarios")
)
else:
return None
def getHTML(request):
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
scenarios = Scenario.objects.all().using(DEFAULT_DB_ALIAS)
if scenarios.count() > 1:
javascript = '''
$("#sourceul li a").click(function(){
$("#source").html($(this).text() + ' <span class="caret"></span>');
$("#sourcescenario").val($(this).text());
});
'''
context = RequestContext(request, {
'javascript': javascript,
'scenarios': scenarios
})
template = Template('''
{% load i18n %}
<form role="form" method="post" action="{{request.prefix}}/execute/launch/scenario_copy/">{% csrf_token %}
<table id="scenarios">
<tr>
{% comment %}Translators: Translation included with Django {% endcomment %}
<th style="padding: 0px 15px;">{% trans 'scenario'|capfirst %}</th>
<th style="padding: 0px 15px;">{% trans 'status'|capfirst %}</th>
<th>{% trans 'label'|capfirst %}</th>
<th>{% trans 'last refresh'|capfirst %}</th>
</tr>
{% for j in scenarios %}{% ifnotequal j.name 'default' %}
<tr>
<td style="padding: 0px 15px;"><input type=checkbox name="{{j.name}}" id="sc{{j.name}}"/>
<label for="sc{{j.name}}"> <strong>{{j.name|capfirst}}</strong>
</label>
</td>
<td style="padding: 0px 15px;">{{j.status}}</td>
<td>{% if j.description %}{{j.description}}{% endif %}</td>
<td>{{j.lastrefresh|date:"DATETIME_FORMAT"}}</td>
</tr>
{% endifnotequal %}{% endfor %}
{% if perms.auth.copy_scenario %}
<tr>
<td><button class="btn btn-primary" name="copy" type="submit" value="{% trans "copy"|capfirst %}" style="width:100%">{% trans "copy"|capfirst %}</button>
</td>
<td style="padding: 0px 15px;" colspan="3">
{% trans "copy"|capfirst %}
<div class="dropdown dropdown-submit-input" style="display: inline-block;">
<button class="btn btn-default dropdown-toggle" id="source" value="" type="button" data-toggle="dropdown" style="min-width: 160px">- <span class="caret"></span></button>
<ul class="dropdown-menu" aria-labelledby="source" id="sourceul" style="top: auto">
{% for j in scenarios %}
{% ifequal j.status 'In use' %}
<li><a name="{{j.name}}">{{j.name}}</a></li>
{% endifequal %}
{% endfor %}
</ul>
</div>
{% trans "into selected scenarios" %}
</td>
</tr>
{% endif %}
{% if perms.auth.release_scenario %}
<tr>
<td><button class="btn btn-primary" name="release" type="submit" value="{% trans "release"|capfirst %}" style="width:100%">{% trans "release"|capfirst %}</button></td>
<td style="padding: 0px 15px;" colspan="3">{% trans "release selected scenarios"|capfirst %}</td>
</tr>
<tr>
<td><button class="btn btn-primary" name="update" type="submit" value="{% trans "update"|capfirst %}" style="width:100%">{% trans "update"|capfirst %}</button></td>
<td style="padding: 0px 15px;" colspan="3"><input class="form-control" name="description" type="text" size="40" placeholder="{% trans "Update description of selected scenarios" %}"/></td>
</tr>
{% endif %}
</table>
<input type="hidden" name="source" id="sourcescenario" value="">
</form>
<script>{{ javascript|safe }}</script>
''')
return template.render(context)
# A list of translation strings from the above
translated = (_("copy"), _("release"),
_("release selected scenarios"),
_("into selected scenarios"), _("update"),
_("Update description of selected scenarios"))
else:
return None
def Run(self):
# Import modules
import cherrypy
from cherrypy.wsgiserver import CherryPyWSGIServer
from subprocess import call, DEVNULL
from win32process import DETACHED_PROCESS, CREATE_NO_WINDOW
# Initialize django
os.environ.setdefault('DJANGO_SETTINGS_MODULE', "freppledb.settings")
os.environ.setdefault('FREPPLE_APP',
os.path.join(sys.path[0], 'custom'))
import django
django.setup()
from django.conf import settings
from django.core.handlers.wsgi import WSGIHandler
from django.contrib.staticfiles.handlers import StaticFilesHandler
# Override the debugging settings
settings.DEBUG = False
settings.TEMPLATE_DEBUG = False
# Sys.path contains the zip file with all packages. We need to put the
# application directory into the path as well.
sys.path += [os.environ['FREPPLE_APP']]
# Append all output to a unbuffered log stream
with open(os.path.join(settings.FREPPLE_LOGDIR, 'service.log'),
'a') as logfile:
sys.stderr = sys.stdout = logfile
try:
# Using the included postgres database
# Check if the database is running. If not, start it.
if os.path.exists(
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql',
'bin', 'pg_ctl.exe')):
status = call([
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql',
'bin', 'pg_ctl.exe'), "--pgdata",
os.path.join(settings.FREPPLE_LOGDIR, 'database'),
"--silent", "status"
],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW)
if status:
print("%s\tStarting the PostgreSQL database" %
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
flush=True)
call([
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql',
'bin', 'pg_ctl.exe'), "--pgdata",
os.path.join(settings.FREPPLE_LOGDIR, 'database'),
"--log",
os.path.join(settings.FREPPLE_LOGDIR, 'database',
'server.log'), "start"
],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL,
creationflags=DETACHED_PROCESS)
# Prepare web server
cherrypy.config.update({
'global': {
'log.screen': False,
'tools.log_tracebacks.on': True,
'engine.autoreload.on': False,
'engine.SIGHUP': None,
'engine.SIGTERM': None
}
})
self.server = CherryPyWSGIServer(
(settings.ADDRESS, settings.PORT),
StaticFilesHandler(WSGIHandler()))
# Synchronize the scenario table with the settings
from freppledb.common.models import Scenario
Scenario.syncWithSettings()
# Infinite loop serving requests
# The loop gets interrupted when the service gets ordered to shut down.
print("%s\tfrePPLe web server listening on http://%s:%d" %
(datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
settings.ADDRESS, settings.PORT),
flush=True)
self.server.start()
print("%s\tStopping service" %
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
flush=True)
# Using the included postgres database?
if os.path.exists(
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql',
'bin', 'pg_ctl.exe')):
# Check if the database is running. If so, stop it.
os.environ['PATH'] = os.path.join(
settings.FREPPLE_HOME, '..', 'pgsql',
'bin') + os.pathsep + os.environ['PATH']
status = call([
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql',
'bin', 'pg_ctl.exe'), "--pgdata",
os.path.join(settings.FREPPLE_LOGDIR, 'database'),
"--silent", "status"
],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW)
if not status:
print("%s\tShutting down the database" %
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
flush=True)
call(
[
os.path.join(settings.FREPPLE_HOME, '..',
'pgsql', 'bin', 'pg_ctl.exe'),
"--pgdata",
os.path.join(settings.FREPPLE_LOGDIR,
'database'),
"--log",
os.path.join(settings.FREPPLE_LOGDIR,
'database', 'server.log'),
"-w", # Wait till it's down
"stop"
],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW)
# Notify the manager
self.stopEvent.set()
except Exception as e:
print("%s\tfrePPLe web server failure: %s" %
(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), e),
flush=True)
def main():
# Environment settings (which are used in the Django settings file and need
# to be updated BEFORE importing the settings)
os.environ.setdefault('FREPPLE_HOME', sys.path[0])
os.environ.setdefault('DJANGO_SETTINGS_MODULE', "freppledb.settings")
os.environ.setdefault('FREPPLE_APP', os.path.join(sys.path[0], 'custom'))
os.environ.setdefault(
'PYTHONPATH',
os.path.join(sys.path[0], 'lib', 'library.zip') + os.pathsep +
os.path.join(sys.path[0], 'lib'))
# Add the custom directory to the Python path.
sys.path += [
os.environ['FREPPLE_APP'],
]
# Initialize django
import django
django.setup()
# Import django
from django.core.management import execute_from_command_line
from django.conf import settings
if os.path.exists(
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin',
'pg_ctl.exe')):
# Using the included postgres database
# Check if the database is running. If not, start it.
os.environ['PATH'] = os.path.join(
settings.FREPPLE_HOME, '..', 'pgsql',
'bin') + os.pathsep + os.environ['PATH']
status = call([
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin',
'pg_ctl.exe'), "--pgdata",
os.path.join(settings.FREPPLE_LOGDIR, 'database'), "--silent",
"status"
],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW)
if status:
print("Starting the PostgreSQL database now",
settings.FREPPLE_LOGDIR)
call(
[
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin',
'pg_ctl.exe'),
"--pgdata",
os.path.join(settings.FREPPLE_LOGDIR, 'database'),
"--log",
os.path.join(settings.FREPPLE_LOGDIR, 'database',
'server.log'),
"-w", # Wait till it's up
"start"
],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW)
# Synchronize the scenario table with the settings
from freppledb.common.models import Scenario
Scenario.syncWithSettings()
# Execute the command
execute_from_command_line(sys.argv)
def getHTML(request):
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
scenarios = Scenario.objects.using(DEFAULT_DB_ALIAS)
if scenarios.count() <= 1:
return None
release_perm = []
copy_perm = []
promote_perm = []
active_scenarios = []
free_scenarios = []
in_use_scenarios = []
dumps = []
# look for dump files in the log folder of production
for f in sorted(os.listdir(settings.FREPPLE_LOGDIR)):
if os.path.isfile(
os.path.join(settings.FREPPLE_LOGDIR, f)
) and f.lower().endswith(".dump"):
dumps.append(f)
for scenario in scenarios:
try:
user = User.objects.using(scenario.name).get(
username=request.user.username
)
if scenario.status != "Free":
in_use_scenarios.append(scenario.name)
else:
free_scenarios.append(scenario.name)
if user.has_perm("common.release_scenario"):
release_perm.append(scenario.name)
if user.has_perm("common.promote_scenario"):
promote_perm.append(scenario.name)
if user.has_perm("common.copy_scenario"):
copy_perm.append(scenario.name)
if user.is_active:
active_scenarios.append(scenario.name)
except Exception:
# database schema is not properly created, scenario is free
free_scenarios.append(scenario.name)
active_scenarios.append(scenario.name)
# If all scenarios are in use and user is inactive in all of them then he won't see the scenario management menu
if len(free_scenarios) == 0 and len(active_scenarios) == 1:
return None
return render_to_string(
"commands/scenario_copy.html",
{
"scenarios": scenarios,
"DEFAULT_DB_ALIAS": DEFAULT_DB_ALIAS,
"current_database": request.database,
"release_perm": release_perm,
"copy_perm": copy_perm,
"promote_perm": promote_perm,
"active_scenarios": active_scenarios,
"free_scenarios": free_scenarios,
"dumps": dumps,
},
request=request,
)
def Run(self):
# Import modules
from cheroot import wsgi
from subprocess import call, DEVNULL
from win32process import DETACHED_PROCESS, CREATE_NO_WINDOW
# Initialize django
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "freppledb.settings")
os.environ.setdefault("FREPPLE_APP",
os.path.join(sys.path[0], "custom"))
import django
django.setup()
from django.conf import settings
from django.core.handlers.wsgi import WSGIHandler
from django.contrib.staticfiles.handlers import StaticFilesHandler
# Override the debugging settings
settings.DEBUG = False
settings.TEMPLATE_DEBUG = False
# Sys.path contains the zip file with all packages. We need to put the
# application directory into the path as well.
sys.path += [os.environ["FREPPLE_APP"]]
# Append all output to a unbuffered log stream
with open(os.path.join(settings.FREPPLE_LOGDIR, "service.log"),
"a") as logfile:
sys.stderr = sys.stdout = logfile
try:
# Using the included postgres database
# Check if the database is running. If not, start it.
if os.path.exists(
os.path.join(settings.FREPPLE_HOME, "..", "pgsql",
"bin", "pg_ctl.exe")):
status = call(
[
os.path.join(
settings.FREPPLE_HOME,
"..",
"pgsql",
"bin",
"pg_ctl.exe",
),
"--pgdata",
os.path.join(settings.FREPPLE_LOGDIR, "database"),
"--silent",
"status",
],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW,
)
if status:
print(
"%s\tStarting the PostgreSQL database" %
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
flush=True,
)
call(
[
os.path.join(
settings.FREPPLE_HOME,
"..",
"pgsql",
"bin",
"pg_ctl.exe",
),
"--pgdata",
os.path.join(settings.FREPPLE_LOGDIR,
"database"),
"--log",
os.path.join(settings.FREPPLE_LOGDIR,
"database", "server.log"),
"start",
],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL,
creationflags=DETACHED_PROCESS,
)
# Prepare web server
self.server = wsgi.Server((settings.ADDRESS, settings.PORT),
StaticFilesHandler(WSGIHandler()))
# Synchronize the scenario table with the settings
from freppledb.common.models import Scenario
Scenario.syncWithSettings()
# Infinite loop serving requests
# The loop gets interrupted when the service gets ordered to shut down.
print(
"%s\tfrePPLe web server listening on http://%s:%d" % (
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
settings.ADDRESS,
settings.PORT,
),
flush=True,
)
self.server.start()
print(
"%s\tStopping service" %
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
flush=True,
)
# Using the included postgres database?
if os.path.exists(
os.path.join(settings.FREPPLE_HOME, "..", "pgsql",
"bin", "pg_ctl.exe")):
# Check if the database is running. If so, stop it.
os.environ["PATH"] = (os.path.join(settings.FREPPLE_HOME,
"..", "pgsql", "bin") +
os.pathsep + os.environ["PATH"])
status = call(
[
os.path.join(
settings.FREPPLE_HOME,
"..",
"pgsql",
"bin",
"pg_ctl.exe",
),
"--pgdata",
os.path.join(settings.FREPPLE_LOGDIR, "database"),
"--silent",
"status",
],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW,
)
if not status:
print(
"%s\tShutting down the database" %
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
flush=True,
)
call(
[
os.path.join(
settings.FREPPLE_HOME,
"..",
"pgsql",
"bin",
"pg_ctl.exe",
),
"--pgdata",
os.path.join(settings.FREPPLE_LOGDIR,
"database"),
"--log",
os.path.join(settings.FREPPLE_LOGDIR,
"database", "server.log"),
"-w", # Wait till it's down
"stop",
],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW,
)
# Notify the manager
self.stopEvent.set()
except Exception as e:
print(
"%s\tfrePPLe web server failure: %s" %
(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), e),
flush=True,
)
def extra_context(reportclass, request, *args, **kwargs):
try:
constraint = int(request.session["constraint"])
except:
constraint = 15
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Loop over all fixtures of all apps and directories
fixtures = set()
folders = list(settings.FIXTURE_DIRS)
for app in get_apps():
if app.__name__.startswith("django"):
continue
folders.append(os.path.join(os.path.dirname(app.__file__), "fixtures"))
for f in folders:
try:
for root, dirs, files in os.walk(f):
for i in files:
if i.endswith(".json"):
fixtures.add(i.split(".")[0])
except:
pass # Silently ignore failures
fixtures = sorted(fixtures)
# Function to convert from bytes to human readabl format
def sizeof_fmt(num):
for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
if abs(num) < 1024.0:
return "%3.1f%sB" % (num, unit)
num /= 1024.0
return "%.1f%sB" % (num, "Yi")
# List available data files
filestoupload = []
if "FILEUPLOADFOLDER" in settings.DATABASES[request.database]:
uploadfolder = settings.DATABASES[request.database]["FILEUPLOADFOLDER"]
if os.path.isdir(uploadfolder):
for file in os.listdir(uploadfolder):
if file.endswith(".csv") or file.endswith(".log"):
filestoupload.append(
[
file,
strftime(
"%Y-%m-%d %H:%M:%S", localtime(os.stat(os.path.join(uploadfolder, file)).st_mtime)
),
sizeof_fmt(os.stat(os.path.join(uploadfolder, file)).st_size),
]
)
# Send to template
odoo = "freppledb.odoo" in settings.INSTALLED_APPS
return {
"capacityconstrained": constraint & 4,
"materialconstrained": constraint & 2,
"leadtimeconstrained": constraint & 1,
"fenceconstrained": constraint & 8,
"scenarios": Scenario.objects.all(),
"fixtures": fixtures,
"openbravo": "freppledb.openbravo" in settings.INSTALLED_APPS,
"odoo": odoo,
"odoo_read": odoo and request.session.get("odoo_read", False),
"odoo_write": odoo and request.session.get("odoo_write", False),
"filestoupload": filestoupload,
"datafolderconfigured": "FILEUPLOADFOLDER" in settings.DATABASES[request.database],
}
def extra_context(reportclass, request, *args, **kwargs):
try:
constraint = int(request.session['constraint'])
except:
constraint = 15
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Collect optional tasks
PlanTaskRegistry.autodiscover()
planning_options = PlanTaskRegistry.getLabels()
# Loop over all fixtures of all apps and directories
fixtures = set()
folders = list(settings.FIXTURE_DIRS)
for app in get_apps():
if app.__name__.startswith('django'):
continue
folders.append(os.path.join(os.path.dirname(app.__file__), 'fixtures'))
for f in folders:
try:
for root, dirs, files in os.walk(f):
for i in files:
if i.endswith('.json'):
fixtures.add(i.split('.')[0])
except:
pass # Silently ignore failures
fixtures = sorted(fixtures)
# Function to convert from bytes to human readabl format
def sizeof_fmt(num):
for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
if abs(num) < 1024.0:
return "%3.1f%sB" % (num, unit)
num /= 1024.0
return "%.1f%sB" % (num, 'Yi')
# List available data files
filestoupload = []
if 'FILEUPLOADFOLDER' in settings.DATABASES[request.database]:
uploadfolder = settings.DATABASES[request.database]['FILEUPLOADFOLDER']
if os.path.isdir(uploadfolder):
for file in os.listdir(uploadfolder):
if file.endswith('.csv') or file.endswith('.log'):
filestoupload.append([
file,
strftime("%Y-%m-%d %H:%M:%S",localtime(os.stat(os.path.join(uploadfolder, file)).st_mtime)),
sizeof_fmt(os.stat(os.path.join(uploadfolder, file)).st_size)
])
# Send to template
return {
'capacityconstrained': constraint & 4,
'materialconstrained': constraint & 2,
'leadtimeconstrained': constraint & 1,
'fenceconstrained': constraint & 8,
'scenarios': Scenario.objects.all(),
'fixtures': fixtures,
'openbravo': 'freppledb.openbravo' in settings.INSTALLED_APPS,
'planning_options': planning_options,
'current_options': request.session.get('env', [ i[0] for i in planning_options ]),
'filestoupload': filestoupload,
'datafolderconfigured': 'FILEUPLOADFOLDER' in settings.DATABASES[request.database]
}
def __call__(self, request):
# Make request information available throughout the application
setattr(_thread_locals, "request", request)
if not hasattr(request, "user"):
request.user = auth.get_user(request)
if not hasattr(request.user, "scenarios"):
# A scenario list is not available on the request
for i in settings.DATABASES:
try:
if settings.DATABASES[i]["regexp"].match(request.path):
scenario = Scenario.objects.using(DEFAULT_DB_ALIAS).get(name=i)
if scenario.status != "In use":
return HttpResponseNotFound("Scenario not in use")
request.prefix = "/%s" % i
request.path_info = request.path_info[len(request.prefix) :]
request.path = request.path[len(request.prefix) :]
request.database = i
if hasattr(request.user, "_state"):
request.user._state.db = i.name
response = self.get_response(request)
if not response.streaming:
# Note: Streaming response get the request field cleared in the
# request_finished signal handler
setattr(_thread_locals, "request", None)
return response
except Exception:
pass
request.prefix = ""
request.database = DEFAULT_DB_ALIAS
if hasattr(request.user, "_state"):
request.user._state.db = DEFAULT_DB_ALIAS
else:
# A list of scenarios is already available
if request.user.is_anonymous:
return self.get_response(request)
default_scenario = None
for i in request.user.scenarios:
if i.name == DEFAULT_DB_ALIAS:
default_scenario = i
try:
if settings.DATABASES[i.name]["regexp"].match(request.path):
request.prefix = "/%s" % i.name
request.path_info = request.path_info[len(request.prefix) :]
request.path = request.path[len(request.prefix) :]
request.database = i.name
request.scenario = i
if hasattr(request.user, "_state"):
request.user._state.db = i.name
request.user.is_superuser = i.is_superuser
response = self.get_response(request)
if not response.streaming:
# Note: Streaming response get the request field cleared in the
# request_finished signal handler
setattr(_thread_locals, "request", None)
return response
except Exception:
pass
request.prefix = ""
request.database = DEFAULT_DB_ALIAS
if hasattr(request.user, "_state"):
request.user._state.db = DEFAULT_DB_ALIAS
if default_scenario:
request.scenario = default_scenario
else:
request.scenario = Scenario(name=DEFAULT_DB_ALIAS)
response = self.get_response(request)
if not response.streaming:
# Note: Streaming response get the request field cleared in the
# request_finished signal handler
setattr(_thread_locals, "request", None)
return response
def handle(self, **options):
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
# Pick up options
force = options['force']
test = 'FREPPLE_TEST' in os.environ
if options['user']:
try:
user = User.objects.all().get(username=options['user'])
except:
raise CommandError("User '%s' not found" % options['user'] )
else:
user = None
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Initialize the task
source = options['source']
try:
sourcescenario = Scenario.objects.using(DEFAULT_DB_ALIAS).get(pk=source)
except:
raise CommandError("No source database defined with name '%s'" % source)
now = datetime.now()
task = None
if 'task' in options and options['task']:
try:
task = Task.objects.all().using(source).get(pk=options['task'])
except:
raise CommandError("Task identifier not found")
if task.started or task.finished or task.status != "Waiting" or task.name not in ('frepple_copy', 'scenario_copy'):
raise CommandError("Invalid task identifier")
task.status = '0%'
task.started = now
else:
task = Task(name='scenario_copy', submitted=now, started=now, status='0%', user=user)
task.processid = os.getpid()
task.save(using=source)
# Validate the arguments
destination = options['destination']
destinationscenario = None
try:
task.arguments = "%s %s" % (source, destination)
if options['description']:
task.arguments += '--description="%s"' % options['description'].replace('"', '\\"')
if force:
task.arguments += " --force"
task.save(using=source)
try:
destinationscenario = Scenario.objects.using(DEFAULT_DB_ALIAS).get(pk=destination)
except:
raise CommandError("No destination database defined with name '%s'" % destination)
if source == destination:
raise CommandError("Can't copy a schema on itself")
if settings.DATABASES[source]['ENGINE'] != settings.DATABASES[destination]['ENGINE']:
raise CommandError("Source and destination scenarios have a different engine")
if sourcescenario.status != 'In use':
raise CommandError("Source scenario is not in use")
if destinationscenario.status != 'Free' and not force:
raise CommandError("Destination scenario is not free")
# Logging message - always logging in the default database
destinationscenario.status = 'Busy'
destinationscenario.save(using=DEFAULT_DB_ALIAS)
# Copying the data
# Commenting the next line is a little more secure, but requires you to create a .pgpass file.
if settings.DATABASES[source]['PASSWORD']:
os.environ['PGPASSWORD'] = settings.DATABASES[source]['PASSWORD']
if os.name == 'nt':
# On windows restoring with pg_restore over a pipe is broken :-(
cmd = "pg_dump -c -Fp %s%s%s%s | psql %s%s%s%s"
else:
cmd = "pg_dump -Fc %s%s%s%s | pg_restore -n public -Fc -c --if-exists %s%s%s -d %s"
commandline = cmd % (
settings.DATABASES[source]['USER'] and ("-U %s " % settings.DATABASES[source]['USER']) or '',
settings.DATABASES[source]['HOST'] and ("-h %s " % settings.DATABASES[source]['HOST']) or '',
settings.DATABASES[source]['PORT'] and ("-p %s " % settings.DATABASES[source]['PORT']) or '',
test and settings.DATABASES[source]['TEST']['NAME'] or settings.DATABASES[source]['NAME'],
settings.DATABASES[destination]['USER'] and ("-U %s " % settings.DATABASES[destination]['USER']) or '',
settings.DATABASES[destination]['HOST'] and ("-h %s " % settings.DATABASES[destination]['HOST']) or '',
settings.DATABASES[destination]['PORT'] and ("-p %s " % settings.DATABASES[destination]['PORT']) or '',
test and settings.DATABASES[destination]['TEST']['NAME'] or settings.DATABASES[destination]['NAME'],
)
with subprocess.Popen(commandline, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) as p:
try:
task.processid = p.pid
task.save(using=source)
p.wait()
except:
p.kill()
p.wait()
# Consider the destination database free again
destinationscenario.status = 'Free'
destinationscenario.lastrefresh = datetime.today()
destinationscenario.save(using=DEFAULT_DB_ALIAS)
raise Exception("Database copy failed")
# Update the scenario table
destinationscenario.status = 'In use'
destinationscenario.lastrefresh = datetime.today()
if 'description' in options:
destinationscenario.description = options['description']
destinationscenario.save(using=DEFAULT_DB_ALIAS)
# Give access to the destination scenario to:
# a) the user doing the copy
# b) all superusers from the source schema
User.objects.using(destination).filter(is_superuser=True).update(is_active=True)
User.objects.using(destination).filter(is_superuser=False).update(is_active=False)
if user:
User.objects.using(destination).filter(username=user.username).update(is_active=True)
# Logging message
task.processid = None
task.status = 'Done'
task.finished = datetime.now()
# Update the task in the destination database
task.message = "Scenario copied from %s" % source
task.save(using=destination)
task.message = "Scenario copied to %s" % destination
# Delete any waiting tasks in the new copy.
# This is needed for situations where the same source is copied to
# multiple destinations at the same moment.
Task.objects.all().using(destination).filter(id__gt=task.id).delete()
except Exception as e:
if task:
task.status = 'Failed'
task.message = '%s' % e
task.finished = datetime.now()
if destinationscenario and destinationscenario.status == 'Busy':
destinationscenario.status = 'Free'
destinationscenario.save(using=DEFAULT_DB_ALIAS)
raise e
finally:
if task:
task.processid = None
task.save(using=source)
settings.DEBUG = tmp_debug
def handle(self, **options):
if options["user"]:
try:
user = User.objects.all().get(username=options["user"])
except Exception:
raise CommandError("User '%s' not found" % options["user"])
else:
user = None
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
now = datetime.now()
task = None
database = options["database"]
if "task" in options and options["task"]:
try:
task = Task.objects.all().using(database).get(
pk=options["task"])
except Exception:
raise CommandError("Task identifier not found")
if (task.started or task.finished or task.status != "Waiting"
or task.name != "scenario_release"):
raise CommandError("Invalid task identifier")
task.status = "0%"
task.started = now
else:
task = Task(
name="scenario_release",
submitted=now,
started=now,
status="0%",
user=user,
)
task.processid = os.getpid()
task.save(using=database)
# Validate the arguments
try:
releasedScenario = None
try:
releasedScenario = Scenario.objects.using(
DEFAULT_DB_ALIAS).get(pk=database)
except Exception:
raise CommandError(
"No destination database defined with name '%s'" %
database)
if database == DEFAULT_DB_ALIAS:
raise CommandError("Production scenario cannot be released.")
if releasedScenario.status != "In use":
raise CommandError("Scenario to release is not in use")
# Update the scenario table, set it free in the production database
releasedScenario.status = "Free"
releasedScenario.lastrefresh = datetime.today()
releasedScenario.save(using=DEFAULT_DB_ALIAS)
# Killing webservice
if "freppledb.webservice" in settings.INSTALLED_APPS:
management.call_command("stopwebservice",
force=True,
database=database)
# Logging message
task.processid = None
task.status = "Done"
task.finished = datetime.now()
# Update the task in the destination database
task.message = "Scenario %s released" % (database, )
task.save(using=database)
except Exception as e:
if task:
task.status = "Failed"
task.message = "%s" % e
task.finished = datetime.now()
if releasedScenario and releasedScenario.status == "Busy":
releasedScenario.status = "Free"
releasedScenario.save(using=DEFAULT_DB_ALIAS)
raise e
finally:
if task:
task.processid = None
task.save(using=database)
def handle(self, *args, **options):
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
# Pick up options
if 'force' in options:
force = options['force']
else:
force = False
test = 'FREPPLE_TEST' in os.environ
if 'user' in options and options['user']:
try:
user = User.objects.all().get(username=options['user'])
except:
raise CommandError("User '%s' not found" % options['user'] )
else:
user = None
# Initialize the task
now = datetime.now()
task = None
if 'task' in options and options['task']:
try:
task = Task.objects.all().get(pk=options['task'])
except:
raise CommandError("Task identifier not found")
if task.started or task.finished or task.status != "Waiting" or task.name != 'copy scenario':
raise CommandError("Invalid task identifier")
task.status = '0%'
task.started = now
else:
task = Task(name='copy scenario', submitted=now, started=now, status='0%', user=user)
task.save()
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Validate the arguments
destinationscenario = None
try:
if len(args) != 2:
raise CommandError("Command takes exactly 2 arguments.")
task.arguments = "%s %s" % (args[0], args[1])
task.save()
source = args[0]
try:
sourcescenario = Scenario.objects.get(pk=source)
except:
raise CommandError("No source database defined with name '%s'" % source)
destination = args[1]
try:
destinationscenario = Scenario.objects.get(pk=destination)
except:
raise CommandError("No destination database defined with name '%s'" % destination)
if source == destination:
raise CommandError("Can't copy a schema on itself")
if settings.DATABASES[source]['ENGINE'] != settings.DATABASES[destination]['ENGINE']:
raise CommandError("Source and destination scenarios have a different engine")
if sourcescenario.status != 'In use':
raise CommandError("Source scenario is not in use")
if destinationscenario.status != 'Free' and not force:
raise CommandError("Destination scenario is not free")
# Logging message - always logging in the default database
destinationscenario.status = 'Busy'
destinationscenario.save()
# Copying the data
# Commenting the next line is a little more secure, but requires you to create a .pgpass file.
if settings.DATABASES[source]['PASSWORD']:
os.environ['PGPASSWORD'] = settings.DATABASES[source]['PASSWORD']
commandline = "pg_dump -c -Fp %s%s%s%s | psql %s%s%s%s" % (
settings.DATABASES[source]['USER'] and ("-U %s " % settings.DATABASES[source]['USER']) or '',
settings.DATABASES[source]['HOST'] and ("-h %s " % settings.DATABASES[source]['HOST']) or '',
settings.DATABASES[source]['PORT'] and ("-p %s " % settings.DATABASES[source]['PORT']) or '',
test and settings.DATABASES[source]['TEST']['NAME'] or settings.DATABASES[source]['NAME'],
settings.DATABASES[destination]['USER'] and ("-U %s " % settings.DATABASES[destination]['USER']) or '',
settings.DATABASES[destination]['HOST'] and ("-h %s " % settings.DATABASES[destination]['HOST']) or '',
settings.DATABASES[destination]['PORT'] and ("-p %s " % settings.DATABASES[destination]['PORT']) or '',
test and settings.DATABASES[destination]['TEST']['NAME'] or settings.DATABASES[destination]['NAME'],
)
ret = subprocess.call(commandline, shell=True, stdout=subprocess.DEVNULL , stderr=subprocess.STDOUT)
if ret:
raise Exception('Exit code of the database copy command is %d' % ret)
# Update the scenario table
destinationscenario.status = 'In use'
destinationscenario.lastrefresh = datetime.today()
if 'description' in options:
destinationscenario.description = options['description']
else:
destinationscenario.description = "Copied from scenario '%s'" % source
destinationscenario.save()
# Give access to the destination scenario to:
# a) the user doing the copy
# b) all superusers from the source schema
User.objects.using(destination).filter(is_superuser=True).update(is_active=True)
User.objects.using(destination).filter(is_superuser=False).update(is_active=False)
if user:
User.objects.using(destination).filter(username=user.username).update(is_active=True)
# Logging message
task.status = 'Done'
task.finished = datetime.now()
except Exception as e:
if task:
task.status = 'Failed'
task.message = '%s' % e
task.finished = datetime.now()
if destinationscenario and destinationscenario.status == 'Busy':
destinationscenario.status = 'Free'
destinationscenario.save()
raise e
finally:
if task:
task.save()
settings.DEBUG = tmp_debug
os.path.join(settings.FREPPLE_LOGDIR, "database"),
"--log",
os.path.join(settings.FREPPLE_LOGDIR, "database", "server.log"),
"-w", # Wait till it's up
"start",
],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
creationflags=CREATE_NO_WINDOW,
)
# Synchronize the scenario table with the settings
from freppledb.common.models import Scenario
Scenario.syncWithSettings()
# Import modules
from django.core.handlers.wsgi import WSGIHandler
from django.contrib.staticfiles.handlers import StaticFilesHandler
from django.db import DEFAULT_DB_ALIAS
# Determine the port number
port = options.port or settings.PORT
# Determine the IP-address to listen on:
# - either as command line argument
# - either 0.0.0.0 by default, which means all active IPv4 interfaces
address = options.address or settings.ADDRESS
# Validate the address and port number
def Run(self):
# Import modules
import cherrypy
from cherrypy.wsgiserver import CherryPyWSGIServer
from stat import S_ISDIR, ST_MODE
from subprocess import call, DEVNULL
from win32process import DETACHED_PROCESS, CREATE_NO_WINDOW
# Initialize django
os.environ.setdefault('DJANGO_SETTINGS_MODULE', "freppledb.settings")
os.environ.setdefault('FREPPLE_APP', os.path.join(sys.path[0],'custom'))
import django
django.setup()
from django.core import management
from django.conf import settings
from django.core.handlers.wsgi import WSGIHandler
from django.contrib.staticfiles.handlers import StaticFilesHandler
# Override the debugging settings
settings.DEBUG = False
settings.TEMPLATE_DEBUG = False
# Sys.path contains the zip file with all packages. We need to put the
# application directory into the path as well.
sys.path += [ os.environ['FREPPLE_APP'] ]
# Append all output to a unbuffered log stream
with open(os.path.join(settings.FREPPLE_LOGDIR,'service.log'), 'a') as logfile:
sys.stderr = sys.stdout = logfile
try:
# Using the included postgres database
# Check if the database is running. If not, start it.
if os.path.exists(os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin', 'pg_ctl.exe')):
status = call([
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin', 'pg_ctl.exe'),
"--pgdata", os.path.join(settings.FREPPLE_LOGDIR, 'database'),
"--silent",
"status"
],
stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW
)
if status:
print("%s\tStarting the PostgreSQL database" % datetime.now().strftime("%Y-%m-%d %H:%M:%S"), flush=True)
call([
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin', 'pg_ctl.exe'),
"--pgdata", os.path.join(settings.FREPPLE_LOGDIR, 'database'),
"--log", os.path.join(settings.FREPPLE_LOGDIR, 'database', 'server.log'),
"start"
],
stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL,
creationflags=DETACHED_PROCESS
)
# Prepare web server
cherrypy.config.update({
'global':{
'log.screen': False,
'tools.log_tracebacks.on': True,
'engine.autoreload.on': False,
'engine.SIGHUP': None,
'engine.SIGTERM': None
}
})
self.server = CherryPyWSGIServer(('127.0.0.1', settings.PORT),
StaticFilesHandler(WSGIHandler())
)
# Synchronize the scenario table with the settings
from freppledb.common.models import Scenario
Scenario.syncWithSettings()
# Infinite loop serving requests
# The loop gets interrupted when the service gets ordered to shut down.
print("%s\tfrePPLe web server listening on http://localhost:%d" % (
datetime.now().strftime("%Y-%m-%d %H:%M:%S"), settings.PORT
), flush=True)
self.server.start()
print("%s\tStopping service" % datetime.now().strftime("%Y-%m-%d %H:%M:%S"), flush=True)
# Using the included postgres database?
if os.path.exists(os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin', 'pg_ctl.exe')):
# Check if the database is running. If so, stop it.
os.environ['PATH'] = os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin') + os.pathsep + os.environ['PATH']
from subprocess import call, DEVNULL
status = call([
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin', 'pg_ctl.exe'),
"--pgdata", os.path.join(settings.FREPPLE_LOGDIR, 'database'),
"--silent",
"status"
],
stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW
)
if not status:
print("%s\tShutting down the database", datetime.now().strftime("%Y-%m-%d %H:%M:%S"), flush=True)
call([
os.path.join(settings.FREPPLE_HOME, '..', 'pgsql', 'bin', 'pg_ctl.exe'),
"--pgdata", os.path.join(settings.FREPPLE_LOGDIR, 'database'),
"--log", os.path.join(settings.FREPPLE_LOGDIR, 'database', 'server.log'),
"-w", # Wait till it's down
"stop"
],
stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL,
creationflags=CREATE_NO_WINDOW
)
# Notify the manager
self.stopEvent.set()
except Exception as e:
print("%s\tfrePPLe web server failure: %s" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"), e), flush=True)
def handle(self, **options):
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
# Pick up options
force = options['force']
test = 'FREPPLE_TEST' in os.environ
if options['user']:
try:
user = User.objects.all().get(username=options['user'])
except:
raise CommandError("User '%s' not found" % options['user'])
else:
user = None
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Initialize the task
source = options['source']
try:
sourcescenario = Scenario.objects.using(DEFAULT_DB_ALIAS).get(
pk=source)
except:
raise CommandError("No source database defined with name '%s'" %
source)
now = datetime.now()
task = None
if 'task' in options and options['task']:
try:
task = Task.objects.all().using(source).get(pk=options['task'])
except:
raise CommandError("Task identifier not found")
if task.started or task.finished or task.status != "Waiting" or task.name not in (
'frepple_copy', 'scenario_copy'):
raise CommandError("Invalid task identifier")
task.status = '0%'
task.started = now
else:
task = Task(name='scenario_copy',
submitted=now,
started=now,
status='0%',
user=user)
task.save(using=source)
# Validate the arguments
destination = options['destination']
destinationscenario = None
try:
task.arguments = "%s %s" % (source, destination)
if options['description']:
task.arguments += '--description="%s"' % options[
'description'].replace('"', '\\"')
if force:
task.arguments += " --force"
task.save(using=source)
try:
destinationscenario = Scenario.objects.using(
DEFAULT_DB_ALIAS).get(pk=destination)
except:
raise CommandError(
"No destination database defined with name '%s'" %
destination)
if source == destination:
raise CommandError("Can't copy a schema on itself")
if settings.DATABASES[source]['ENGINE'] != settings.DATABASES[
destination]['ENGINE']:
raise CommandError(
"Source and destination scenarios have a different engine")
if sourcescenario.status != 'In use':
raise CommandError("Source scenario is not in use")
if destinationscenario.status != 'Free' and not force:
raise CommandError("Destination scenario is not free")
# Logging message - always logging in the default database
destinationscenario.status = 'Busy'
destinationscenario.save(using=DEFAULT_DB_ALIAS)
# Copying the data
# Commenting the next line is a little more secure, but requires you to create a .pgpass file.
if settings.DATABASES[source]['PASSWORD']:
os.environ['PGPASSWORD'] = settings.DATABASES[source][
'PASSWORD']
if os.name == 'nt':
# On windows restoring with pg_restore over a pipe is broken :-(
cmd = "pg_dump -c -Fp %s%s%s%s | psql %s%s%s%s"
else:
cmd = "pg_dump -Fc %s%s%s%s | pg_restore -n public -Fc -c --if-exists %s%s%s -d %s"
commandline = cmd % (
settings.DATABASES[source]['USER'] and
("-U %s " % settings.DATABASES[source]['USER']) or '',
settings.DATABASES[source]['HOST'] and
("-h %s " % settings.DATABASES[source]['HOST']) or '',
settings.DATABASES[source]['PORT'] and
("-p %s " % settings.DATABASES[source]['PORT']) or '',
test and settings.DATABASES[source]['TEST']['NAME']
or settings.DATABASES[source]['NAME'],
settings.DATABASES[destination]['USER'] and
("-U %s " % settings.DATABASES[destination]['USER']) or '',
settings.DATABASES[destination]['HOST'] and
("-h %s " % settings.DATABASES[destination]['HOST']) or '',
settings.DATABASES[destination]['PORT'] and
("-p %s " % settings.DATABASES[destination]['PORT']) or '',
test and settings.DATABASES[destination]['TEST']['NAME']
or settings.DATABASES[destination]['NAME'],
)
ret = subprocess.call(commandline,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
if ret:
raise Exception(
'Exit code of the database copy command is %d' % ret)
# Update the scenario table
destinationscenario.status = 'In use'
destinationscenario.lastrefresh = datetime.today()
if 'description' in options:
destinationscenario.description = options['description']
destinationscenario.save(using=DEFAULT_DB_ALIAS)
# Give access to the destination scenario to:
# a) the user doing the copy
# b) all superusers from the source schema
User.objects.using(destination).filter(is_superuser=True).update(
is_active=True)
User.objects.using(destination).filter(is_superuser=False).update(
is_active=False)
if user:
User.objects.using(destination).filter(
username=user.username).update(is_active=True)
# Logging message
task.status = 'Done'
task.finished = datetime.now()
# Update the task in the destination database
task.message = "Scenario copied from %s" % source
task.save(using=destination)
task.message = "Scenario copied to %s" % destination
# Delete any waiting tasks in the new copy.
# This is needed for situations where the same source is copied to
# multiple destinations at the same moment.
Task.objects.all().using(destination).filter(
id__gt=task.id).delete()
except Exception as e:
if task:
task.status = 'Failed'
task.message = '%s' % e
task.finished = datetime.now()
if destinationscenario and destinationscenario.status == 'Busy':
destinationscenario.status = 'Free'
destinationscenario.save(using=DEFAULT_DB_ALIAS)
raise e
finally:
if task:
task.save(using=source)
settings.DEBUG = tmp_debug
def handle(self, *args, **options):
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
# Pick up options
if 'force' in options:
force = options['force']
else:
force = False
test = 'FREPPLE_TEST' in os.environ
if 'user' in options and options['user']:
try:
user = User.objects.all().get(username=options['user'])
except:
raise CommandError("User '%s' not found" % options['user'])
else:
user = None
# Initialize the task
now = datetime.now()
task = None
if 'task' in options and options['task']:
try:
task = Task.objects.all().get(pk=options['task'])
except:
raise CommandError("Task identifier not found")
if task.started or task.finished or task.status != "Waiting" or task.name != 'copy scenario':
raise CommandError("Invalid task identifier")
task.status = '0%'
task.started = now
else:
task = Task(name='copy scenario',
submitted=now,
started=now,
status='0%',
user=user)
task.save()
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Validate the arguments
destinationscenario = None
try:
if len(args) != 2:
raise CommandError("Command takes exactly 2 arguments.")
task.arguments = "%s %s" % (args[0], args[1])
task.save()
source = args[0]
try:
sourcescenario = Scenario.objects.get(pk=source)
except:
raise CommandError(
"No source database defined with name '%s'" % source)
destination = args[1]
try:
destinationscenario = Scenario.objects.get(pk=destination)
except:
raise CommandError(
"No destination database defined with name '%s'" %
destination)
if source == destination:
raise CommandError("Can't copy a schema on itself")
if settings.DATABASES[source]['ENGINE'] != settings.DATABASES[
destination]['ENGINE']:
raise CommandError(
"Source and destination scenarios have a different engine")
if sourcescenario.status != 'In use':
raise CommandError("Source scenario is not in use")
if destinationscenario.status != 'Free' and not force:
raise CommandError("Destination scenario is not free")
# Logging message - always logging in the default database
destinationscenario.status = 'Busy'
destinationscenario.save()
# Copying the data
# Commenting the next line is a little more secure, but requires you to create a .pgpass file.
if settings.DATABASES[source]['PASSWORD']:
os.environ['PGPASSWORD'] = settings.DATABASES[source][
'PASSWORD']
commandline = "pg_dump -c -Fp %s%s%s%s | psql %s%s%s%s" % (
settings.DATABASES[source]['USER'] and
("-U %s " % settings.DATABASES[source]['USER']) or '',
settings.DATABASES[source]['HOST'] and
("-h %s " % settings.DATABASES[source]['HOST']) or '',
settings.DATABASES[source]['PORT'] and
("-p %s " % settings.DATABASES[source]['PORT']) or '',
test and settings.DATABASES[source]['TEST']['NAME']
or settings.DATABASES[source]['NAME'],
settings.DATABASES[destination]['USER'] and
("-U %s " % settings.DATABASES[destination]['USER']) or '',
settings.DATABASES[destination]['HOST'] and
("-h %s " % settings.DATABASES[destination]['HOST']) or '',
settings.DATABASES[destination]['PORT'] and
("-p %s " % settings.DATABASES[destination]['PORT']) or '',
test and settings.DATABASES[destination]['TEST']['NAME']
or settings.DATABASES[destination]['NAME'],
)
ret = subprocess.call(commandline,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
if ret:
raise Exception(
'Exit code of the database copy command is %d' % ret)
# Update the scenario table
destinationscenario.status = 'In use'
destinationscenario.lastrefresh = datetime.today()
if 'description' in options:
destinationscenario.description = options['description']
else:
destinationscenario.description = "Copied from scenario '%s'" % source
destinationscenario.save()
# Give access to the destination scenario to:
# a) the user doing the copy
# b) all superusers from the source schema
User.objects.using(destination).filter(is_superuser=True).update(
is_active=True)
User.objects.using(destination).filter(is_superuser=False).update(
is_active=False)
if user:
User.objects.using(destination).filter(
username=user.username).update(is_active=True)
# Logging message
task.status = 'Done'
task.finished = datetime.now()
except Exception as e:
if task:
task.status = 'Failed'
task.message = '%s' % e
task.finished = datetime.now()
if destinationscenario and destinationscenario.status == 'Busy':
destinationscenario.status = 'Free'
destinationscenario.save()
raise e
finally:
if task:
task.save()
settings.DEBUG = tmp_debug
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program. If not, see <http://www.gnu.org/licenses/>.
#
r'''
This command is the wrapper for all administrative actions on frePPLe.
'''
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "freppledb.settings")
# Synchronize the scenario table with the settings
from freppledb.common.models import Scenario
Scenario.syncWithSettings()
# Run the command
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
def handle(self, **options):
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
# Pick up options
force = options["force"]
promote = options["promote"]
test = "FREPPLE_TEST" in os.environ
if options["user"]:
try:
user = User.objects.all().get(username=options["user"])
except Exception:
raise CommandError("User '%s' not found" % options["user"])
else:
user = None
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
# Initialize the task
source = options["source"]
try:
sourcescenario = Scenario.objects.using(DEFAULT_DB_ALIAS).get(pk=source)
except Exception:
raise CommandError("No source database defined with name '%s'" % source)
now = datetime.now()
task = None
if "task" in options and options["task"]:
try:
task = Task.objects.all().using(source).get(pk=options["task"])
except Exception:
raise CommandError("Task identifier not found")
if (
task.started
or task.finished
or task.status != "Waiting"
or task.name != "scenario_copy"
):
raise CommandError("Invalid task identifier")
task.status = "0%"
task.started = now
else:
task = Task(
name="scenario_copy", submitted=now, started=now, status="0%", user=user
)
task.processid = os.getpid()
task.save(using=source)
# Validate the arguments
destination = options["destination"]
destinationscenario = None
try:
task.arguments = "%s%s %s" % (
("--dumpfile=%s " % options["dumpfile"]) if options["dumpfile"] else "",
source,
destination,
)
if options["description"]:
task.arguments += '--description="%s"' % options["description"].replace(
'"', '\\"'
)
if force:
task.arguments += " --force"
task.save(using=source)
try:
destinationscenario = Scenario.objects.using(DEFAULT_DB_ALIAS).get(
pk=destination
)
except Exception:
raise CommandError(
"No destination database defined with name '%s'" % destination
)
if source == destination:
raise CommandError("Can't copy a schema on itself")
if sourcescenario.status != "In use":
raise CommandError("Source scenario is not in use")
if destinationscenario.status != "Free" and not force and not promote:
# make sure destination scenario is properly built otherwise it is considered Free
scenario_is_free = False
try:
User.objects.using(
destination
).all().count() # fails if scenario not properly built
except Exception:
scenario_is_free = True
if not scenario_is_free:
raise CommandError("Destination scenario is not free")
if promote and (
destination != DEFAULT_DB_ALIAS or source == DEFAULT_DB_ALIAS
):
raise CommandError(
"Incorrect source or destination database with promote flag"
)
# check that dump file exists
if options["dumpfile"] and not os.path.isfile(
os.path.join(settings.FREPPLE_LOGDIR, options["dumpfile"])
):
raise CommandError("Cannot find dump file %s" % options["dumpfile"])
# Logging message - always logging in the default database
destinationscenario.status = "Busy"
destinationscenario.save(using=DEFAULT_DB_ALIAS)
# tables excluded from promotion task
excludedTables = [
"common_user",
"common_scenario",
"auth_group",
"auth_group_permission",
"auth_permission",
"django_content_type",
"common_comment",
"common_notification",
"common_follower",
"common_user_groups",
"common_attribute",
"common_user_user_permissions",
"common_preferences",
"reportmanager_report",
"reportmanager_column",
"execute_schedule",
]
# Copying the data
# Commenting the next line is a little more secure, but requires you to create a .pgpass file.
if not options["dumpfile"]:
if settings.DATABASES[source]["PASSWORD"]:
os.environ["PGPASSWORD"] = settings.DATABASES[source]["PASSWORD"]
if os.name == "nt":
# On windows restoring with pg_restore over a pipe is broken :-(
cmd = "pg_dump -c -Fp %s%s%s%s%s | psql %s%s%s%s"
else:
cmd = "pg_dump -Fc %s%s%s%s%s | pg_restore -n public -Fc -c --if-exists %s%s%s -d %s"
commandline = cmd % (
settings.DATABASES[source]["USER"]
and ("-U %s " % settings.DATABASES[source]["USER"])
or "",
settings.DATABASES[source]["HOST"]
and ("-h %s " % settings.DATABASES[source]["HOST"])
or "",
settings.DATABASES[source]["PORT"]
and ("-p %s " % settings.DATABASES[source]["PORT"])
or "",
("%s " % (" -T ".join(["", *excludedTables])))
if destination == DEFAULT_DB_ALIAS
else "",
test
and settings.DATABASES[source]["TEST"]["NAME"]
or settings.DATABASES[source]["NAME"],
settings.DATABASES[destination]["USER"]
and ("-U %s " % settings.DATABASES[destination]["USER"])
or "",
settings.DATABASES[destination]["HOST"]
and ("-h %s " % settings.DATABASES[destination]["HOST"])
or "",
settings.DATABASES[destination]["PORT"]
and ("-p %s " % settings.DATABASES[destination]["PORT"])
or "",
test
and settings.DATABASES[destination]["TEST"]["NAME"]
or settings.DATABASES[destination]["NAME"],
)
else:
cmd = "pg_restore -n public -Fc -c --if-exists --no-password %s%s%s -d %s %s"
commandline = cmd % (
settings.DATABASES[destination]["USER"]
and ("-U %s " % settings.DATABASES[destination]["USER"])
or "",
settings.DATABASES[destination]["HOST"]
and ("-h %s " % settings.DATABASES[destination]["HOST"])
or "",
settings.DATABASES[destination]["PORT"]
and ("-p %s " % settings.DATABASES[destination]["PORT"])
or "",
test
and settings.DATABASES[destination]["TEST"]["NAME"]
or settings.DATABASES[destination]["NAME"],
os.path.join(settings.FREPPLE_LOGDIR, options["dumpfile"]),
)
with subprocess.Popen(
commandline,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
) as p:
try:
task.processid = p.pid
task.save(using=source)
p.wait()
# Successful copy can still leave warnings and errors
# To confirm copy is ok, let's check that the scenario copy task exists
# in the destination database
t = Task.objects.using(destination).filter(id=task.id).first()
if not t or t.name != task.name or t.submitted != task.submitted:
destinationscenario.status = "Free"
destinationscenario.lastrefresh = datetime.today()
destinationscenario.save(using=DEFAULT_DB_ALIAS)
raise Exception("Database copy failed")
t.status = "Done"
t.finished = datetime.now()
t.message = "Scenario copied from %s" % source
t.save(
using=destination,
update_fields=["status", "finished", "message"],
)
except Exception:
p.kill()
p.wait()
# Consider the destination database free again
if destination != DEFAULT_DB_ALIAS:
destinationscenario.status = "Free"
destinationscenario.lastrefresh = datetime.today()
destinationscenario.save(using=DEFAULT_DB_ALIAS)
raise Exception("Database copy failed")
# Check the permissions after restoring a backup.
if (
options["dumpfile"]
and task.user
and not User.objects.using(destination)
.filter(username=task.user.username, is_active=True)
.count()
):
# Restoring a backup shouldn't give a user access to data he didn't have access to before...
raise Exception(
"Permission denied - you did't have access rights to the scenario that was backed up"
)
# Update the scenario table
destinationscenario.status = "In use"
destinationscenario.lastrefresh = datetime.today()
if options["description"]:
destinationscenario.description = options["description"]
destinationscenario.save(using=DEFAULT_DB_ALIAS)
# Delete parameter that marks a running worker
if destination != DEFAULT_DB_ALIAS:
try:
Parameter.objects.using(destination).filter(
name="Worker alive"
).delete()
except BaseException:
pass
# Give access to the destination scenario to:
# a) the user doing the copy
# b) all active superusers from the source schema
# unless it's a promotion
if destination != DEFAULT_DB_ALIAS:
User.objects.using(destination).filter(
is_superuser=True, is_active=True
).update(is_active=True)
User.objects.using(destination).filter(is_superuser=False).update(
is_active=False
)
if user:
User.objects.using(destination).filter(
username=user.username
).update(is_active=True)
# Delete data files present in the scenario folders
if destination != DEFAULT_DB_ALIAS and settings.DATABASES[destination][
"FILEUPLOADFOLDER"
] not in (
settings.DATABASES[DEFAULT_DB_ALIAS]["FILEUPLOADFOLDER"],
settings.DATABASES[source]["FILEUPLOADFOLDER"],
):
FileManager.cleanFolder(0, destination)
FileManager.cleanFolder(1, destination)
# Logging message
task.processid = None
task.status = "Done"
task.finished = datetime.now()
# Update the task in the destination database
dest_task = Task(
name=task.name,
submitted=task.submitted,
started=task.started,
finished=task.finished,
arguments=task.arguments,
status="Done",
message=task.message,
user=user,
)
if options["dumpfile"]:
dest_task.message = "Scenario restored from %s" % options["dumpfile"]
elif promote:
dest_task.message = "Scenario promoted from %s" % source
else:
dest_task.message = "Scenario copied from %s" % source
dest_task.save(using=destination)
if options["dumpfile"]:
task.message = "Scenario %s restored from %s" % (
destination,
options["dumpfile"],
)
else:
task.message = "Scenario copied to %s" % destination
# Delete any waiting tasks in the new copy.
# This is needed for situations where the same source is copied to
# multiple destinations at the same moment.
if not options["dumpfile"]:
Task.objects.all().using(destination).filter(id__gt=task.id).delete()
# Don't automate any task in the new copy
if not promote:
for i in ScheduledTask.objects.all().using(destination):
i.next_run = None
i.data.pop("starttime", None)
i.data.pop("monday", None)
i.data.pop("tuesday", None)
i.data.pop("wednesday", None)
i.data.pop("thursday", None)
i.data.pop("friday", None)
i.data.pop("saturday", None)
i.data.pop("sunday", None)
i.save(using=destination)
if options["dumpfile"]:
setattr(_thread_locals, "database", destination)
call_command("migrate", database=destination)
delattr(_thread_locals, "database")
except Exception as e:
if task:
task.status = "Failed"
task.message = "%s" % e
task.finished = datetime.now()
if destinationscenario and destinationscenario.status == "Busy":
if destination == DEFAULT_DB_ALIAS:
destinationscenario.status = "In use"
else:
destinationscenario.status = "Free"
destinationscenario.save(using=DEFAULT_DB_ALIAS)
raise e
finally:
if task:
task.processid = None
task.save(using=source)
settings.DEBUG = tmp_debug
def getHTML(request):
# Synchronize the scenario table with the settings
Scenario.syncWithSettings()
scenarios = Scenario.objects.using(DEFAULT_DB_ALIAS)
if scenarios.count() > 1:
javascript = """
$(".scenariorelease").on("click", function(event) {
event.preventDefault();
var target = "/" + $(this).attr("data-target");
if (target == "/default")
target = "";
$.ajax({
url: target + "/execute/launch/scenario_copy/",
type: 'POST',
data: {release: 1},
complete: function() {
$("#scenariotoast").addClass("show");
$("#scenariotoast span").text("Scenario released");
setTimeout(function(){$("#scenariotoast").removeClass("show") }, 3000);
if (target == url_prefix) window.location.href = "/"; }
});
});
$(".scenariocopy").on("click", function(event) {
event.preventDefault();
var source = "/" + $(this).attr("data-source");
if (source == "/default")
source = "";
$.ajax({
url: source + "/execute/launch/scenario_copy/",
type: 'POST',
data: {
copy: 1,
source: $(this).attr("data-source"),
destination: $(this).attr("data-target")
},
success: function() {
$("#scenariotoast").addClass("show");
$("#scenariotoast span").text("Launched copy task");
setTimeout(function(){$("#scenariotoast").removeClass("show") }, 3000);
}});
});
$(".scenariolabel").on("change", function(event) {
event.preventDefault();
var target = "/" + $(this).attr("data-target");
if (target == "/default")
target = "";
$.ajax({
url: target + "/execute/launch/scenario_copy/",
type: 'POST',
data: {
update: 1,
description: $(this).val()
},
success: function() {
$("#scenariotoast").addClass("show");
$("#scenariotoast span").text("Updated description");
setTimeout(function(){
$("#scenariotoast").removeClass("show");
}, 3000);
}
});
});
"""
context = RequestContext(
request, {"javascript": javascript, "scenarios": scenarios}
)
template = Template(
"""
{% load i18n %}
<div id="scenariotoast" class="toast"><div style="position: relative; left: -50%">
<h1><span class="btn btn-primary"></span></h1>
</div></div>
<table id="scenarios">
<tr>
{% comment %}Translators: Translation included with Django {% endcomment %}
<th style="padding:5px 10px 5px 10px; text-align: center">{% trans 'scenario'|capfirst %}</th>
<th style="padding:5px 10px 5px 10px; text-align: center">{% trans 'action'|capfirst %}</th>
<th style="padding:5px 10px 5px 10px; text-align: center">
<span data-toggle="tooltip" data-placement="top" data-html="true"
data-original-title="<b>In use</b>: Contains data<br><b>Free</b>: Available to copy data into<br><b>Busy</b>: Data copy in progress">
{% trans 'status'|capfirst %}
<span class="fa fa-question-circle"></span>
</span>
</th>
<th style="padding:5px 10px 5px 10px; text-align: center">
<span data-toggle="tooltip" data-placement="top" data-original-title="Label shown in the scenario dropdown list">
{% trans 'label'|capfirst %}
<span class="fa fa-question-circle"></span>
</span></th>
<th style="padding:5px 10px 5px 10px; text-align: center">
<span data-toggle="tooltip" data-placement="top" data-original-title="Date of the last action">
{% trans 'last modified'|capfirst %}
<span class="fa fa-question-circle"></span>
</span>
</th>
</tr>
{% for j in scenarios %}
<tr>
<td style="padding:5px">
<strong>{{j.name|capfirst}}</strong>
</td>
<td style="padding:5px 10px 5px 10px">
{% if j.name != 'default' and j.status == 'Free' and perms.common.copy_scenario %}
<div class="btn-group btn-block">
<button class="btn btn-block btn-primary dropdown-toggle" type="button" data-toggle="dropdown">
{% trans 'copy'|capfirst %} <span class="caret"></span>
</button>
<ul class="dropdown-menu" rol="menu">
{% for k in scenarios %}{% if k.status == 'In use' %}
<li><a class="scenariocopy" href="#" data-source="{{ k.name }}" data-target="{{ j.name }}">
Copy from {{ k.name|capfirst }}
</a></li>
{% endif %}{% endfor %}
</ul>
</div>
{% elif j.name != 'default' and j.status == 'In use' and perms.common.release_scenario %}
<div class="btn-group btn-block">
<button class="btn btn-block btn-primary dropdown-toggle" type="button" data-toggle="dropdown">
{% trans 'release'|capfirst %} <span class="caret"></span>
</button>
<ul class="dropdown-menu" rol="menu">
<li><a class="scenariorelease" href="#" data-target="{{ j.name }}">
{% trans "You will lose ALL data in this scenario!" %}
</a></li>
</ul>
</div>
{% endif %}
</td>
{% with mystatus=j.status|lower %}
<td style="padding:5px 10px 5px 10px; text-align: center">{% trans mystatus|capfirst %}</td>
{% endwith %}
<td style="padding:5px 10px 5px 10px">
<input class="scenariolabel" type="text" size="20" data-target="{{ j.name }}"
value="{% if j.description %}{{j.description|escape}}{% else %}{{ j.name }}{% endif %}">
</td>
<td style="padding:5px 10px 5px 10px; text-align: center">{{j.lastrefresh|date:"DATETIME_FORMAT"}}</td>
</tr>
{% endfor %}
</table>
<script>{{ javascript|safe }}</script>
"""
)
return template.render(context)
# A list of translation strings from the above
translated = (
_("copy"),
_("release"),
_("release selected scenarios"),
_("into selected scenarios"),
_("update"),
_("Update description of selected scenarios"),
_("You will lose ALL data in this scenario!"),
)
else:
return None
def __call__(self, request):
# Make request information available throughout the application
setattr(_thread_locals, "request", request)
if not hasattr(request, "user"):
request.user = auth.get_user(request)
# Log out automatically after inactivity
if (
not request.user.is_anonymous
and settings.SESSION_LOGOUT_IDLE_TIME
and "freppledb.common.middleware.AutoLoginAsAdminUser"
not in settings.MIDDLEWARE
):
now = datetime.now()
if "last_request" in request.session:
idle_time = now - datetime.strptime(
request.session["last_request"], "%y-%m-%d %H:%M:%S"
)
if idle_time > timedelta(minutes=settings.SESSION_LOGOUT_IDLE_TIME):
logout(request)
info(
request,
_("Your session has expired. Please login again to continue."),
)
if request.headers.get("x-requested-with") == "XMLHttpRequest":
return HttpResponse(
status=401,
content=_(
"Your session has expired. Please login again to continue."
),
)
else:
return HttpResponseRedirect(request.path_info)
else:
request.session["last_request"] = now.strftime("%y-%m-%d %H:%M:%S")
else:
request.session["last_request"] = now.strftime("%y-%m-%d %H:%M:%S")
# Keep last_login date up to date
if not request.user.is_anonymous:
last_login = getattr(request.user, "last_login", None)
now = timezone.now()
if not last_login or now - last_login > timedelta(hours=1):
user_logged_in.send(
sender=request.user.__class__, request=request, user=request.user
)
if not hasattr(request.user, "scenarios"):
# A scenario list is not available on the request
for i in settings.DATABASES:
try:
if settings.DATABASES[i]["regexp"].match(request.path):
scenario = Scenario.objects.using(DEFAULT_DB_ALIAS).get(name=i)
if scenario.status != "In use":
return HttpResponseNotFound("Scenario not in use")
request.prefix = "/%s" % i
request.path_info = request.path_info[len(request.prefix) :]
request.path = request.path[len(request.prefix) :]
request.database = i
if hasattr(request.user, "_state"):
request.user._state.db = i.name
response = self.get_response(request)
if not response.streaming:
# Note: Streaming response get the request field cleared in the
# request_finished signal handler
setattr(_thread_locals, "request", None)
return response
except Exception:
pass
request.prefix = ""
request.database = DEFAULT_DB_ALIAS
if hasattr(request.user, "_state"):
request.user._state.db = DEFAULT_DB_ALIAS
else:
# A list of scenarios is already available
if request.user.is_anonymous:
return self.get_response(request)
default_scenario = None
for i in request.user.scenarios:
if i.name == DEFAULT_DB_ALIAS:
default_scenario = i
try:
if settings.DATABASES[i.name]["regexp"].match(request.path):
request.prefix = "/%s" % i.name
request.path_info = request.path_info[len(request.prefix) :]
request.path = request.path[len(request.prefix) :]
request.database = i.name
request.scenario = i
if hasattr(request.user, "_state"):
request.user._state.db = i.name
request.user.is_superuser = i.is_superuser
request.user.horizonlength = i.horizonlength
request.user.horizontype = i.horizontype
request.user.horizonbefore = i.horizonbefore
request.user.horizonbuckets = i.horizonbuckets
request.user.horizonstart = i.horizonstart
request.user.horizonend = i.horizonend
request.user.horizonunit = i.horizonunit
response = self.get_response(request)
if not response.streaming:
# Note: Streaming response get the request field cleared in the
# request_finished signal handler
setattr(_thread_locals, "request", None)
return response
except Exception:
pass
request.prefix = ""
request.database = DEFAULT_DB_ALIAS
if hasattr(request.user, "_state"):
request.user._state.db = DEFAULT_DB_ALIAS
if default_scenario:
request.scenario = default_scenario
else:
request.scenario = Scenario(name=DEFAULT_DB_ALIAS)
response = self.get_response(request)
if not response.streaming:
# Note: Streaming response get the request field cleared in the
# request_finished signal handler
setattr(_thread_locals, "request", None)
return response
