def get_files_to_create():
"""
Treat the queries in the waiting queue.
Get the queries from the database and put them into the waiting list.
If a the waiting list is empty or the worker is already run, do nothing, else, launch the worker.
"""
redis_server = Redis.from_url(REDIS_URL)
queries_to_treat = query_to_treat_api.get_all()
try:
for query_to_treat in queries_to_treat:
Redis.rpush(redis_server, 'list_ids', str(query_to_treat.query.id))
query_to_treat.delete_database()
if not Redis.exists(redis_server, 'is_working'):
Redis.set(redis_server, 'is_working', False)
if Redis.get(redis_server, 'is_working') == 'False':
Redis.set(redis_server, 'is_working', True)
if Redis.exists(redis_server, 'list_ids'):
run_worker(redis_server)
Redis.set(redis_server, 'is_working', False)
except ConnectionError, e:
log_file = LogFile(application="Custom Queries",
message="Redis not reachable, is it running?",
additionalInformation={'message': e.message},
timestamp=datetime.now())
log_file_api.upsert(log_file)
def run_worker(redis_server):
"""
Worker algorithm.
While the waiting list is not empty, treat the query in order.
:param redis_server: Redis server instance.
"""
try:
while Redis.exists(redis_server, 'list_ids'):
current_id = Redis.lpop(redis_server, 'list_ids').decode()
redis_server.set('current_id', current_id)
try:
query = temp_user_query_api.get_by_id(current_id)
query.create_outputs_file()
except DoesNotExist as e:
logger.warning("run_worker threw an exception: {0}".format(
str(e)))
redis_server.set('current_id', "None")
except ConnectionError as e:
log_file = LogFile(application="Custom Queries",
message="Redis not reachable, is it running?",
additionalInformation={'message': str(e)},
timestamp=datetime.now())
log_file_api.upsert(log_file)
def create_outputs_file(self):
""" Create the output files
"""
try:
self.init_data_result()
except Exception as e:
current_step = self.get_current_step()
log_file = LogFile(application="Custom Queries",
message="The step #" +
str(current_step.absolute_position) + ": " +
str(current_step.step.name) +
" is not a valid timestamp." + str(e),
additionalInformation={
'query_name': self.query_name,
'Status': "Result builder"
},
timestamp=datetime.now())
log_file_api.upsert(log_file)
self.update_message("Error during result files creation.")
return
self.update_message("Result files created.")
# Get the user to send him an email
try:
history = HistoryQuery.objects.get(query_id=str(self.id))
user = User.objects.get(id=history.user_id)
send_mail_query_end(query=self, user=user)
except Exception as e:
logger.warning(
"create_outputs_file threw an exception: {0}".format(str(e)))
self.update_status(2)
def create_outputs_file(self):
""" Create the output files
"""
try:
self.init_data_result()
except Exception, e:
current_step = self.get_current_step()
log_file = LogFile(application="Custom Queries",
message="The step #"
+ str(current_step.absolute_position)
+ ": " + str(current_step.step.name)
+ " is not a valid timestamp."
+ str(e.message),
additionalInformation={'query_name': self.query_name,
'Status': "Result builder"},
timestamp=datetime.now())
log_file_api.upsert(log_file)
self.update_message("Error during result files creation.")
return
def recover_query_steps(request, history_id):
"""
"""
user_step = None
previous_step = False
next_step = False
history_step = False
# Validate the user step
if request.method == 'POST':
# Get the form's value
user_query_id = request.POST.get('user_query', None)
step_back = request.POST.get('step_back', None)
try:
# Load the query
user_query = temp_user_query_api.get_by_id(user_query_id)
except DoesNotExist:
# The query doesn't exist anymore.
messages.add_message(
request, messages.ERROR,
"A problem occurred during your information's transportation."
" Please do it again.")
return redirect(reverse("core_custom_queries_app_index"))
# Get the current step information
user_step = user_query.get_current_step()
form_type = user_step.form_type
# If the user want to go back to the previous step
if step_back == 'True':
previous_step = True
else:
error = False
# The step is a a single choice step
if form_type == 'radio':
choices = request.POST.get('choices', None)
if choices:
user_step.choices.append(choices)
else:
messages.add_message(request, messages.WARNING,
'You have to choose one.')
error = True
# The step is a a multiple choices step
elif form_type == 'check':
list_choices = request.POST.getlist('choices')
# Get the step restrictions and validate the choices against the restrictions
min_range = user_step.step.data_min_range
if min_range != '' and min_range:
min_range = int(min_range)
max_range = user_step.step.data_max_range
# data_infinity = user_step.step.data_infinity
if max_range != '' and max_range:
max_range = int(max_range)
if max_range < len(list_choices) or len(
list_choices) < min_range:
error = True
messages.add_message(
request, messages.WARNING,
'You have to choose between ' +
str(min_range) + ' and ' + str(max_range) +
' choice.')
else:
if len(list_choices) < min_range:
error = True
messages.add_message(
request, messages.WARNING,
'You have to choose at least one.')
if not error:
user_step.choices = list_choices
# The step is a a date step
else:
time_from = request.POST['timeFrom']
time_to = request.POST['timeTo']
if time_from == "" or time_to == "":
error = True
messages.add_message(request, messages.WARNING,
'You have to fill both date.')
else:
try:
time_from = datetime.strptime(time_from,
"%Y-%m-%d %H:%M")
time_to = datetime.strptime(time_to, "%Y-%m-%d %H:%M")
if time_from > time_to:
error = True
messages.add_message(
request, messages.WARNING,
'Date From has to be before date To')
date_range = user_step.step.date_range
if date_range and date_range != '':
date_range = int(date_range)
if (time_to - time_from).days > date_range:
error = True
messages.add_message(
request, messages.WARNING,
'The maximum time range you can query in is '
+ str(date_range) + ' day(s).')
except Exception as e:
error = True
messages.add_message(request, messages.ERROR, e)
if not error:
user_step.choices = (request.POST['timeFrom'],
request.POST['timeTo'])
if not error:
user_step.update_choices_to_db()
next_step = True
# Select and load the current step
else:
# Load history queries
try:
history_query = history_query_api.get_by_id(history_id)
user_query = temp_user_query_api.get_by_id(history_query.query_id)
except DoesNotExist:
messages.add_message(request, messages.ERROR,
"A problem occurred during the redirection.")
return redirect(reverse("core_custom_queries_app_index"))
history_step = True
try: # Load the current step
if previous_step is True:
# Load the previous step
user_step = user_query.get_previous_query_able_step()
user_query.update_current_position_into_the_db()
user_step.transform_choices_files_list_to_dict()
history_id = user_query.save_to_history(user_id=request.user.id,
history_id=history_id)
elif next_step is True:
# Load the next step
user_step = user_query.get_and_load_choices_next_step()
user_query.update_current_position_into_the_db()
history_id = user_query.save_to_history(user_id=request.user.id,
history_id=history_id)
elif history_step is True:
if user_query.history.message == "Error during output file creation.":
# An error occurred
messages.add_message(
request, messages.ERROR,
"An internal problem occurred during the output file creation. "
"The administrator has been noticed.")
return redirect(reverse("core_custom_queries_app_index"))
else:
# Load the history query
user_step = user_query.handle_history()
user_step.transform_choices_files_list_to_dict()
except Exception as e:
# Create the log file from the error
log_file = LogFile(
application="Custom Queries",
message=str(e),
additionalInformation={
'query_name':
user_query.query.name,
'step_name':
user_query.list_steps[user_query.current_position].step.name,
'user_choices':
user_query.get_previous_choices()
},
timestamp=datetime.now())
log_file_api.upsert(log_file)
try:
user_query.update_message("Error during query step.")
except ValidationError as e:
logger.warning(
"recover_query_steps threw an exception: {0}".format(str(e)))
messages.add_message(
request, messages.ERROR,
"An internal problem occurred, the administrator has been notified."
)
return redirect(reverse("core_custom_queries_app_index"))
if user_step is None:
# Go to result page
user_query.add_query_to_waiting_line()
user_query.update_message("Pending.")
user_query.update_status(1)
messages.add_message(request, messages.SUCCESS,
"The output file will be created soon.")
return redirect(reverse("core_custom_queries_app_index"))
else:
context = _create_step_context(user_query, user_step, history_id)
assets = {
"js": [
{
"path":
'core_custom_queries_app/user/js/custom_queries.js',
"is_raw": False
},
{
"path":
'core_custom_queries_app/user/js/custom_queries_step.js',
"is_raw": False
},
{
"path":
"core_custom_queries_app/libs/bootstrap-datetimepicker/js/bootstrap-datetimepicker.js",
"is_raw": False
},
],
'css': [
'core_custom_queries_app/libs/bootstrap-datetimepicker/css/bootstrap-datetimepicker.min.css'
]
}
# for the rendering
return render(request,
'core_custom_queries_app/user/query_step.html',
assets=assets,
context=context)
def choose_query(request):
"""
Select a query from the one into the database to be able to use it.
Used by the user.
If there is no query into the database, an error message is shown to the user.
If no query has been selected, an error message is shown to the user.
Send to the query_step if a query has been selected.
If history queries are available, they are show with the following option:
_History deletion
_History recovery
Args:
request:
Returns:
"""
formset = None
formset_history = None
history_queries = None
# Come back from the form with the query selected
if request.method == 'POST':
# Get the forms variable
query_id = request.POST.get('query', None)
to_recover = request.POST.get('to_recover', None)
number_forms = request.POST.get('form-TOTAL_FORMS', None)
# if a variable is none, come back with an error message
if query_id is None and to_recover is None and number_forms is None:
messages.add_message(request, messages.INFO,
"You have to select a query.")
else:
if query_id is not None:
return redirect(
reverse("core_custom_queries_app_steps",
kwargs={'query_id': query_id}))
# History selected
elif to_recover is not None and to_recover != '':
return redirect(
reverse("core_custom_queries_app_recover_steps",
kwargs={'history_id': to_recover}))
# Delete history query
else:
number_forms = int(number_forms)
formset_history = formset_factory(FormHistory,
extra=number_forms,
can_delete=True)
formset = formset_history(request.POST, request.FILES)
if formset.is_valid():
if len(formset.deleted_forms) == 0:
messages.add_message(request, messages.ERROR,
"Please select a query first.")
for form in formset.deleted_forms:
data = form.cleaned_data
if 'DELETE' in data:
id_history = ObjectId(data['id_history'])
history = history_query_api.get_by_id(id_history)
temp_query_id = history.query_id
query = temp_user_query_api.get_by_id(
temp_query_id)
query.delete_database()
messages.add_message(request, messages.SUCCESS,
"Queries successfully deleted.")
# Create the query form from the ones into the database
id_list = list()
# Get the available templates
available_template_version_managers = template_version_manager_api.get_active_global_version_manager(
)
for available_template_version_manager in available_template_version_managers:
for available_template_id in available_template_version_manager.versions:
id_list.append(str(available_template_id))
# Get the queries available for the current user
if request.user.is_staff:
# FIXME: call specialized api
queries = dyn_query_api.get_all().filter(
schema__in=id_list).values_list('pk', 'name')
else:
# FIXME: call specialized api
queries = dyn_query_api.get_all().filter(schema__in=id_list).filter(
group='user').values_list('pk', 'name')
number_queries = len(queries)
form = None
if number_queries != 0:
form = FormChooseQuery(query=queries)
else:
# If no query available, set an error message
message = "No query available."
messages.add_message(request, messages.ERROR, message)
try:
# If histories exist load them
history_queries = history_query_api.get_all_by_user_id(
str(request.user.id))
number_history = len(history_queries)
except Exception:
number_history = 0
if number_history > 0:
# Create the formset with the histories values
formset_history = formset_factory(FormHistory,
extra=number_history,
can_delete=True)
data = dict()
data['form-TOTAL_FORMS'] = number_history
data['form-INITIAL_FORMS'] = number_history
data['form-MAX_NUM_FORMS'] = number_history
position = 0
for history_query in history_queries:
try:
query_id = str(history_query.query_id)
data['form-' + str(position) + '-id_history'] = str(
history_query.id)
query = temp_user_query_api.get_by_id(query_id)
data['form-' + str(position) +
'-query_name'] = query.query.name
data['form-' + str(position) +
'-query_last_modified'] = query.last_modified
h_message = history_query.message
if h_message == "Pending.":
redis_server = Redis.from_url(REDIS_URL)
try:
if redis_server.exists("current_id"):
current_id = redis_server.get(
"current_id").decode()
if current_id == query_id:
h_message = "Pending: The file creation will begin shortly."
if redis_server.exists("list_ids"):
list_id = redis_server.lrange('list_ids', 0, -1)
try:
position_id = list_id.index(query_id) + 1
h_message = "Pending - Waiting list: " + str(
position_id) + "/" + str(len(list_id))
except ValueError as e:
logger.warning(
"choose_query threw an exception: {0}".
format(str(e)))
except ConnectionError as e:
log_file = LogFile(
application="Custom Queries",
message="Redis not reachable, is it running?",
additionalInformation={'message': str(e)},
timestamp=datetime.now())
log_file_api.upsert(log_file)
data['form-' + str(position) + '-query_message'] = h_message
data['form-' + str(position) +
'-status'] = history_query.status
data['form-' + str(position) + '-DELETE'] = ''
position += 1
except DoesNotExist as e:
logger.warning("choose_query threw an exception: {0}".format(
str(e)))
formset = formset_history(data)
context = {
'number_queries': number_queries,
'number_history': number_history,
'form': form,
'formset': formset,
'formset_history': formset_history
}
assets = {
"js": [
{
"path": 'core_custom_queries_app/user/js/custom_queries.js',
"is_raw": False
},
],
}
return render(request,
'core_custom_queries_app/user/select_query.html',
assets=assets,
context=context)
# Load the next step
user_step = user_query.get_and_load_choices_next_step()
user_query.update_current_position_into_the_db()
else:
# Error validation
user_step.transform_choices_files_list_to_dict()
history_id = user_query.save_to_history(user_id=str(request.user.id),
history_id=history_id)
except Exception, e:
# Create the log file from the error
log_file = LogFile(
application="Custom Queries",
message=e.message,
additionalInformation={
'query_name':
user_query.query.name,
'step_name':
user_query.list_steps[user_query.current_position].step.name,
'user_choices':
user_query.get_previous_choices()
},
timestamp=datetime.now())
log_file_api.upsert(log_file)
try:
user_query.update_message("Error during query step.")
except ValidationError:
pass
messages.add_message(
request, messages.ERROR,
"An internal problem occurred, the administrator has been notified."
)