def receive(self, text_data=None, bytes_data=None):
try:
if text_data == "RUN_THE_MODEL":
from data_handler.models import (DataFile, DataHandlerSession)
member_data_file = DataFile.objects.get(member=self.user)
data_session = DataHandlerSession.objects.get(
data_handler_id=member_data_file)
donation_cols = data_session.donation_columns
# run_model(data_session.data_file_path, donation_cols)
# run_model_data = run_model(data_session.base_data_file_path, donation_cols, self)
run_model_data = run_model(data_session.data_file_path,
donation_cols, self)
if run_model_data:
cprint("Run model completed!", 'green')
data_session.pdf_report_file_path = run_model_data.get(
'PDF_FILE')
data_session.csv_report_file_path = run_model_data.get(
'CSV_FILE')
data_session.is_process_complete = True
data_session.save()
member_data_file.is_run_the_model = True
member_data_file.save()
cprint("save is run the model", 'yellow')
cprint('save to db done', 'yellow')
# self.send(text_data="Run complete", close=True)
self.send("Complete Successfully!")
except Exception as ex:
cprint(traceback.format_exc(), 'red')
cprint(str(ex), 'red')
log_exception(ex)
def post(self, request):
try:
# ^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[a-zA-Z]).{8,}$
import re
member = Member.objects.get(email=request.user.email)
# 'password', 'new-password', 'verify-new-password'
if request.POST.get('password') == '':
messages.error(request, 'Password is empty!!')
elif request.POST.get("new-password") == "":
messages.error(request, 'New Password is empty!!')
elif request.POST.get("verify-new-password") == "":
messages.error(request, 'You have to verify new password is empty!!')
elif request.POST.get("verify-new-password") != request.POST.get("new-password"):
messages.error(request, 'Password not verified or matched!!')
else:
if member.check_password(request.POST.get("password")) is True:
pattern = re.compile(r"^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[a-zA-Z]).{8,}$")
if pattern.match(request.POST.get('new-password')):
member.set_password(request.POST.get("new-password"))
member.save()
update_session_auth_hash(request, member)
messages.success(request, 'Your password has been updated!')
else:
messages.error(request, 'Your password not match password requirements!')
else:
messages.error(request, 'Your old password is not correct!')
return redirect(reverse('profile-change-password'))
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
messages.error(request, 'There is errors!, try again latter')
def post(self, request):
try:
member = Member.objects.get(email=request.user.email)
# ['csrfmiddlewaretoken', 'org_name', 'org_website', 'organizationType', 'annualRevenue', 'job_title', 'total_staff', 'num_of_volunteer']
member.org_name = request.POST.get("org_name").strip()
member.org_website = request.POST.get("org_website").strip()
if request.POST.get("org_type") != "Other":
member.org_type = request.POST.get("org_type").strip()
else:
member.org_type = request.POST.get("other-org-type").strip()
member.annual_revenue = request.POST.get("annualRevenue").strip()
member.job_title = request.POST.get("job_title").strip()
member.total_staff = request.POST.get("total_staff").strip()
member.num_of_volunteer = request.POST.get(
"num_of_volunteer").strip()
member.num_of_board_members = request.POST.get(
"num_of_board_members").strip()
member.save()
messages.success(request,
'your info have been updated successfully!')
return redirect(reverse('profile-info'))
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
messages.error(request, 'There is errors!, try again latter')
def post(self, request, format=None):
try:
post_data = request.POST
# cprint(request.POST, "blue")
filter_array = json.loads(request.POST.get("filtersArray"))
filter_report_section = request.POST.get("reportSectionName")
# cprint(filter_array, "yellow")
# cprint(filter_report_section, "red")
reports_headers = ReportGenerator.generate_reports_table_header(
filter_array)
# cprint(reports_headers, "green")
reports = ReportGenerator.generate_report(filter_report_section,
filter_array,
reports_headers,
request.user.pk)
# ReportGenerator.generate_reports(filter_array)
# cprint(reports.keys(), 'green')
# cprint(simple_json.dumps(reports.get("data")), 'green')
# cprint(json.dumps(reports.get("data")), 'green')
# cprint(type(reports.get("data")), 'cyan')
# cprint(reports, 'blue')
# cprint(reports_headers, 'red')
# cprint(reports.get("data"), 'blue')
return Response(
{
"table_header": reports_headers,
"report_data": reports.get('data'),
"report_section_name": filter_report_section
},
status=200)
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def delete_all_member_data_file_info(member_data_file):
"""
this function will take DataFile object, to reset all member data file
Args:
member_data_file:
Returns:
"""
try:
from data_handler.models import DataHandlerSession
for dfile in DataHandlerSession.objects.filter(
data_handler_id=member_data_file):
delete_data_file(dfile.data_file_path)
DataHandlerSession.objects.filter(
data_handler_id=member_data_file).delete()
# member_data_session = DataHandlerSession.objects.get(data_handler_id=member_data_file)
# member_data_file.data_file_path = "None"
# member_data_file.file_upload_procedure = "None"
# member_data_file.all_records_count = 0
# member_data_file.selected_columns = ""
# member_data_file.selected_columns_dtypes = ""
# member_data_file.donor_id_column = ""
# member_data_file.is_donor_id_selected = False
# member_data_file.unique_id_column = ""
# member_data_file.all_columns_with_dtypes = ""
# member_data_file.is_process_complete = False
# member_data_file.save()
except DataHandlerSession.DoesNotExist:
cprint('DataHandlerSession.DoesNotExist', 'red')
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def get(self, request):
try:
member = Member.objects.get(email=request.user.email)
return render(request, "members_app/profile/change-password.html", context={"member": member})
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def post(self, request, format=None):
try:
post_data = request.POST
reports_section_name = request.POST.get("reports_section_name")
all_filter_cookies = request.POST.get("all_filter_cookies")
all_filter_cookies = json.loads(all_filter_cookies)
displayed_columns = request.POST.get("displayed_columns")
displayed_columns = json.loads(displayed_columns)
cprint(reports_section_name, 'red')
cprint(all_filter_cookies, 'yellow')
cprint(len(all_filter_cookies), 'green')
cprint(displayed_columns, 'blue')
report_obj = ReportGenerator(reports_section_name,
displayed_columns, all_filter_cookies)
cprint(report_obj, 'magenta')
return Response(
{
"table_header": report_obj.get_displayed_columns(),
'report_rows': report_obj.get_rows_file()
},
status=200)
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def get(self, request):
try:
member = Member.objects.get(email=request.user.email)
return render(request, "members_app/profile/information.html",
context={"member": member, 'annual_revenue': ANNUAL_REVENUE, 'org_types': ORGANIZATION_TYPES})
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def disconnect(self, close_code):
try:
self.disconnect(close_code)
self.close()
cprint("Close the connection", 'green')
except Exception as ex:
cprint(traceback.format_exc(), 'red')
cprint(str(ex), 'red')
log_exception(ex)
def check_empty_df(file_path):
try:
df = get_df_from_data_file(file_path)
if df.empty is True:
return True
return False
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def get_selected_columns_as_list(self):
# return self.selected_columns.split("|")
# return sorted(self.selected_columns.split("|"))
try:
return self.selected_columns.split("|")
except AttributeError as aex:
pass
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def validate_empty(self, value):
try:
tmp_val = value
if tmp_val == "" or tmp_val is None or tmp_val == "nan" or tmp_val == "Nan":
return True
else:
return False
except Exception as ex:
cprint(traceback.format_exc(), 'red')
cprint(str(ex), 'red')
log_exception(traceback.format_exc())
def update_rows_data(file_path, data_json, column_names, columns_with_dtypes):
# pd.describe_option("display.float_format")
# pd.set_option("display.float_format", "{:.2f}".format)
try:
data_file = Path(file_path)
all_rows = []
rows_and_values = {}
df = get_df_from_data_file(file_path)
for key, value in data_json.items():
# ROW_0 [{'colName': 'Cand_Name', 'colValue': '858f'}]
rows_and_values[key.split('_')[1]] = value
# df2 = copy.deepcopy(df[column_names])
# df2 = copy.deepcopy(df)
df2 = df.copy()
current_value = ""
for key, value in rows_and_values.items():
# {"0": [{"colName", "colValue"}, {"colName", "colValue"}]
# 0 [{'colName': 'Cand_Name', 'colValue': '858fx'}]
# print(key, value)
for val in value:
# cprint(df2[val['colName']].dtype, 'yellow')
current_value = val['colValue']
if df2[val['colName']].dtype == 'int64':
if current_value.isdigit():
df2.at[int(key), val['colName']] = current_value
else:
df2[val['colName']] = df2[val['colName']].astype(str)
df2.at[int(key), val['colName']] = current_value
# df2[val['colName']] = df2[val['colName']].astype(int)
# df2[val['colName']] = pd.to_numeric(df2[val['colName']], errors='ignore', downcast='float')
# df2[val['colName']] = pd.to_numeric(df2[val['colName']])
# df2[val['colName']] = df2[val['colName']].astype('int64')
else:
df2.at[int(key), val['colName']] = current_value
# cprint(df2[val['colName']].dtype, 'blue')
# save all changes to the file
if data_file.suffix == ".xlsx":
df2.to_excel(data_file.as_posix(), header=True, index=False)
elif data_file.suffix == ".csv":
df2.to_csv(data_file.as_posix(), header=True, index=False)
# cprint(df2.dtypes, 'magenta')
# return "Data saved successfully"
return current_value, "Data saved successfully"
except Exception as ex:
cprint(str(ex), 'red')
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def clean_currency(x: str):
""" If the value is a string, then remove currency symbol and delimiters
otherwise, the value is numeric and can be converted
"""
try:
# x = str(x)
if isinstance(x, str):
if x.startswith("$"):
return x.replace('$', '').replace(',', '')
# return float(x)
return x
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def get_df_from_data_file(file_path):
try:
from data_handler.models import DataHandlerSession
data_file = Path(file_path)
file_object = DataHandlerSession.objects.filter(
data_file_path=data_file.as_posix()).first()
df = None
# cprint(file_object.get_all_data_file_columns, 'yellow')
if file_object:
if data_file.exists():
if data_file.suffix == ".xlsx":
df = pd.read_excel(data_file.as_posix())
elif data_file.suffix == ".csv":
df = pd.read_csv(data_file.as_posix(),
sep=',',
skipinitialspace=True)
else:
if data_file.exists():
if data_file.suffix == ".xlsx":
df = pd.read_excel(data_file.as_posix())
elif data_file.suffix == ".csv":
df = pd.read_csv(data_file.as_posix(), sep=',')
# cprint(df.dtypes, 'red')
# this for fill the empty cells with its own empty values
float_cols = df.select_dtypes(include=['float64']).columns
str_cols = df.select_dtypes(include=['object']).columns
int_cols = df.select_dtypes(include=['int64']).columns
df.loc[:, float_cols] = df.loc[:, float_cols].fillna(0)
df.loc[:, int_cols] = df.loc[:, int_cols].fillna(0)
df.loc[:, str_cols] = df.loc[:, str_cols].fillna('NULL')
df_clone = df.copy()
# this loop to convert bool dtype to string
for co in df_clone.columns.tolist():
if df_clone[co].dtype == 'bool':
# cprint(df_clone[co].dtype, 'blue')
df_clone[co] = df_clone[co].apply(str)
# cprint(df_clone[co].dtype, 'green')
elif df_clone[co].dtype == 'float64':
df_clone[co] = df_clone[co].round().astype(int)
# cprint(df_clone.dtypes, 'green')
return df_clone
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def send_data(obj):
try:
obj.send(text_data=f"member_data_file")
time.sleep(3)
obj.send(text_data='Now sleep after sleep 3')
time.sleep(2)
obj.send(text_data=f"data_session")
time.sleep(2)
obj.send(text_data='Request complete', close=True)
except Exception as ex:
cprint(traceback.format_exc(), 'red')
cprint(str(ex), 'red')
obj.send(
text_data=str(ex)) # to display the error in the session modal
log_exception(ex)
def save_data_file_rounded(file_path):
data_file = Path(file_path)
df = get_df_from_data_file(file_path)
df_copy = df.copy()
saved_logged_msg = '' # the info log will save, contains columns name, columns dtypes
saved_logged_cols_base = [
] # the columns with dtype log will save, contains columns name, columns dtypes
saved_logged_cols_after = [] # the columns with converted dtype
new_cleand_cols = [
] # this list all hold all columns without any spaces or whitespaces
# Volunteered in the past
# cprint(df.dtypes, 'green')
try:
for col in df_copy.columns.tolist():
new_cleand_cols.append(col.strip())
saved_logged_cols_base.append(f"{col}: {df_copy[col].dtype}")
if df_copy[col].dtype == "float64":
df_copy[col] = df_copy[col].round().astype('int64')
elif df_copy[col].dtype == "object":
df_copy[col] = df_copy[col].str.strip()
df_copy[col] = df_copy[col].apply(clean_currency)
if df_copy[col].dtype == "bool":
df_copy[col] = df_copy[col].astype(str)
saved_logged_cols_after.append(f"{col}: {df_copy[col].dtype}")
# the messages will save the logs of data file columns
msg_str_before = '\n'.join(saved_logged_cols_base)
msg_str_after = '\n'.join(saved_logged_cols_after)
saved_logged_msg = "\nMain Column with Data type: \n[\n {} \n]\n Converted Columns data type: \n[\n {} \n]\n".format(
msg_str_before, msg_str_after)
log_info(saved_logged_msg)
delete_data_file(file_path)
if data_file.suffix == ".xlsx":
df_copy.to_excel(data_file.as_posix(),
header=new_cleand_cols,
index=False)
elif data_file.suffix == ".csv":
df_copy.to_csv(data_file.as_posix(),
header=new_cleand_cols,
index=False,
sep=',')
cprint("save done", 'green')
except Exception as ex:
cprint(traceback.format_exc(), 'red')
delete_data_file(file_path)
log_exception(traceback.format_exc())
def post(self, request):
try:
member = Member.objects.get(email=request.user.email)
member.first_name = request.POST.get("first-name").strip()
member.last_name = request.POST.get("last-name").strip()
member.full_name = f'{request.POST.get("first-name").strip()} {request.POST.get("last-name").strip()}'
member.email = request.POST.get("email").strip()
member.phone = request.POST.get("phone").strip()
member.save()
messages.success(request, 'your info have been updated successfully!')
return redirect(reverse('profile-personal'))
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
messages.error(request, 'There is errors!, try again latter')
def get_all_columns_with_dtypes(self):
columns_with_dtypes = {}
try:
all_cols = self.selected_columns_dtypes.split("|")
for col in all_cols:
col_name, col_dtype = col.split(":")
columns_with_dtypes[col_name] = col_dtype
except ValueError:
pass
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
finally:
return columns_with_dtypes
def get_all_data_file_columns(self):
cols_all_dtype = {}
try:
all_cols_str = self.all_columns_with_dtypes.split("|")
for col in all_cols_str:
if col != "":
col_nm, col_tp = col.split(":")
cols_all_dtype[col_nm] = col_tp
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
except Exception as ex:
log_exception(traceback.format_exc())
finally:
return cols_all_dtype
def check_data_or_num(params: str):
try:
data_or_num = ''
last_param = params.strip().split('/')[-1]
if last_param == '':
last_param = params.strip().split('/')[-2]
if last_param.isdigit():
data_or_num = int(last_param)
else:
data_or_num = 'data'
return data_or_num
except AttributeError:
pass
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def remove_spaces_from_columns_names(file_path):
"""
this function will take dataframe path and save the file without
spaces in the columns name
"""
try:
path_obj = Path(file_path)
df = get_df_from_data_file(file_path)
df.columns = df.columns.str.strip()
delete_data_file(file_path)
if path_obj.suffix == ".xlsx":
df.to_excel(path_obj.as_posix(), index=False)
elif path_obj.suffix == ".csv":
df.to_csv(path_obj.as_posix(), index=False, sep=',')
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def get_rows_data_by_columns(file_path, columns, records_count,
columns_with_types, all_original_columns):
try:
all_rows = []
# print(columns)
df = get_df_from_data_file(file_path)
records_count = int(records_count)
previous_50_count = int(records_count - 50)
print(previous_50_count, records_count)
# df2 = df.loc[previous_50_count:records_count, columns]
# print(df2)
current_record_data = {}
for index, row in islice(df[columns].iterrows(), previous_50_count,
records_count):
# print(index)
# index is the index in the data frame
# row is the series object
idx = index
for col in columns:
# print(row[col])
# print(idx, col)
tmp_cell_val = row[col]
current_record_data["PANDAS_ID"] = idx
tmp_cell_val = replace_nan_value(tmp_cell_val)
# tmp_cell_val = tmp_cell_val.rstrip('0').rstrip('.') if '.' in tmp_cell_val else tmp_cell_val
# print(columns_with_types[col], tmp_cell_val)
current_record_data[col] = validate_obj.detect_and_validate(
tmp_cell_val,
dtype=columns_with_types[col],
original_dtype=all_original_columns[col])
# print(idx, "--> ", current_record_data[col])
all_rows.insert(0, current_record_data)
current_record_data = {}
# cprint(len(all_rows), 'yellow')
# pprint(all_rows[0])
# check if the length of all_rows < 0 means no records to show
if len(all_rows) <= 0:
return 0
else:
return all_rows
except Exception as ex:
# cprint(str(ex), 'red')
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def download_dashboard_pdf(request):
try:
from django.core.files.storage import FileSystemStorage
request.build_absolute_uri('/')
html_string = render_to_string('members_app/profile/dashboard.html')
html = HTML(string=html_string)
html.write_pdf(target='/tmp/dashboard.pdf');
fs = FileSystemStorage('/tmp')
with fs.open('dashboard.pdf') as pdf:
response = HttpResponse(pdf, content_type='application/pdf')
response['Content-Disposition'] = 'attachment; filename="dashboard.pdf"'
return response
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def extract_all_columns_with_dtypes(file_name):
try:
all_columns = {} # hold all columns in the file
df = get_df_from_data_file(file_name)
# iterating the columns
for col in df.columns:
# print(col)
# print(type(df.dtypes[col]))
all_columns[col] = str(df.dtypes[col])
# all_columns = sorted(all_columns)
# print(all_columns)
return all_columns
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def download_data_file_converter(member_data_file):
selected_columns = member_data_file.get_selected_columns_as_list
data_file_path = Path(member_data_file.data_file_path)
df = get_df_from_data_file(data_file_path)
try:
if data_file_path.suffix == ".xlsx":
df.to_excel(data_file_path.as_posix(),
header=True,
index=False,
columns=selected_columns)
elif data_file_path.suffix == ".csv":
df.to_csv(data_file_path.as_posix(),
header=True,
index=False,
columns=selected_columns)
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def get(self, request):
try:
context = {}
member = Member.objects.get(email=request.user.email)
context['member'] = member
from data_handler.models import (DataFile, DataHandlerSession)
from membership.models import Subscription
import pytz
from django.utils import timezone
subscription_obj = Subscription.objects.filter(
member_id=member).first()
context['sub_obj'] = subscription_obj
member_data_file = DataFile.objects.get(member=member)
member_data_session = DataHandlerSession.objects.filter(
data_handler_id=member_data_file)
today = datetime.now(tz=pytz.UTC)
between = subscription_obj.subscription_period_end - today # this to get how many days left to end of subscription
records_left = calculate_records_left_percentage(
member_data_file, member_data_session.first())
# for i in dir(member_data_file):
# if not i.startswith("__"):
# cprint(i, 'cyan')
if member_data_file.data_sessions_set.count() > 0:
context['has_session'] = True
context['is_process_complete'] = member_data_session.first(
).is_process_complete
context['data_session'] = member_data_session.first()
context['days_left'] = between.days,
context['records_left'] = records_left
else:
context['has_session'] = False
context['is_process_complete'] = False
context['data_session'] = None
context['days_left'] = None
context['records_left'] = None
return render(request,
"members_app/profile/overview.html",
context=context)
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def detect_and_validate(self, val="", dtype=None, original_dtype=None):
"""
this method will call all validation methods in this class
Returns:
"""
# [unique identifier (id), textual field, numeric field, donation field]
try:
the_value = val
if "unique identifier" in dtype:
san = self.sanitize_numeric_data(the_value, 'unique_id')
elif "text field" in dtype:
san = self.sanitize_numeric_data(the_value, 'text')
elif "numeric field" in dtype:
san = self.sanitize_numeric_data(the_value, 'numeric')
elif "donation field" in dtype:
san = self.sanitize_numeric_data(the_value, 'donation')
if san.status:
# check if the value contain any error or not valid data
the_validate_dict_values = self.return_the_error_msg(
True,
san.value,
"data not valid",
data_type=f"{dtype}",
org_dtype=original_dtype)
# cprint(the_validate_dict_values, 'blue')
else:
the_validate_dict_values = self.return_the_error_msg(
False,
san.value,
"valid data",
data_type=f"{dtype}",
org_dtype=original_dtype)
# print(the_validate_dict_values)
# print(dtype)
return the_validate_dict_values
except Exception as ex:
cprint(traceback.format_exc(), 'red')
cprint(str(ex), 'red')
log_exception(traceback.format_exc())
def post(self, request):
try:
if request.is_ajax():
from data_handler.models import (DataFile, DataHandlerSession)
member = Member.objects.get(email=request.user.email)
member_data_file = DataFile.objects.get(member=member)
member_data_session = DataHandlerSession.objects.filter(
data_handler_id=member_data_file)
# check if there is session to return the correct value to display on the dashboard
if member_data_session.count() > 0:
records_left = calculate_records_left_percentage(
member_data_file, member_data_session.first())
return JsonResponse(data={"value": int(records_left)},
status=200)
else:
return JsonResponse(data={"value": int(0)}, status=200)
except Exception as ex:
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
def extract_all_column_names(file_name):
try:
all_columns = [] # hold all columns in the file
df = get_df_from_data_file(file_name)
full_file_path = Path(file_name)
# iterating the columns
for col in df.columns:
# print(col)
# print(type(df.dtypes[col]))
all_columns.append(col)
# all_columns = df.columns
# all_columns = sorted(all_columns)
# print(all_columns)
return all_columns
except Exception as ex:
cprint(str(ex), 'red')
cprint(traceback.format_exc(), 'red')
log_exception(traceback.format_exc())
