def receive(self, text_data=None, bytes_data=None): try: if text_data == "RUN_THE_MODEL": from data_handler.models import (DataFile, DataHandlerSession) member_data_file = DataFile.objects.get(member=self.user) data_session = DataHandlerSession.objects.get( data_handler_id=member_data_file) donation_cols = data_session.donation_columns # run_model(data_session.data_file_path, donation_cols) # run_model_data = run_model(data_session.base_data_file_path, donation_cols, self) run_model_data = run_model(data_session.data_file_path, donation_cols, self) if run_model_data: cprint("Run model completed!", 'green') data_session.pdf_report_file_path = run_model_data.get( 'PDF_FILE') data_session.csv_report_file_path = run_model_data.get( 'CSV_FILE') data_session.is_process_complete = True data_session.save() member_data_file.is_run_the_model = True member_data_file.save() cprint("save is run the model", 'yellow') cprint('save to db done', 'yellow') # self.send(text_data="Run complete", close=True) self.send("Complete Successfully!") except Exception as ex: cprint(traceback.format_exc(), 'red') cprint(str(ex), 'red') log_exception(ex)
def post(self, request): try: # ^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[a-zA-Z]).{8,}$ import re member = Member.objects.get(email=request.user.email) # 'password', 'new-password', 'verify-new-password' if request.POST.get('password') == '': messages.error(request, 'Password is empty!!') elif request.POST.get("new-password") == "": messages.error(request, 'New Password is empty!!') elif request.POST.get("verify-new-password") == "": messages.error(request, 'You have to verify new password is empty!!') elif request.POST.get("verify-new-password") != request.POST.get("new-password"): messages.error(request, 'Password not verified or matched!!') else: if member.check_password(request.POST.get("password")) is True: pattern = re.compile(r"^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[a-zA-Z]).{8,}$") if pattern.match(request.POST.get('new-password')): member.set_password(request.POST.get("new-password")) member.save() update_session_auth_hash(request, member) messages.success(request, 'Your password has been updated!') else: messages.error(request, 'Your password not match password requirements!') else: messages.error(request, 'Your old password is not correct!') return redirect(reverse('profile-change-password')) except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc()) messages.error(request, 'There is errors!, try again latter')
def post(self, request): try: member = Member.objects.get(email=request.user.email) # ['csrfmiddlewaretoken', 'org_name', 'org_website', 'organizationType', 'annualRevenue', 'job_title', 'total_staff', 'num_of_volunteer'] member.org_name = request.POST.get("org_name").strip() member.org_website = request.POST.get("org_website").strip() if request.POST.get("org_type") != "Other": member.org_type = request.POST.get("org_type").strip() else: member.org_type = request.POST.get("other-org-type").strip() member.annual_revenue = request.POST.get("annualRevenue").strip() member.job_title = request.POST.get("job_title").strip() member.total_staff = request.POST.get("total_staff").strip() member.num_of_volunteer = request.POST.get( "num_of_volunteer").strip() member.num_of_board_members = request.POST.get( "num_of_board_members").strip() member.save() messages.success(request, 'your info have been updated successfully!') return redirect(reverse('profile-info')) except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc()) messages.error(request, 'There is errors!, try again latter')
def post(self, request, format=None): try: post_data = request.POST # cprint(request.POST, "blue") filter_array = json.loads(request.POST.get("filtersArray")) filter_report_section = request.POST.get("reportSectionName") # cprint(filter_array, "yellow") # cprint(filter_report_section, "red") reports_headers = ReportGenerator.generate_reports_table_header( filter_array) # cprint(reports_headers, "green") reports = ReportGenerator.generate_report(filter_report_section, filter_array, reports_headers, request.user.pk) # ReportGenerator.generate_reports(filter_array) # cprint(reports.keys(), 'green') # cprint(simple_json.dumps(reports.get("data")), 'green') # cprint(json.dumps(reports.get("data")), 'green') # cprint(type(reports.get("data")), 'cyan') # cprint(reports, 'blue') # cprint(reports_headers, 'red') # cprint(reports.get("data"), 'blue') return Response( { "table_header": reports_headers, "report_data": reports.get('data'), "report_section_name": filter_report_section }, status=200) except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def delete_all_member_data_file_info(member_data_file): """ this function will take DataFile object, to reset all member data file Args: member_data_file: Returns: """ try: from data_handler.models import DataHandlerSession for dfile in DataHandlerSession.objects.filter( data_handler_id=member_data_file): delete_data_file(dfile.data_file_path) DataHandlerSession.objects.filter( data_handler_id=member_data_file).delete() # member_data_session = DataHandlerSession.objects.get(data_handler_id=member_data_file) # member_data_file.data_file_path = "None" # member_data_file.file_upload_procedure = "None" # member_data_file.all_records_count = 0 # member_data_file.selected_columns = "" # member_data_file.selected_columns_dtypes = "" # member_data_file.donor_id_column = "" # member_data_file.is_donor_id_selected = False # member_data_file.unique_id_column = "" # member_data_file.all_columns_with_dtypes = "" # member_data_file.is_process_complete = False # member_data_file.save() except DataHandlerSession.DoesNotExist: cprint('DataHandlerSession.DoesNotExist', 'red') except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def get(self, request): try: member = Member.objects.get(email=request.user.email) return render(request, "members_app/profile/change-password.html", context={"member": member}) except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def post(self, request, format=None): try: post_data = request.POST reports_section_name = request.POST.get("reports_section_name") all_filter_cookies = request.POST.get("all_filter_cookies") all_filter_cookies = json.loads(all_filter_cookies) displayed_columns = request.POST.get("displayed_columns") displayed_columns = json.loads(displayed_columns) cprint(reports_section_name, 'red') cprint(all_filter_cookies, 'yellow') cprint(len(all_filter_cookies), 'green') cprint(displayed_columns, 'blue') report_obj = ReportGenerator(reports_section_name, displayed_columns, all_filter_cookies) cprint(report_obj, 'magenta') return Response( { "table_header": report_obj.get_displayed_columns(), 'report_rows': report_obj.get_rows_file() }, status=200) except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def get(self, request): try: member = Member.objects.get(email=request.user.email) return render(request, "members_app/profile/information.html", context={"member": member, 'annual_revenue': ANNUAL_REVENUE, 'org_types': ORGANIZATION_TYPES}) except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def disconnect(self, close_code): try: self.disconnect(close_code) self.close() cprint("Close the connection", 'green') except Exception as ex: cprint(traceback.format_exc(), 'red') cprint(str(ex), 'red') log_exception(ex)
def check_empty_df(file_path): try: df = get_df_from_data_file(file_path) if df.empty is True: return True return False except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def get_selected_columns_as_list(self): # return self.selected_columns.split("|") # return sorted(self.selected_columns.split("|")) try: return self.selected_columns.split("|") except AttributeError as aex: pass except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def validate_empty(self, value): try: tmp_val = value if tmp_val == "" or tmp_val is None or tmp_val == "nan" or tmp_val == "Nan": return True else: return False except Exception as ex: cprint(traceback.format_exc(), 'red') cprint(str(ex), 'red') log_exception(traceback.format_exc())
def update_rows_data(file_path, data_json, column_names, columns_with_dtypes): # pd.describe_option("display.float_format") # pd.set_option("display.float_format", "{:.2f}".format) try: data_file = Path(file_path) all_rows = [] rows_and_values = {} df = get_df_from_data_file(file_path) for key, value in data_json.items(): # ROW_0 [{'colName': 'Cand_Name', 'colValue': '858f'}] rows_and_values[key.split('_')[1]] = value # df2 = copy.deepcopy(df[column_names]) # df2 = copy.deepcopy(df) df2 = df.copy() current_value = "" for key, value in rows_and_values.items(): # {"0": [{"colName", "colValue"}, {"colName", "colValue"}] # 0 [{'colName': 'Cand_Name', 'colValue': '858fx'}] # print(key, value) for val in value: # cprint(df2[val['colName']].dtype, 'yellow') current_value = val['colValue'] if df2[val['colName']].dtype == 'int64': if current_value.isdigit(): df2.at[int(key), val['colName']] = current_value else: df2[val['colName']] = df2[val['colName']].astype(str) df2.at[int(key), val['colName']] = current_value # df2[val['colName']] = df2[val['colName']].astype(int) # df2[val['colName']] = pd.to_numeric(df2[val['colName']], errors='ignore', downcast='float') # df2[val['colName']] = pd.to_numeric(df2[val['colName']]) # df2[val['colName']] = df2[val['colName']].astype('int64') else: df2.at[int(key), val['colName']] = current_value # cprint(df2[val['colName']].dtype, 'blue') # save all changes to the file if data_file.suffix == ".xlsx": df2.to_excel(data_file.as_posix(), header=True, index=False) elif data_file.suffix == ".csv": df2.to_csv(data_file.as_posix(), header=True, index=False) # cprint(df2.dtypes, 'magenta') # return "Data saved successfully" return current_value, "Data saved successfully" except Exception as ex: cprint(str(ex), 'red') cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def clean_currency(x: str): """ If the value is a string, then remove currency symbol and delimiters otherwise, the value is numeric and can be converted """ try: # x = str(x) if isinstance(x, str): if x.startswith("$"): return x.replace('$', '').replace(',', '') # return float(x) return x except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def get_df_from_data_file(file_path): try: from data_handler.models import DataHandlerSession data_file = Path(file_path) file_object = DataHandlerSession.objects.filter( data_file_path=data_file.as_posix()).first() df = None # cprint(file_object.get_all_data_file_columns, 'yellow') if file_object: if data_file.exists(): if data_file.suffix == ".xlsx": df = pd.read_excel(data_file.as_posix()) elif data_file.suffix == ".csv": df = pd.read_csv(data_file.as_posix(), sep=',', skipinitialspace=True) else: if data_file.exists(): if data_file.suffix == ".xlsx": df = pd.read_excel(data_file.as_posix()) elif data_file.suffix == ".csv": df = pd.read_csv(data_file.as_posix(), sep=',') # cprint(df.dtypes, 'red') # this for fill the empty cells with its own empty values float_cols = df.select_dtypes(include=['float64']).columns str_cols = df.select_dtypes(include=['object']).columns int_cols = df.select_dtypes(include=['int64']).columns df.loc[:, float_cols] = df.loc[:, float_cols].fillna(0) df.loc[:, int_cols] = df.loc[:, int_cols].fillna(0) df.loc[:, str_cols] = df.loc[:, str_cols].fillna('NULL') df_clone = df.copy() # this loop to convert bool dtype to string for co in df_clone.columns.tolist(): if df_clone[co].dtype == 'bool': # cprint(df_clone[co].dtype, 'blue') df_clone[co] = df_clone[co].apply(str) # cprint(df_clone[co].dtype, 'green') elif df_clone[co].dtype == 'float64': df_clone[co] = df_clone[co].round().astype(int) # cprint(df_clone.dtypes, 'green') return df_clone except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def send_data(obj): try: obj.send(text_data=f"member_data_file") time.sleep(3) obj.send(text_data='Now sleep after sleep 3') time.sleep(2) obj.send(text_data=f"data_session") time.sleep(2) obj.send(text_data='Request complete', close=True) except Exception as ex: cprint(traceback.format_exc(), 'red') cprint(str(ex), 'red') obj.send( text_data=str(ex)) # to display the error in the session modal log_exception(ex)
def save_data_file_rounded(file_path): data_file = Path(file_path) df = get_df_from_data_file(file_path) df_copy = df.copy() saved_logged_msg = '' # the info log will save, contains columns name, columns dtypes saved_logged_cols_base = [ ] # the columns with dtype log will save, contains columns name, columns dtypes saved_logged_cols_after = [] # the columns with converted dtype new_cleand_cols = [ ] # this list all hold all columns without any spaces or whitespaces # Volunteered in the past # cprint(df.dtypes, 'green') try: for col in df_copy.columns.tolist(): new_cleand_cols.append(col.strip()) saved_logged_cols_base.append(f"{col}: {df_copy[col].dtype}") if df_copy[col].dtype == "float64": df_copy[col] = df_copy[col].round().astype('int64') elif df_copy[col].dtype == "object": df_copy[col] = df_copy[col].str.strip() df_copy[col] = df_copy[col].apply(clean_currency) if df_copy[col].dtype == "bool": df_copy[col] = df_copy[col].astype(str) saved_logged_cols_after.append(f"{col}: {df_copy[col].dtype}") # the messages will save the logs of data file columns msg_str_before = '\n'.join(saved_logged_cols_base) msg_str_after = '\n'.join(saved_logged_cols_after) saved_logged_msg = "\nMain Column with Data type: \n[\n {} \n]\n Converted Columns data type: \n[\n {} \n]\n".format( msg_str_before, msg_str_after) log_info(saved_logged_msg) delete_data_file(file_path) if data_file.suffix == ".xlsx": df_copy.to_excel(data_file.as_posix(), header=new_cleand_cols, index=False) elif data_file.suffix == ".csv": df_copy.to_csv(data_file.as_posix(), header=new_cleand_cols, index=False, sep=',') cprint("save done", 'green') except Exception as ex: cprint(traceback.format_exc(), 'red') delete_data_file(file_path) log_exception(traceback.format_exc())
def post(self, request): try: member = Member.objects.get(email=request.user.email) member.first_name = request.POST.get("first-name").strip() member.last_name = request.POST.get("last-name").strip() member.full_name = f'{request.POST.get("first-name").strip()} {request.POST.get("last-name").strip()}' member.email = request.POST.get("email").strip() member.phone = request.POST.get("phone").strip() member.save() messages.success(request, 'your info have been updated successfully!') return redirect(reverse('profile-personal')) except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc()) messages.error(request, 'There is errors!, try again latter')
def get_all_columns_with_dtypes(self): columns_with_dtypes = {} try: all_cols = self.selected_columns_dtypes.split("|") for col in all_cols: col_name, col_dtype = col.split(":") columns_with_dtypes[col_name] = col_dtype except ValueError: pass except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc()) finally: return columns_with_dtypes
def get_all_data_file_columns(self): cols_all_dtype = {} try: all_cols_str = self.all_columns_with_dtypes.split("|") for col in all_cols_str: if col != "": col_nm, col_tp = col.split(":") cols_all_dtype[col_nm] = col_tp except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc()) except Exception as ex: log_exception(traceback.format_exc()) finally: return cols_all_dtype
def check_data_or_num(params: str): try: data_or_num = '' last_param = params.strip().split('/')[-1] if last_param == '': last_param = params.strip().split('/')[-2] if last_param.isdigit(): data_or_num = int(last_param) else: data_or_num = 'data' return data_or_num except AttributeError: pass except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def remove_spaces_from_columns_names(file_path): """ this function will take dataframe path and save the file without spaces in the columns name """ try: path_obj = Path(file_path) df = get_df_from_data_file(file_path) df.columns = df.columns.str.strip() delete_data_file(file_path) if path_obj.suffix == ".xlsx": df.to_excel(path_obj.as_posix(), index=False) elif path_obj.suffix == ".csv": df.to_csv(path_obj.as_posix(), index=False, sep=',') except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def get_rows_data_by_columns(file_path, columns, records_count, columns_with_types, all_original_columns): try: all_rows = [] # print(columns) df = get_df_from_data_file(file_path) records_count = int(records_count) previous_50_count = int(records_count - 50) print(previous_50_count, records_count) # df2 = df.loc[previous_50_count:records_count, columns] # print(df2) current_record_data = {} for index, row in islice(df[columns].iterrows(), previous_50_count, records_count): # print(index) # index is the index in the data frame # row is the series object idx = index for col in columns: # print(row[col]) # print(idx, col) tmp_cell_val = row[col] current_record_data["PANDAS_ID"] = idx tmp_cell_val = replace_nan_value(tmp_cell_val) # tmp_cell_val = tmp_cell_val.rstrip('0').rstrip('.') if '.' in tmp_cell_val else tmp_cell_val # print(columns_with_types[col], tmp_cell_val) current_record_data[col] = validate_obj.detect_and_validate( tmp_cell_val, dtype=columns_with_types[col], original_dtype=all_original_columns[col]) # print(idx, "--> ", current_record_data[col]) all_rows.insert(0, current_record_data) current_record_data = {} # cprint(len(all_rows), 'yellow') # pprint(all_rows[0]) # check if the length of all_rows < 0 means no records to show if len(all_rows) <= 0: return 0 else: return all_rows except Exception as ex: # cprint(str(ex), 'red') cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def download_dashboard_pdf(request): try: from django.core.files.storage import FileSystemStorage request.build_absolute_uri('/') html_string = render_to_string('members_app/profile/dashboard.html') html = HTML(string=html_string) html.write_pdf(target='/tmp/dashboard.pdf'); fs = FileSystemStorage('/tmp') with fs.open('dashboard.pdf') as pdf: response = HttpResponse(pdf, content_type='application/pdf') response['Content-Disposition'] = 'attachment; filename="dashboard.pdf"' return response except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def extract_all_columns_with_dtypes(file_name): try: all_columns = {} # hold all columns in the file df = get_df_from_data_file(file_name) # iterating the columns for col in df.columns: # print(col) # print(type(df.dtypes[col])) all_columns[col] = str(df.dtypes[col]) # all_columns = sorted(all_columns) # print(all_columns) return all_columns except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def download_data_file_converter(member_data_file): selected_columns = member_data_file.get_selected_columns_as_list data_file_path = Path(member_data_file.data_file_path) df = get_df_from_data_file(data_file_path) try: if data_file_path.suffix == ".xlsx": df.to_excel(data_file_path.as_posix(), header=True, index=False, columns=selected_columns) elif data_file_path.suffix == ".csv": df.to_csv(data_file_path.as_posix(), header=True, index=False, columns=selected_columns) except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def get(self, request): try: context = {} member = Member.objects.get(email=request.user.email) context['member'] = member from data_handler.models import (DataFile, DataHandlerSession) from membership.models import Subscription import pytz from django.utils import timezone subscription_obj = Subscription.objects.filter( member_id=member).first() context['sub_obj'] = subscription_obj member_data_file = DataFile.objects.get(member=member) member_data_session = DataHandlerSession.objects.filter( data_handler_id=member_data_file) today = datetime.now(tz=pytz.UTC) between = subscription_obj.subscription_period_end - today # this to get how many days left to end of subscription records_left = calculate_records_left_percentage( member_data_file, member_data_session.first()) # for i in dir(member_data_file): # if not i.startswith("__"): # cprint(i, 'cyan') if member_data_file.data_sessions_set.count() > 0: context['has_session'] = True context['is_process_complete'] = member_data_session.first( ).is_process_complete context['data_session'] = member_data_session.first() context['days_left'] = between.days, context['records_left'] = records_left else: context['has_session'] = False context['is_process_complete'] = False context['data_session'] = None context['days_left'] = None context['records_left'] = None return render(request, "members_app/profile/overview.html", context=context) except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def detect_and_validate(self, val="", dtype=None, original_dtype=None): """ this method will call all validation methods in this class Returns: """ # [unique identifier (id), textual field, numeric field, donation field] try: the_value = val if "unique identifier" in dtype: san = self.sanitize_numeric_data(the_value, 'unique_id') elif "text field" in dtype: san = self.sanitize_numeric_data(the_value, 'text') elif "numeric field" in dtype: san = self.sanitize_numeric_data(the_value, 'numeric') elif "donation field" in dtype: san = self.sanitize_numeric_data(the_value, 'donation') if san.status: # check if the value contain any error or not valid data the_validate_dict_values = self.return_the_error_msg( True, san.value, "data not valid", data_type=f"{dtype}", org_dtype=original_dtype) # cprint(the_validate_dict_values, 'blue') else: the_validate_dict_values = self.return_the_error_msg( False, san.value, "valid data", data_type=f"{dtype}", org_dtype=original_dtype) # print(the_validate_dict_values) # print(dtype) return the_validate_dict_values except Exception as ex: cprint(traceback.format_exc(), 'red') cprint(str(ex), 'red') log_exception(traceback.format_exc())
def post(self, request): try: if request.is_ajax(): from data_handler.models import (DataFile, DataHandlerSession) member = Member.objects.get(email=request.user.email) member_data_file = DataFile.objects.get(member=member) member_data_session = DataHandlerSession.objects.filter( data_handler_id=member_data_file) # check if there is session to return the correct value to display on the dashboard if member_data_session.count() > 0: records_left = calculate_records_left_percentage( member_data_file, member_data_session.first()) return JsonResponse(data={"value": int(records_left)}, status=200) else: return JsonResponse(data={"value": int(0)}, status=200) except Exception as ex: cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())
def extract_all_column_names(file_name): try: all_columns = [] # hold all columns in the file df = get_df_from_data_file(file_name) full_file_path = Path(file_name) # iterating the columns for col in df.columns: # print(col) # print(type(df.dtypes[col])) all_columns.append(col) # all_columns = df.columns # all_columns = sorted(all_columns) # print(all_columns) return all_columns except Exception as ex: cprint(str(ex), 'red') cprint(traceback.format_exc(), 'red') log_exception(traceback.format_exc())