def save(self, **kwargs): self.name = slugify(self.name) super().save(**kwargs) self.do_migrations() self.model_cleanup() for fn in self._POST_SAVE_SIGNALS: fn(self)
def from_csv_file(filename, file): """ Import a file from an upload, using its filename as the data source name. """ csv = clean_csv_headers(file.read().decode("utf-8")) name = filename no_ext = re.findall(r"^(.*)\.csv$", name) if no_ext: name = no_ext[0] dynmodel = from_csv(slugify(name), csv) with StringIO() as fio: fio.write(csv) dynmodel.csv_file.save(name, fio) dynmodel.save() return dynmodel
def begin(request): """ Entry point for setting up the rest of the system. At this point the user has logged in using the default login and are now getting ready to configure the database, schema (via Google sheets URL) and any authentication backends (Google Oauth2, Slack, etc). """ if request.method == "GET": # Don't go back into this flow if we've already done it addnew = request.GET.get("addnew") models_count = models.DynamicModel.objects.count() if not addnew and models_count: return redirect('/admin/') context = { "first_run": True, } creds_model = get_credentials_model() if creds_model: context["service_account_email"] = get_service_account_email( creds_model) if CredentialStore.objects.count(): context["first_run"] = False # NOTE: this leaves this module separated from collaborate, but # still compatable in its presence try: AppSetting = apps.get_model("collaborative", "appsetting") app_setting = AppSetting.objects.filter( name="initial_setup_completed").count() if app_setting: context["first_run"] = False except LookupError: pass return render(request, 'begin.html', context) elif request.method == "POST": # For CSV URL/Google Sheets (public) csv_url = request.POST.get("csv_url") # Private Sheet toggle (all other private sheet stuff will # be ignored if this wasn't checked in the form # NOTE: Why getlist? Well, django POST with checkboxes only # returns blank strings as checkbox values. So we check the # list to see if it has any length (not not) and go with that. # (A list like this [''] is truthy, this [] is not!) # Unsolved SO issue: https://stackoverflow.com/questions/47374647 csv_sheets_private = request.POST.getlist("csv_google_sheet_private") # Private Google Sheet (Service Account Credentials JSON) ... # this only gets displayed when there are no other creds uploaded csv_google_credentials_file = request.FILES.get( "csv_google_credentials") # Also see if we've already saved a dynamic model, we will use # the credentials from this file and the email for the UI creds_model = get_credentials_model() service_account_email = None if creds_model: service_account_email = get_service_account_email(creds_model) # Screendoor sd_api_key = request.POST.get("sd_api_key") sd_project_id = request.POST.get("sd_project_id") sd_form_id = request.POST.get("sd_form_id") # CSV File Upload csv_file = request.FILES.get("csv_file_upload") # TODO: move all this into a form context = { "csv_name": request.POST.get("csv_name"), "csv_url": csv_url, "sd_name": request.POST.get("sd_name"), "sd_api_key": sd_api_key, "sd_project_id": sd_project_id, "sd_form_id": sd_form_id, "service_account_email": service_account_email, } try: if csv_url and csv_sheets_private: name = slugify(request.POST.get("csv_name")) # pull the credentials from FILE or saved model credentials = None if csv_google_credentials_file: credentials = csv_google_credentials_file.read().decode( "utf-8") cr, _ = CredentialStore.objects.get_or_create( name="csv_google_credentials", ) cr.credentials = credentials cr.save() elif creds_model: credentials = creds_model.credentials else: # TODO: raise if we got a check marked private, # but couldn't find the credential and none uploaded # here. we should direct them to the google page pass dynmodel = from_private_sheet( name, csv_url, credentials=credentials, ) elif csv_url: name = slugify(request.POST.get("csv_name")) dynmodel = from_csv_url( name, csv_url, ) elif sd_api_key: name = slugify(request.POST.get("sd_name")) max_import_records = None if hasattr(settings, "MAX_IMPORT_RECORDS"): max_import_records = settings.MAX_IMPORT_RECORDS dynmodel = from_screendoor( name, sd_api_key, int(sd_project_id), form_id=int(sd_form_id) if sd_form_id else None, max_import_records=max_import_records, ) elif csv_file: dynmodel = from_csv_file( csv_file.name, csv_file, ) else: return render(request, 'begin.html', { "errors": "No data source selected!", **context }) except Exception as e: # TODO: roll back if not isinstance(e, GenericCSVError): raise e return render(request, 'begin.html', { "errors": e.render(), **context }) # handles valid URLs to non-CSV data and also just bad URLs except UnsupportedFormat as e: # TODO: roll back err_msg = _("Invalid data source. Please make sure you " "linked to a valid CSV data source.") return render(request, 'begin.html', { "errors": err_msg, **context }) except ConnectionError as e: # TODO: roll back err_msg = _("Invalid URL. Please make sure there aren't " "typos in the URL, and that the data isn't " "protected. If you're trying to use a protected " "Google Sheet, you need to use the private Sheet " "authenticator, below.") return render(request, 'begin.html', { "errors": err_msg, **context }) return redirect('csv_models:refine-and-import', dynmodel.id)
def begin(request): """ Entry point for setting up the rest of the system. At this point the user has logged in using the default login and are now getting ready to configure the database, schema (via Google sheets URL) and any authentication backends (Google Oauth2, Slack, etc). """ if request.method == "GET": # Don't go back into this flow if we've already done it addnew = request.GET.get("addnew") models_count = models.DynamicModel.objects.count() if addnew: return render(request, 'begin.html', {}) elif models_count: return redirect('/admin/') return render(request, 'begin.html', {}) elif request.method == "POST": # get params from request csv_url = request.POST.get("csv_url") csv_google_sheets_auth_code = request.POST.get( "csv_google_sheets_auth_code" ) sd_api_key = request.POST.get("sd_api_key") sd_project_id = request.POST.get("sd_project_id") sd_form_id = request.POST.get("sd_form_id") # TODO: move all this into a form context = { "csv_name": request.POST.get("csv_name"), "csv_url": csv_url, "csv_google_sheets_auth_code": csv_google_sheets_auth_code, "sd_name": request.POST.get("sd_name"), "sd_api_key": sd_api_key, "sd_project_id": sd_project_id, "sd_form_id": sd_form_id, } try: if csv_url and csv_google_sheets_auth_code: name = slugify(request.POST.get("csv_name")) dynmodel = from_private_sheet( name, csv_url, auth_code=csv_google_sheets_auth_code, ) elif csv_url: name = slugify(request.POST.get("csv_name")) dynmodel = from_csv_url( name, csv_url, csv_google_sheets_auth_code=csv_google_sheets_auth_code ) elif sd_api_key: name = slugify(request.POST.get("sd_name")) dynmodel = from_screendoor( name, sd_api_key, int(sd_project_id), form_id=int(sd_form_id) if sd_form_id else None ) except (UniqueColumnError, DataSourceExistsError) as e: return render(request, 'begin.html', { "errors": e.render(), **context }) # handles valid URLs to non-CSV data and also just bad URLs except UnsupportedFormat as e: err_msg = _( "Invalid data source. Please make sure you " "linked to a valid CSV data source." ) return render(request, 'begin.html', { "errors": err_msg, **context }) except ConnectionError as e: err_msg = _( "Invalid URL. Please make sure there aren't " "typos in the URL, and that the data isn't " "protected. If you're trying to use a protected " "Google Sheet, you need to use the private Sheet " "authenticator, below." ) return render(request, 'begin.html', { "errors": err_msg, **context }) return redirect('csv_models:refine-and-import', dynmodel.id)