def save(self, **kwargs):
self.name = slugify(self.name)
super().save(**kwargs)
self.do_migrations()
self.model_cleanup()
for fn in self._POST_SAVE_SIGNALS:
fn(self)
def from_csv_file(filename, file):
"""
Import a file from an upload, using its filename as the data
source name.
"""
csv = clean_csv_headers(file.read().decode("utf-8"))
name = filename
no_ext = re.findall(r"^(.*)\.csv$", name)
if no_ext:
name = no_ext[0]
dynmodel = from_csv(slugify(name), csv)
with StringIO() as fio:
fio.write(csv)
dynmodel.csv_file.save(name, fio)
dynmodel.save()
return dynmodel
def begin(request):
"""
Entry point for setting up the rest of the system. At this point
the user has logged in using the default login and are now getting
ready to configure the database, schema (via Google sheets URL) and
any authentication backends (Google Oauth2, Slack, etc).
"""
if request.method == "GET":
# Don't go back into this flow if we've already done it
addnew = request.GET.get("addnew")
models_count = models.DynamicModel.objects.count()
if not addnew and models_count:
return redirect('/admin/')
context = {
"first_run": True,
}
creds_model = get_credentials_model()
if creds_model:
context["service_account_email"] = get_service_account_email(
creds_model)
if CredentialStore.objects.count():
context["first_run"] = False
# NOTE: this leaves this module separated from collaborate, but
# still compatable in its presence
try:
AppSetting = apps.get_model("collaborative", "appsetting")
app_setting = AppSetting.objects.filter(
name="initial_setup_completed").count()
if app_setting:
context["first_run"] = False
except LookupError:
pass
return render(request, 'begin.html', context)
elif request.method == "POST":
# For CSV URL/Google Sheets (public)
csv_url = request.POST.get("csv_url")
# Private Sheet toggle (all other private sheet stuff will
# be ignored if this wasn't checked in the form
# NOTE: Why getlist? Well, django POST with checkboxes only
# returns blank strings as checkbox values. So we check the
# list to see if it has any length (not not) and go with that.
# (A list like this [''] is truthy, this [] is not!)
# Unsolved SO issue: https://stackoverflow.com/questions/47374647
csv_sheets_private = request.POST.getlist("csv_google_sheet_private")
# Private Google Sheet (Service Account Credentials JSON) ...
# this only gets displayed when there are no other creds uploaded
csv_google_credentials_file = request.FILES.get(
"csv_google_credentials")
# Also see if we've already saved a dynamic model, we will use
# the credentials from this file and the email for the UI
creds_model = get_credentials_model()
service_account_email = None
if creds_model:
service_account_email = get_service_account_email(creds_model)
# Screendoor
sd_api_key = request.POST.get("sd_api_key")
sd_project_id = request.POST.get("sd_project_id")
sd_form_id = request.POST.get("sd_form_id")
# CSV File Upload
csv_file = request.FILES.get("csv_file_upload")
# TODO: move all this into a form
context = {
"csv_name": request.POST.get("csv_name"),
"csv_url": csv_url,
"sd_name": request.POST.get("sd_name"),
"sd_api_key": sd_api_key,
"sd_project_id": sd_project_id,
"sd_form_id": sd_form_id,
"service_account_email": service_account_email,
}
try:
if csv_url and csv_sheets_private:
name = slugify(request.POST.get("csv_name"))
# pull the credentials from FILE or saved model
credentials = None
if csv_google_credentials_file:
credentials = csv_google_credentials_file.read().decode(
"utf-8")
cr, _ = CredentialStore.objects.get_or_create(
name="csv_google_credentials", )
cr.credentials = credentials
cr.save()
elif creds_model:
credentials = creds_model.credentials
else:
# TODO: raise if we got a check marked private,
# but couldn't find the credential and none uploaded
# here. we should direct them to the google page
pass
dynmodel = from_private_sheet(
name,
csv_url,
credentials=credentials,
)
elif csv_url:
name = slugify(request.POST.get("csv_name"))
dynmodel = from_csv_url(
name,
csv_url,
)
elif sd_api_key:
name = slugify(request.POST.get("sd_name"))
max_import_records = None
if hasattr(settings, "MAX_IMPORT_RECORDS"):
max_import_records = settings.MAX_IMPORT_RECORDS
dynmodel = from_screendoor(
name,
sd_api_key,
int(sd_project_id),
form_id=int(sd_form_id) if sd_form_id else None,
max_import_records=max_import_records,
)
elif csv_file:
dynmodel = from_csv_file(
csv_file.name,
csv_file,
)
else:
return render(request, 'begin.html', {
"errors": "No data source selected!",
**context
})
except Exception as e:
# TODO: roll back
if not isinstance(e, GenericCSVError):
raise e
return render(request, 'begin.html', {
"errors": e.render(),
**context
})
# handles valid URLs to non-CSV data and also just bad URLs
except UnsupportedFormat as e:
# TODO: roll back
err_msg = _("Invalid data source. Please make sure you "
"linked to a valid CSV data source.")
return render(request, 'begin.html', {
"errors": err_msg,
**context
})
except ConnectionError as e:
# TODO: roll back
err_msg = _("Invalid URL. Please make sure there aren't "
"typos in the URL, and that the data isn't "
"protected. If you're trying to use a protected "
"Google Sheet, you need to use the private Sheet "
"authenticator, below.")
return render(request, 'begin.html', {
"errors": err_msg,
**context
})
return redirect('csv_models:refine-and-import', dynmodel.id)
def begin(request):
"""
Entry point for setting up the rest of the system. At this point
the user has logged in using the default login and are now getting
ready to configure the database, schema (via Google sheets URL) and
any authentication backends (Google Oauth2, Slack, etc).
"""
if request.method == "GET":
# Don't go back into this flow if we've already done it
addnew = request.GET.get("addnew")
models_count = models.DynamicModel.objects.count()
if addnew:
return render(request, 'begin.html', {})
elif models_count:
return redirect('/admin/')
return render(request, 'begin.html', {})
elif request.method == "POST":
# get params from request
csv_url = request.POST.get("csv_url")
csv_google_sheets_auth_code = request.POST.get(
"csv_google_sheets_auth_code"
)
sd_api_key = request.POST.get("sd_api_key")
sd_project_id = request.POST.get("sd_project_id")
sd_form_id = request.POST.get("sd_form_id")
# TODO: move all this into a form
context = {
"csv_name": request.POST.get("csv_name"),
"csv_url": csv_url,
"csv_google_sheets_auth_code": csv_google_sheets_auth_code,
"sd_name": request.POST.get("sd_name"),
"sd_api_key": sd_api_key,
"sd_project_id": sd_project_id,
"sd_form_id": sd_form_id,
}
try:
if csv_url and csv_google_sheets_auth_code:
name = slugify(request.POST.get("csv_name"))
dynmodel = from_private_sheet(
name, csv_url, auth_code=csv_google_sheets_auth_code,
)
elif csv_url:
name = slugify(request.POST.get("csv_name"))
dynmodel = from_csv_url(
name, csv_url,
csv_google_sheets_auth_code=csv_google_sheets_auth_code
)
elif sd_api_key:
name = slugify(request.POST.get("sd_name"))
dynmodel = from_screendoor(
name,
sd_api_key,
int(sd_project_id),
form_id=int(sd_form_id) if sd_form_id else None
)
except (UniqueColumnError, DataSourceExistsError) as e:
return render(request, 'begin.html', {
"errors": e.render(),
**context
})
# handles valid URLs to non-CSV data and also just bad URLs
except UnsupportedFormat as e:
err_msg = _(
"Invalid data source. Please make sure you "
"linked to a valid CSV data source."
)
return render(request, 'begin.html', {
"errors": err_msg,
**context
})
except ConnectionError as e:
err_msg = _(
"Invalid URL. Please make sure there aren't "
"typos in the URL, and that the data isn't "
"protected. If you're trying to use a protected "
"Google Sheet, you need to use the private Sheet "
"authenticator, below."
)
return render(request, 'begin.html', {
"errors": err_msg,
**context
})
return redirect('csv_models:refine-and-import', dynmodel.id)