def import_data(self):
"""
Perform a (re)import on a previously loaded model. This takes
the loaded columns into account, ignoring any new columns that
may exist in the spreadsheet.
"""
if not self.columns:
logger.warn("Attempted to import source without columns. Baililng")
return [
"Data source hasn't been configured. Re-import this source " \
"using the confiruation wizard."
]
csv = None
if self.csv_url and self.csv_google_refresh_token:
oauther = GoogleOAuth(get_setting("GOOGLE_CLIENT_ID"),
get_setting("GOOGLE_CLIENT_SECRET"))
access_data = oauther.get_access_data(
refresh_token=self.csv_google_refresh_token)
token = access_data["access_token"]
csv = PrivateSheetImporter(token).get_csv_from_url(self.csv_url)
elif self.csv_url:
csv = fetch_csv(self.csv_url)
elif self.sd_api_key:
importer = ScreendoorImporter(api_key=self.sd_api_key)
csv = importer.build_csv(self.sd_project_id,
form_id=self.sd_form_id)
else:
raise NotImplementedError("Invalid data source for %s" % self)
return import_records(csv, self.get_model(), self)
def run_inspectdb(table_name=None):
"""
Run inspectdb against our secondary in-memory database. Basically,
we're running the equivalent of this manage.py command:
./manage.py inspectdb --database schemabuilding
Returns the generated models.py
"""
cmd = inspectdb.Command()
options = {
'verbosity': 1,
'settings': None,
'pythonpath': None,
'traceback': False,
'no_color': False,
'force_color': False,
'table': [table_name] or [],
'database': get_setting("CSV_MODELS_TEMP_DB"),
'include_partitions': False,
'include_views': False
}
# This command returns a generator of models.py text lines
gen = cmd.handle_inspection(options)
return "\n".join(list(gen))
def refine_and_import(request, id):
"""
Allow the user to modify the auto-generated column types and
names. This is done before we import the dynmodel data.
If this succeeds, we do some preliminary checks against the
CSV file to make sure there aren't duplicate headers/etc.
Then we do the import. On success, this redirects to the URL
specified by the CSV_MODELS_WIZARD_REDIRECT_TO setting if
it exists.
"""
dynmodel = get_object_or_404(models.DynamicModel, id=id)
if request.method == "GET":
refine_form = SchemaRefineForm({
"columns": dynmodel.columns
})
return render(request, 'refine-and-import.html', {
"form": refine_form,
"dynmodel": dynmodel,
})
elif request.method == "POST":
refine_form = SchemaRefineForm(request.POST)
if not refine_form.is_valid():
return render(request, 'refine-and-import.html', {
"form": refine_form,
"dynmodel": dynmodel,
})
columns = refine_form.cleaned_data["columns"]
dynmodel.columns = columns
# Alter the DB
dynmodel.save()
dynmodel.refresh_from_db()
errors = dynmodel.import_data()
if errors:
logger.error("Import errors: %s" % errors)
return render(request, 'refine-and-import.html', {
"form": refine_form,
"dynmodel": dynmodel,
"errors": errors,
})
# this will re-run the admin setup and build up
# the related fields properly
logger.info("Doing post-import, re-setup save for admin...")
dynmodel.save()
next = get_setting("CSV_MODELS_WIZARD_REDIRECT_TO")
if next:
return redirect(next)
return render(request, "import-complete.html", {
"dynmodel": dynmodel,
"n_records": Model.objects.count(),
})
def from_private_sheet(name, sheet_url, auth_code=None, refresh_token=None):
"""
Build a model from a private Google Sheet, given an OAuth auth code
or refresh token, private sheet URL and name. This, of course,
assumes the user has already gone through the Google Auth flow and
explicitly granted Sheets view access.
"""
GOOGLE_CLIENT_ID = get_setting("GOOGLE_CLIENT_ID")
GOOGLE_CLIENT_SECRET = get_setting("GOOGLE_CLIENT_SECRET")
oauther = GoogleOAuth(GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET)
access_data = oauther.get_access_data(code=auth_code,
refresh_token=refresh_token)
token = access_data["access_token"]
csv = PrivateSheetImporter(token).get_csv_from_url(sheet_url)
return from_csv(
name, csv,
**dict(
csv_url=sheet_url,
csv_google_refresh_token=refresh_token
or access_data["refresh_token"],
))
def test_can_refine_existing_model(self, fetch_csv):
fetch_csv.return_value = CSV
to_url = reverse(
'csv_models:refine-and-import', args=[self.dynmodel.id]
)
to_url += "?next=/next/url"
self.assertEqual(self.dynmodel.columns[0]["type"], "text")
self.columns[0]["type"] = "datetime"
response = self.client.post(to_url, {
"columns": json.dumps(self.columns),
})
next = get_setting("CSV_MODELS_WIZARD_REDIRECT_TO")
if next:
self.assertEqual(response.status_code, 302)
self.assertTrue(response.url.startswith(next))
else:
self.assertEqual(response.status_code, 200)
# make sure the field was actually updated
self.dynmodel.refresh_from_db()
self.assertEqual(self.dynmodel.columns[0]["type"], "datetime")
def extract_fields(models_py):
"""
Take a models.py string and extract the lines that
declare model fields into a dictionary of the following
format:
{ "FIELD_NAME": "models.FieldType(arguments...)" }
"""
MAX_HSIZE = get_setting("CSV_MODELS_MAX_HEADER_LENGTH", 40)
fields = {}
dedupe_fields = {}
for line in models_py.split("\n"):
line = line.strip()
if not line:
logger.debug("Skipping blank line")
continue
# skip imports
if re.match(".*from.*import.*", line):
logger.debug("Skipping import line: %s" % line)
continue
# skip class declarations
if re.match(r".*class\s+.*models\.Model.*", line):
logger.debug("Skipping class declaration for line: %s" % line)
continue
# extract field name
field_matches = re.match(
r"\s*([^\s\-=\(\)]+)\s*=\s*(models\.[A-Za-z0-9]+\(.*\))$", line)
if not field_matches or not field_matches.groups():
logger.debug("No matches for line: %s" % line)
continue
field, declaration = field_matches.groups()
logger.debug("field: %s, declaration: %s" % (field, declaration))
# NOTE: we don't want to override the datatype of a provided id
# or primary key field. django will auto-create this with the
# correct type and we can use it accordingly as the primary
# field type, unique source of truth.
if field in ["id", "pk"]:
logger.debug("Skipping id or pk field: %s" % declaration)
continue
# shorten field names
if len(field) > MAX_HSIZE:
field = field[:MAX_HSIZE]
# fields cannot end w/ underscore
while field[-1] == "_":
field = field[:-1]
# Dedupe the field names. second dupe gets _2 added, etc
postfix_str = ""
if field in dedupe_fields:
dedupe_fields[field] += 1
postfix_str = "_%s" % dedupe_fields[field]
else:
dedupe_fields[field] = 1
# append dedupe tring if necessary
if postfix_str:
field += postfix_str
fields[field] = declaration
return fields
UniqueColumnError,
DataSourceExistsError,
NoPrivateSheetCredentialsError,
)
from django_models_from_csv.models import DynamicModel
from django_models_from_csv.utils.common import get_setting, slugify
from django_models_from_csv.utils.csv import fetch_csv, clean_csv_headers
from django_models_from_csv.utils.models_py import (
fix_models_py, extract_field_declaration_args_eval, extract_field_type,
extract_fields)
from django_models_from_csv.utils.screendoor import ScreendoorImporter
from django_models_from_csv.utils.google_sheets import PrivateSheetImporter
logger = logging.getLogger(__name__)
CSV_MODELS_TEMP_DB = get_setting("CSV_MODELS_TEMP_DB")
@transaction.atomic(using=CSV_MODELS_TEMP_DB)
def execute_sql(sql, params=None):
"""
Run a SQL query against our secondary in-memory database. Returns
the table name, if this was a CREATE TABLE command, otherwise just
returns None.
"""
conn = connections[CSV_MODELS_TEMP_DB]
cursor = conn.cursor()
if params is None:
cursor.execute(sql)
else:
cursor.execute(sql, params)
def refine_and_import(request, id):
"""
Allow the user to modify the auto-generated column types and
names. This is done before we import the dynmodel data.
If this succeeds, we do some preliminary checks against the
CSV file to make sure there aren't duplicate headers/etc.
Then we do the import. On success, this redirects to the URL
specified by the CSV_MODELS_WIZARD_REDIRECT_TO setting if
it exists.
"""
dynmodel = get_object_or_404(models.DynamicModel, id=id)
if request.method == "GET":
refine_form = SchemaRefineForm({"columns": dynmodel.columns})
return render(request, 'refine-and-import.html', {
"form": refine_form,
"dynmodel": dynmodel,
})
elif request.method == "POST":
refine_form = SchemaRefineForm(request.POST)
if not refine_form.is_valid():
return render(request, 'refine-and-import.html', {
"form": refine_form,
"dynmodel": dynmodel,
})
columns = refine_form.cleaned_data["columns"]
dynmodel.columns = columns
errors = None
try:
max_import_records = None
if hasattr(settings, "MAX_IMPORT_RECORDS"):
max_import_records = settings.MAX_IMPORT_RECORDS
# Alter the DB
dynmodel.save()
dynmodel.refresh_from_db()
errors = dynmodel.import_data(
max_import_records=max_import_records, )
except Exception as e:
if not isinstance(e, GenericCSVError):
raise e
# if we have one of these errors, it's most likely
# due to a change in the data source (credentials
# revoked, URL unshared, etc)
return render(
request, 'refine-and-import.html', {
"error_message": e.render(),
"form": refine_form,
"dynmodel": dynmodel,
})
if errors:
logger.error("Import errors: %s" % errors)
return render(request, 'refine-and-import.html', {
"form": refine_form,
"dynmodel": dynmodel,
"errors": errors,
})
# this will re-run the admin setup and build up
# the related fields properly
logger.info("Doing post-import, re-setup save for admin...")
dynmodel.save()
next = get_setting("CSV_MODELS_WIZARD_REDIRECT_TO")
if next:
return redirect(next)
return render(request, "import-complete.html", {
"dynmodel": dynmodel,
"n_records": Model.objects.count(),
})
inheritance,
{
# "form": create_taggable_form(Model),
"resource_class": resource,
# "fields": fields,
# "readonly_fields": ro_fields,
})
def should_register_admin(self, Model):
return self.include in str(Model)
def register(self):
try:
create_models()
except Exception:
pass
for Model in apps.get_models():
if not self.should_register_admin(Model):
continue
ModelAdmin = self.create_admin(Model)
self.attempt_register(Model, ModelAdmin)
if get_setting("CSV_MODELS_AUTO_REGISTER_ADMIN", True):
# Auto-register Admins for all dymamic models. This avoids needing to
# write code and inject it into an admin.py for our auto-generated
# models
auto_reg = AdminAutoRegistration(include="django_models_from_csv.models")
auto_reg.register()