def import_data(self): """ Perform a (re)import on a previously loaded model. This takes the loaded columns into account, ignoring any new columns that may exist in the spreadsheet. """ if not self.columns: logger.warn("Attempted to import source without columns. Baililng") return [ "Data source hasn't been configured. Re-import this source " \ "using the confiruation wizard." ] csv = None if self.csv_url and self.csv_google_refresh_token: oauther = GoogleOAuth(get_setting("GOOGLE_CLIENT_ID"), get_setting("GOOGLE_CLIENT_SECRET")) access_data = oauther.get_access_data( refresh_token=self.csv_google_refresh_token) token = access_data["access_token"] csv = PrivateSheetImporter(token).get_csv_from_url(self.csv_url) elif self.csv_url: csv = fetch_csv(self.csv_url) elif self.sd_api_key: importer = ScreendoorImporter(api_key=self.sd_api_key) csv = importer.build_csv(self.sd_project_id, form_id=self.sd_form_id) else: raise NotImplementedError("Invalid data source for %s" % self) return import_records(csv, self.get_model(), self)
def run_inspectdb(table_name=None): """ Run inspectdb against our secondary in-memory database. Basically, we're running the equivalent of this manage.py command: ./manage.py inspectdb --database schemabuilding Returns the generated models.py """ cmd = inspectdb.Command() options = { 'verbosity': 1, 'settings': None, 'pythonpath': None, 'traceback': False, 'no_color': False, 'force_color': False, 'table': [table_name] or [], 'database': get_setting("CSV_MODELS_TEMP_DB"), 'include_partitions': False, 'include_views': False } # This command returns a generator of models.py text lines gen = cmd.handle_inspection(options) return "\n".join(list(gen))
def refine_and_import(request, id): """ Allow the user to modify the auto-generated column types and names. This is done before we import the dynmodel data. If this succeeds, we do some preliminary checks against the CSV file to make sure there aren't duplicate headers/etc. Then we do the import. On success, this redirects to the URL specified by the CSV_MODELS_WIZARD_REDIRECT_TO setting if it exists. """ dynmodel = get_object_or_404(models.DynamicModel, id=id) if request.method == "GET": refine_form = SchemaRefineForm({ "columns": dynmodel.columns }) return render(request, 'refine-and-import.html', { "form": refine_form, "dynmodel": dynmodel, }) elif request.method == "POST": refine_form = SchemaRefineForm(request.POST) if not refine_form.is_valid(): return render(request, 'refine-and-import.html', { "form": refine_form, "dynmodel": dynmodel, }) columns = refine_form.cleaned_data["columns"] dynmodel.columns = columns # Alter the DB dynmodel.save() dynmodel.refresh_from_db() errors = dynmodel.import_data() if errors: logger.error("Import errors: %s" % errors) return render(request, 'refine-and-import.html', { "form": refine_form, "dynmodel": dynmodel, "errors": errors, }) # this will re-run the admin setup and build up # the related fields properly logger.info("Doing post-import, re-setup save for admin...") dynmodel.save() next = get_setting("CSV_MODELS_WIZARD_REDIRECT_TO") if next: return redirect(next) return render(request, "import-complete.html", { "dynmodel": dynmodel, "n_records": Model.objects.count(), })
def from_private_sheet(name, sheet_url, auth_code=None, refresh_token=None): """ Build a model from a private Google Sheet, given an OAuth auth code or refresh token, private sheet URL and name. This, of course, assumes the user has already gone through the Google Auth flow and explicitly granted Sheets view access. """ GOOGLE_CLIENT_ID = get_setting("GOOGLE_CLIENT_ID") GOOGLE_CLIENT_SECRET = get_setting("GOOGLE_CLIENT_SECRET") oauther = GoogleOAuth(GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET) access_data = oauther.get_access_data(code=auth_code, refresh_token=refresh_token) token = access_data["access_token"] csv = PrivateSheetImporter(token).get_csv_from_url(sheet_url) return from_csv( name, csv, **dict( csv_url=sheet_url, csv_google_refresh_token=refresh_token or access_data["refresh_token"], ))
def test_can_refine_existing_model(self, fetch_csv): fetch_csv.return_value = CSV to_url = reverse( 'csv_models:refine-and-import', args=[self.dynmodel.id] ) to_url += "?next=/next/url" self.assertEqual(self.dynmodel.columns[0]["type"], "text") self.columns[0]["type"] = "datetime" response = self.client.post(to_url, { "columns": json.dumps(self.columns), }) next = get_setting("CSV_MODELS_WIZARD_REDIRECT_TO") if next: self.assertEqual(response.status_code, 302) self.assertTrue(response.url.startswith(next)) else: self.assertEqual(response.status_code, 200) # make sure the field was actually updated self.dynmodel.refresh_from_db() self.assertEqual(self.dynmodel.columns[0]["type"], "datetime")
def extract_fields(models_py): """ Take a models.py string and extract the lines that declare model fields into a dictionary of the following format: { "FIELD_NAME": "models.FieldType(arguments...)" } """ MAX_HSIZE = get_setting("CSV_MODELS_MAX_HEADER_LENGTH", 40) fields = {} dedupe_fields = {} for line in models_py.split("\n"): line = line.strip() if not line: logger.debug("Skipping blank line") continue # skip imports if re.match(".*from.*import.*", line): logger.debug("Skipping import line: %s" % line) continue # skip class declarations if re.match(r".*class\s+.*models\.Model.*", line): logger.debug("Skipping class declaration for line: %s" % line) continue # extract field name field_matches = re.match( r"\s*([^\s\-=\(\)]+)\s*=\s*(models\.[A-Za-z0-9]+\(.*\))$", line) if not field_matches or not field_matches.groups(): logger.debug("No matches for line: %s" % line) continue field, declaration = field_matches.groups() logger.debug("field: %s, declaration: %s" % (field, declaration)) # NOTE: we don't want to override the datatype of a provided id # or primary key field. django will auto-create this with the # correct type and we can use it accordingly as the primary # field type, unique source of truth. if field in ["id", "pk"]: logger.debug("Skipping id or pk field: %s" % declaration) continue # shorten field names if len(field) > MAX_HSIZE: field = field[:MAX_HSIZE] # fields cannot end w/ underscore while field[-1] == "_": field = field[:-1] # Dedupe the field names. second dupe gets _2 added, etc postfix_str = "" if field in dedupe_fields: dedupe_fields[field] += 1 postfix_str = "_%s" % dedupe_fields[field] else: dedupe_fields[field] = 1 # append dedupe tring if necessary if postfix_str: field += postfix_str fields[field] = declaration return fields
UniqueColumnError, DataSourceExistsError, NoPrivateSheetCredentialsError, ) from django_models_from_csv.models import DynamicModel from django_models_from_csv.utils.common import get_setting, slugify from django_models_from_csv.utils.csv import fetch_csv, clean_csv_headers from django_models_from_csv.utils.models_py import ( fix_models_py, extract_field_declaration_args_eval, extract_field_type, extract_fields) from django_models_from_csv.utils.screendoor import ScreendoorImporter from django_models_from_csv.utils.google_sheets import PrivateSheetImporter logger = logging.getLogger(__name__) CSV_MODELS_TEMP_DB = get_setting("CSV_MODELS_TEMP_DB") @transaction.atomic(using=CSV_MODELS_TEMP_DB) def execute_sql(sql, params=None): """ Run a SQL query against our secondary in-memory database. Returns the table name, if this was a CREATE TABLE command, otherwise just returns None. """ conn = connections[CSV_MODELS_TEMP_DB] cursor = conn.cursor() if params is None: cursor.execute(sql) else: cursor.execute(sql, params)
def refine_and_import(request, id): """ Allow the user to modify the auto-generated column types and names. This is done before we import the dynmodel data. If this succeeds, we do some preliminary checks against the CSV file to make sure there aren't duplicate headers/etc. Then we do the import. On success, this redirects to the URL specified by the CSV_MODELS_WIZARD_REDIRECT_TO setting if it exists. """ dynmodel = get_object_or_404(models.DynamicModel, id=id) if request.method == "GET": refine_form = SchemaRefineForm({"columns": dynmodel.columns}) return render(request, 'refine-and-import.html', { "form": refine_form, "dynmodel": dynmodel, }) elif request.method == "POST": refine_form = SchemaRefineForm(request.POST) if not refine_form.is_valid(): return render(request, 'refine-and-import.html', { "form": refine_form, "dynmodel": dynmodel, }) columns = refine_form.cleaned_data["columns"] dynmodel.columns = columns errors = None try: max_import_records = None if hasattr(settings, "MAX_IMPORT_RECORDS"): max_import_records = settings.MAX_IMPORT_RECORDS # Alter the DB dynmodel.save() dynmodel.refresh_from_db() errors = dynmodel.import_data( max_import_records=max_import_records, ) except Exception as e: if not isinstance(e, GenericCSVError): raise e # if we have one of these errors, it's most likely # due to a change in the data source (credentials # revoked, URL unshared, etc) return render( request, 'refine-and-import.html', { "error_message": e.render(), "form": refine_form, "dynmodel": dynmodel, }) if errors: logger.error("Import errors: %s" % errors) return render(request, 'refine-and-import.html', { "form": refine_form, "dynmodel": dynmodel, "errors": errors, }) # this will re-run the admin setup and build up # the related fields properly logger.info("Doing post-import, re-setup save for admin...") dynmodel.save() next = get_setting("CSV_MODELS_WIZARD_REDIRECT_TO") if next: return redirect(next) return render(request, "import-complete.html", { "dynmodel": dynmodel, "n_records": Model.objects.count(), })
inheritance, { # "form": create_taggable_form(Model), "resource_class": resource, # "fields": fields, # "readonly_fields": ro_fields, }) def should_register_admin(self, Model): return self.include in str(Model) def register(self): try: create_models() except Exception: pass for Model in apps.get_models(): if not self.should_register_admin(Model): continue ModelAdmin = self.create_admin(Model) self.attempt_register(Model, ModelAdmin) if get_setting("CSV_MODELS_AUTO_REGISTER_ADMIN", True): # Auto-register Admins for all dymamic models. This avoids needing to # write code and inject it into an admin.py for our auto-generated # models auto_reg = AdminAutoRegistration(include="django_models_from_csv.models") auto_reg.register()