def upload(request): """Handle upload POST calls from UF.""" if request.method != 'POST': return HttpResponseNotAllowed('No GET allowed') api_key = request.GET.get('api_key') username = request.GET.get('username') config_entity_id = request.GET.get('config_entity__id') file_name = request.GET.get('file_name') progress_id = request.GET.get('X-Progress-ID') user = get_user_model().objects.get(api_key__key=api_key, username=username) config_entity = ConfigEntity.objects.filter( id=config_entity_id).get_subclass() upload_task = create_upload_task( user, file_name, config_entity, extra_data_dict={'X-Progress-ID': progress_id}) upload_handler = UploadProgressTaskHandler(max_progress=40, upload_task=upload_task) request.upload_handlers.insert(0, upload_handler) if request.FILES['files[]'].size > settings.MAX_UPLOAD_SIZE: return HttpResponseBadRequest( "The maximum file size for uploads is {} but the file you selected is {}" .format(filesizeformat(settings.MAX_UPLOAD_SIZE), filesizeformat(request.FILES['files[]'].size))) # send the inital progress of 0% upload_task.send_progress() with tempfile.NamedTemporaryFile(delete=False) as tmpfile: for chunk in request.FILES['files[]'].chunks(): tmpfile.write(chunk) try: process_file(tmpfile.name, should_create_db_entity=True, user=user, config_entity=config_entity, upload_task=upload_task) except: upload_task.send_error() raise return HttpResponse()
def upload(request): """Handle upload POST calls from UF.""" if request.method != 'POST': return HttpResponseNotAllowed('No GET allowed') api_key = request.GET.get('api_key') username = request.GET.get('username') config_entity_id = request.GET.get('config_entity__id') file_name = request.GET.get('file_name') progress_id = request.GET.get('X-Progress-ID') user = get_user_model().objects.get(api_key__key=api_key, username=username) config_entity = ConfigEntity.objects.filter(id=config_entity_id).get_subclass() upload_task = create_upload_task( user, file_name, config_entity, extra_data_dict={'X-Progress-ID': progress_id} ) upload_handler = UploadProgressTaskHandler(max_progress=40, upload_task=upload_task) request.upload_handlers.insert(0, upload_handler) if request.FILES['files[]'].size > settings.MAX_UPLOAD_SIZE: return HttpResponseBadRequest( "The maximum file size for uploads is {} but the file you selected is {}".format( filesizeformat(settings.MAX_UPLOAD_SIZE), filesizeformat(request.FILES['files[]'].size) ) ) # send the inital progress of 0% upload_task.send_progress() with tempfile.NamedTemporaryFile(delete=False) as tmpfile: for chunk in request.FILES['files[]'].chunks(): tmpfile.write(chunk) try: process_file(tmpfile.name, should_create_db_entity=True, user=user, config_entity=config_entity, upload_task=upload_task) except: upload_task.send_error() raise return HttpResponse()
def create_db_entity(pg_dump_fpath, db_entity_key, db_entity_name, table_name, layer_count, user, config_entity, **kwargs): """ Create a DbEntity and all associated layers, etc. The majority of the processing occurs in the post-save methods on DbEntity objects, this method simply gets objects in the necessary state to trigger it. """ logger.debug("Creating DbEntity %s with pg_dump file %s", db_entity_key, pg_dump_fpath) if 'upload_task' in kwargs: upload_task = kwargs['upload_task'] else: # we're calling this from the command line # for testing purposes upload_task = create_upload_task( user, pg_dump_fpath, config_entity, extra_data_dict={'X-Progress-ID': 'unused'}) # later post-save processes expect a zipped sql file zipped_sql_fpath = "{}.zip".format(pg_dump_fpath) with ZipFile(zipped_sql_fpath, 'w') as zipped_sql: zipped_sql.write(pg_dump_fpath) # the UploadDataset represents the processing of a single file upload_dataset_task = UploadDatasetTask.objects.create( upload_task=upload_task, dataset_id=-1, file_path=zipped_sql_fpath, filename=db_entity_key, progress=upload_task.progress, status=upload_task.status, extra=upload_task.extra) # the schema metadata, has information necessary for Django to create # new data models based on the upload. The DbEntity post-save # logic uses this. schema_metadata = get_schema_metadata(pg_dump_fpath, table_name) upload_dataset_task.metadata = schema_metadata logger.debug("Saving DbEntity %s and inititialzing post-save processing.", db_entity_key) upload_dataset_task.save() db_entity = DbEntity(creator=user, updater=user, name=db_entity_name, key=db_entity_key, url='file://{}'.format(zipped_sql_fpath), setup_percent_complete=0, schema=config_entity.schema()) # setting `_config_entity` and then calling `save()` triggers # the post-save processing flow, which, among other things, # loads the data into the database, creates layers and runs # required updates to other model objects to be aware of this # layer. db_entity._config_entity = config_entity db_entity.save() db_entity.categories.add( Category.objects.get(key=DbEntityCategoryKey.KEY_CLASSIFICATION, value=DbEntityCategoryKey.REFERENCE)) upload_dataset_task.progress = 100 upload_dataset_task.status = UploadDatasetTask.SUCCESS upload_dataset_task.ended_on = timezone.now() upload_dataset_task.save() upload_dataset_task.send_progress() finished_dataset_count = UploadDatasetTask.objects.filter( upload_task=upload_task, status=UploadDatasetTask.SUCCESS).count() if finished_dataset_count == layer_count: upload_task.progress = 100 upload_task.status = UploadTask.SUCCESS upload_task.ended_on = timezone.now() upload_task.save() upload_task.send_progress() message_kwargs = dict(event="doCreateDbEntity", id=db_entity.id, name=db_entity_name, key=db_entity_key, config_entity=config_entity.id, file_dataset=upload_dataset_task.id) # send websockets `doCreateDbEntity` signal to the browser send_message_to_client(user.id, message_kwargs)
def create_db_entity(pg_dump_fpath, db_entity_key, db_entity_name, table_name, layer_count, user, config_entity, **kwargs): """ Create a DbEntity and all associated layers, etc. The majority of the processing occurs in the post-save methods on DbEntity objects, this method simply gets objects in the necessary state to trigger it. """ logger.debug("Creating DbEntity %s with pg_dump file %s", db_entity_key, pg_dump_fpath) if 'upload_task' in kwargs: upload_task = kwargs['upload_task'] else: # we're calling this from the command line # for testing purposes upload_task = create_upload_task( user, pg_dump_fpath, config_entity, extra_data_dict={'X-Progress-ID': 'unused'} ) # later post-save processes expect a zipped sql file zipped_sql_fpath = "{}.zip".format(pg_dump_fpath) with ZipFile(zipped_sql_fpath, 'w') as zipped_sql: zipped_sql.write(pg_dump_fpath) # the UploadDataset represents the processing of a single file upload_dataset_task = UploadDatasetTask.objects.create( upload_task=upload_task, dataset_id=-1, file_path=zipped_sql_fpath, filename=db_entity_key, progress=upload_task.progress, status=upload_task.status, extra=upload_task.extra ) # the schema metadata, has information necessary for Django to create # new data models based on the upload. The DbEntity post-save # logic uses this. schema_metadata = get_schema_metadata(pg_dump_fpath, table_name) upload_dataset_task.metadata = schema_metadata logger.debug("Saving DbEntity %s and inititialzing post-save processing.", db_entity_key) upload_dataset_task.save() db_entity = DbEntity( creator=user, updater=user, name=db_entity_name, key=db_entity_key, url='file://{}'.format(zipped_sql_fpath), setup_percent_complete=0, schema=config_entity.schema() ) # setting `_config_entity` and then calling `save()` triggers # the post-save processing flow, which, among other things, # loads the data into the database, creates layers and runs # required updates to other model objects to be aware of this # layer. db_entity._config_entity = config_entity db_entity.save() db_entity.categories.add( Category.objects.get( key=DbEntityCategoryKey.KEY_CLASSIFICATION, value=DbEntityCategoryKey.REFERENCE ) ) upload_dataset_task.progress = 100 upload_dataset_task.status = UploadDatasetTask.SUCCESS upload_dataset_task.ended_on = timezone.now() upload_dataset_task.save() upload_dataset_task.send_progress() finished_dataset_count = UploadDatasetTask.objects.filter(upload_task=upload_task, status=UploadDatasetTask.SUCCESS).count() if finished_dataset_count == layer_count: upload_task.progress = 100 upload_task.status = UploadTask.SUCCESS upload_task.ended_on = timezone.now() upload_task.save() upload_task.send_progress() message_kwargs = dict( event="doCreateDbEntity", id=db_entity.id, name=db_entity_name, key=db_entity_key, config_entity=config_entity.id, file_dataset=upload_dataset_task.id ) # send websockets `doCreateDbEntity` signal to the browser send_message_to_client( user.id, message_kwargs )