def _export_query_results(job, class_name, unique_id, filename, query, event='queryResultExportCompleted'):
try:
job.status = "Exporting"
job.save()
# add timestamp to filename to prevent conflicts
filename = filename + "_" + timestamp()
result = OGRETL().export(filename, query, 'CSV', geom=None)
job.data = '/' + filename + ".csv"
job.save()
send_message_to_client(job.user.id, dict(event=event,
class_name=class_name,
unique_id=unique_id,
job_id=str(job.hashid)))
job.status = 'Complete'
except Exception, e:
logger.error(e)
job.status = "Failed"
exc_type, exc_value, exc_traceback = sys.exc_info()
readable_exception = traceback.format_exception(exc_type, exc_value, exc_traceback)
job.data = readable_exception
job.save()
send_message_to_client(job.user.id, dict(event=job.type + " failed", trace=readable_exception))
def send_fail_message(config_entity, bundle, job, publisher_name, signal_proportion_lookup_name, user):
job.status = "Failed"
exc_type, exc_value, exc_traceback = sys.exc_info()
readable_exception = traceback.format_exception(exc_type, exc_value, exc_traceback)
job.data = readable_exception
event = 'postSavePublisherFailed'
logger.error("Sending Failed message %s for signal %s to user %s with %s readable_exception %s" % \
(event, signal_proportion_lookup_name, user.username, unicode(bundle), readable_exception))
send_message_to_client(user.id, dict(
event=event,
config_entity_id=config_entity and config_entity.id,
config_entity_class_name=config_entity and config_entity.__class__.__name__,
# Send the key since the id of new instances might be meaningless to the client
# If it hasn't updated the record's id yet
class_name=bundle.class_name_for_client,
ids=bundle.client_instance_ids,
keys=bundle.keys,
# Used for Features and other things that have a class scope key
class_key=bundle.class_key,
publisher_name=publisher_name,
# The client can display a capitalized version of this to describe the progress
progress_description=humanize(signal_proportion_lookup_name),
trace=readable_exception)
)
return readable_exception
def send_error(self):
"""
Send error information to frontend via websockets
"""
kwargs = self.send_error_kwargs()
logger.error("Error UploadHandler: Sending error to client: %s" % kwargs)
send_message_to_client(self.user.id, kwargs)
def _export_layer(job, layer_id):
try:
layer = Layer.objects.get(id=layer_id)
job.status = "Exporting"
job.save()
db_entity = layer.db_entity_interest.db_entity
geometry_type = layer.medium_subclassed.geometry_type
export_file, filename = export_db_entity_to_file(db_entity, geometry_type)
job.status = "Zipping"
job.save()
zip_file_gdb(export_file)
shutil.rmtree(export_file)
job.data = '/' + filename + ".zip"
job.save()
send_message_to_client(job.user.id, dict(event='layerExportCompleted',
layer_id=layer_id,
job_id=str(job.hashid)))
job.status = 'Complete'
except Exception, e:
job.status = "Failed"
exc_type, exc_value, exc_traceback = sys.exc_info()
readable_exception = traceback.format_exception(exc_type, exc_value, exc_traceback)
job.data = readable_exception
job.save()
send_message_to_client(job.user.id, dict(event=job.type + " failed", trace=readable_exception))
def _export_query_results(job, class_name, unique_id, filename, query, event='queryResultExportCompleted'):
try:
job.status = "Exporting"
job.save()
# add timestamp to filename to prevent conflicts
filename = filename + "_" + timestamp()
result = OGRETL().export(filename, query, 'CSV', geom=None)
job.data = '/' + filename + ".csv"
job.save()
send_message_to_client(job.user.id, dict(event=event,
class_name=class_name,
unique_id=unique_id,
job_id=str(job.hashid)))
job.status = 'Complete'
except Exception, e:
logger.error(e)
job.status = "Failed"
exc_type, exc_value, exc_traceback = sys.exc_info()
readable_exception = traceback.format_exception(exc_type, exc_value, exc_traceback)
job.data = readable_exception
job.save()
send_message_to_client(job.user.id, dict(event=job.type + " failed", trace=readable_exception))
def send_fail_message(config_entity, bundle, job, publisher_name,
signal_proportion_lookup_name, user):
job.status = "Failed"
exc_type, exc_value, exc_traceback = sys.exc_info()
readable_exception = traceback.format_exception(exc_type, exc_value,
exc_traceback)
job.data = readable_exception
event = 'postSavePublisherFailed'
logger.error("Sending Failed message %s for signal %s to user %s with %s readable_exception %s" % \
(event, signal_proportion_lookup_name, user.username, unicode(bundle), readable_exception))
send_message_to_client(
user.id,
dict(
event=event,
config_entity_id=config_entity and config_entity.id,
config_entity_class_name=config_entity
and config_entity.__class__.__name__,
# Send the key since the id of new instances might be meaningless to the client
# If it hasn't updated the record's id yet
class_name=bundle.class_name_for_client,
ids=bundle.client_instance_ids,
keys=bundle.keys,
# Used for Features and other things that have a class scope key
class_key=bundle.class_key,
publisher_name=publisher_name,
# The client can display a capitalized version of this to describe the progress
progress_description=humanize(signal_proportion_lookup_name),
trace=readable_exception))
return readable_exception
def _export_layer(job, layer_id):
try:
layer = Layer.objects.get(id=layer_id)
job.status = "Exporting"
job.save()
db_entity = layer.db_entity_interest.db_entity
geometry_type = layer.medium_subclassed.geometry_type
export_file, filename = export_db_entity_to_file(db_entity, geometry_type)
job.status = "Zipping"
job.save()
zip_file_gdb(export_file)
shutil.rmtree(export_file)
job.data = '/' + filename + ".zip"
job.save()
send_message_to_client(job.user.id, dict(event='layerExportCompleted',
layer_id=layer_id,
job_id=str(job.hashid)))
job.status = 'Complete'
except Exception, e:
job.status = "Failed"
exc_type, exc_value, exc_traceback = sys.exc_info()
readable_exception = traceback.format_exception(exc_type, exc_value, exc_traceback)
job.data = readable_exception
job.save()
send_message_to_client(job.user.id, dict(event=job.type + " failed", trace=readable_exception))
def send_error(self):
"""
Send error information to frontend via websockets
"""
kwargs = self.send_error_kwargs()
logger.error("Error UploadHandler: Sending error to client: %s" %
kwargs)
send_message_to_client(self.user.id, kwargs)
def progress(self, proportion, **kwargs):
send_message_to_client(
kwargs['user'].id,
dict(event='postSavePublisherProportionCompleted',
job_id=str(kwargs['job'].hashid),
config_entity_id=self.config_entity.id,
ids=[kwargs['analysis_module'].id],
class_name='AnalysisModule',
key=kwargs['analysis_module'].key,
proportion=proportion))
def report_progress(self, proportion, **kwargs):
send_message_to_client(kwargs['user'].id, dict(
event='postSavePublisherProportionCompleted',
job_id=str(kwargs['job'].hashid),
config_entity_id=self.config_entity.id,
ids=[kwargs['analysis_module'].id],
class_name='AnalysisModule',
key=kwargs['analysis_module'].key,
proportion=proportion)
)
logger.info("Progress {0}".format(proportion))
def send_progress(self):
"""
Send progress information to frontend via websockets
"""
self.load_extra()
if 'X-Progress-ID' not in self.extra:
raise Exception('UploadTask must have a X-Progress-ID in'
' order to send messages through websockets')
if self.progress is None:
raise Exception('UploadTask cannot send_progress if progress'
' is None')
kwargs = self.send_progress_kwargs()
logger.debug("UploadHandler: Sending progress: %s" % kwargs)
send_message_to_client(self.user.id, kwargs)
def merge_progress(self, proportion, **kwargs):
send_message_to_client(
kwargs["user"].id,
dict(
event="postSavePublisherProportionCompleted",
job_id=str(kwargs["job"].hashid),
config_entity_id=self.config_entity.id,
ids=[kwargs["analysis_module"].id],
class_name="AnalysisModule",
key=kwargs["analysis_module"].key,
proportion=proportion,
),
)
logger.info("Progress {0}".format(proportion))
def send_progress(self):
"""
Send progress information to frontend via websockets
"""
self.load_extra()
if 'X-Progress-ID' not in self.extra:
raise Exception('UploadTask must have a X-Progress-ID in'
' order to send messages through websockets')
if self.progress is None:
raise Exception('UploadTask cannot send_progress if progress'
' is None')
kwargs = self.send_progress_kwargs()
logger.debug("UploadHandler: Sending progress: %s" % kwargs)
send_message_to_client(self.user.id, kwargs)
def create_db_entity(pg_dump_fpath, db_entity_key, db_entity_name, table_name, layer_count, user, config_entity, **kwargs):
"""
Create a DbEntity and all associated layers, etc. The majority of the
processing occurs in the post-save methods on DbEntity objects, this method
simply gets objects in the necessary state to trigger it.
"""
logger.debug("Creating DbEntity %s with pg_dump file %s", db_entity_key, pg_dump_fpath)
if 'upload_task' in kwargs:
upload_task = kwargs['upload_task']
else:
# we're calling this from the command line
# for testing purposes
upload_task = create_upload_task(
user,
pg_dump_fpath,
config_entity,
extra_data_dict={'X-Progress-ID': 'unused'}
)
# later post-save processes expect a zipped sql file
zipped_sql_fpath = "{}.zip".format(pg_dump_fpath)
with ZipFile(zipped_sql_fpath, 'w') as zipped_sql:
zipped_sql.write(pg_dump_fpath)
# the UploadDataset represents the processing of a single file
upload_dataset_task = UploadDatasetTask.objects.create(
upload_task=upload_task,
dataset_id=-1,
file_path=zipped_sql_fpath,
filename=db_entity_key,
progress=upload_task.progress,
status=upload_task.status,
extra=upload_task.extra
)
# the schema metadata, has information necessary for Django to create
# new data models based on the upload. The DbEntity post-save
# logic uses this.
schema_metadata = get_schema_metadata(pg_dump_fpath, table_name)
upload_dataset_task.metadata = schema_metadata
logger.debug("Saving DbEntity %s and inititialzing post-save processing.", db_entity_key)
upload_dataset_task.save()
db_entity = DbEntity(
creator=user,
updater=user,
name=db_entity_name,
key=db_entity_key,
url='file://{}'.format(zipped_sql_fpath),
setup_percent_complete=0,
schema=config_entity.schema()
)
# setting `_config_entity` and then calling `save()` triggers
# the post-save processing flow, which, among other things,
# loads the data into the database, creates layers and runs
# required updates to other model objects to be aware of this
# layer.
db_entity._config_entity = config_entity
db_entity.save()
db_entity.categories.add(
Category.objects.get(
key=DbEntityCategoryKey.KEY_CLASSIFICATION,
value=DbEntityCategoryKey.REFERENCE
)
)
upload_dataset_task.progress = 100
upload_dataset_task.status = UploadDatasetTask.SUCCESS
upload_dataset_task.ended_on = timezone.now()
upload_dataset_task.save()
upload_dataset_task.send_progress()
finished_dataset_count = UploadDatasetTask.objects.filter(upload_task=upload_task, status=UploadDatasetTask.SUCCESS).count()
if finished_dataset_count == layer_count:
upload_task.progress = 100
upload_task.status = UploadTask.SUCCESS
upload_task.ended_on = timezone.now()
upload_task.save()
upload_task.send_progress()
message_kwargs = dict(
event="doCreateDbEntity",
id=db_entity.id,
name=db_entity_name,
key=db_entity_key,
config_entity=config_entity.id,
file_dataset=upload_dataset_task.id
)
# send websockets `doCreateDbEntity` signal to the browser
send_message_to_client(
user.id,
message_kwargs
)
'setup_percent_complete') + 100 * publishing_info['proportion']
instance.save()
logger.info("Instance percent after %s" %
instance.setup_percent_complete)
send_message_to_client(
user.id,
dict(
event=event,
job_id=str(job.hashid),
config_entity_id=config_entity and config_entity.id,
config_entity_class_name=config_entity
and config_entity.__class__.__name__,
# Send the key since the id of new instances might be meaningless to the client
# If it hasn't updated the record's id yet
publisher_name=publishing_info['publisher_name'],
class_name=bundle.class_name_for_client,
ids=bundle.client_instance_ids,
keys=bundle.keys,
# Used for Features and other things that have a class scope key
class_key=bundle.class_key,
# Send the proportion of work that completing this signal signifies--0 to 1
proportion=publishing_info['proportion'],
# The client can display a this to describe the progress
progress_description=humanize(publishing_info['signal_path']),
))
# Find all dependent signals of this one and run each in parallel
for dependent_signal_path in publishing_info['dependent_signal_paths']:
# Recurse
post_save_publishing(dependent_signal_path, config_entity, user,
def post_save_publishing(signal_path, config_entity, user, **kwargs):
"""
The initial entry point and recursive entry point for all post save publishing methods
:signal_path - the full module path of the signal that called this
:param kwargs:
signal_proportion_lookup - A dictionary of signal names to the proportion complete of the overall post save.
The signal matching signal_path will be sought in the dictionary
config_entity - The scope of whatever being post-saved, whether a config_entity or something within it
dependent_signal_paths - Full module signal paths called in sequentially by this publisher
crud_type - CrudKey.CREATE|CLONE|UPDATE|SYNC|DELETE
instance_class - Optional. Overrides the class of the instance for use in communicating with the client.
This is used when the client only cares about a base class, such as Feature or to for DbEntityInterest
to be a DbEntity
client_instance_path - Optional. Property path to resolve the instance to another instance for the client.
(this is only used to convert DbEntityInterest to DbEntity)
"""
api_key = ApiKey.objects.get(user=user).key
# Gather instance ids, class, and optional instance keys
bundle = InstanceBundle(**merge(kwargs, dict(user_id=user.id)))
# Pass the arguments to the task and run via celery. Note that kwargs is being treated
# as a dict here and passed along
logger.info("Django post save: %s" % unicode(bundle))
# Send the start event to the client if we aren't recursing.
if not kwargs.get('recurse', False):
event = 'postSavePublisherStarted'
logger.info("Sending start message %s to user %s with %s" %
(event, user.username, unicode(bundle)))
send_message_to_client(
user.id,
dict(
event=event,
config_entity_id=config_entity and config_entity.id,
config_entity_class_name=config_entity
and config_entity.__class__.__name__,
class_name=bundle.class_name_for_client,
# Always send 0 for initial
proportion=0,
ids=bundle.client_instance_ids,
keys=bundle.keys,
class_key=bundle.class_key))
# Start Celery
logger.info("Starting post save publishing with signal path %s" %
signal_path)
job = start_and_track_task(
_post_save_publishing,
api_key,
config_entity,
user,
**merge(
remove_keys(kwargs, ['instance']),
dict(
# If we are recursing (already in a celery worker, don't start a new celery task
# When we get dependency order figured out, we can do this, but there's probably
# a better way via the Task object or something
current_job=kwargs.get('job', None),
signal_path=signal_path,
crud_type=kwargs.get('crud_type'),
bundle=bundle)))
return HttpResponse(job.hashid)
def analysis_module_task(job, user, config_entity_id, key, kwargs):
config_entity = ConfigEntity.objects.get(id=config_entity_id).subclassed
analysis_module = AnalysisModule.objects.get(config_entity=config_entity,
key=key)
# Set again for new instance
analysis_module._started = True
try:
# TODO progress calls should be moved to each module so the status bar increments on the client
# logger.info('AnalysisModule %s Started for ConfigEntity %s with kwarg keys' % (analysis_module.name, config_entity.name, ', '.join(kwargs or dict().keys())))
send_message_to_client(
user.id,
dict(event='postSavePublisherStarted'.format(capitalize(key)),
job_id=str(job.hashid),
config_entity_id=config_entity.id,
ids=[analysis_module.id],
class_name='AnalysisModule',
key=analysis_module.key))
# Call each tool's update method
for analysis_tool in analysis_module.analysis_tools.all(
).select_subclasses():
updated_kwargs = deepcopy(kwargs)
updated_kwargs.update(
dict(analysis_module=analysis_module,
user=user,
job=job,
key=key))
analysis_tool.update(**updated_kwargs)
# Call the post save publisher
from footprint.main.publishing.config_entity_publishing import post_save_config_entity_analysis_module
post_save_config_entity_analysis_module.send(
sender=config_entity.__class__,
config_entity=config_entity,
analysis_module=analysis_module)
logger.info('AnalysisModule %s Completed for ConfigEntity %s' %
(analysis_module.name, config_entity.name))
logger.info('Sending message to client postSavePublisherCompleted to user %s for module %s and config entity %s' % \
(user.username, analysis_module.name, config_entity.name))
send_message_to_client(
user.id,
dict(event='postSavePublisherCompleted',
job_id=str(job.hashid),
config_entity_id=config_entity.id,
ids=[analysis_module.id],
class_name='AnalysisModule',
key=analysis_module.key))
analysis_module.completed = datetime.utcnow().replace(tzinfo=utc)
analysis_module.save()
analysis_module._started = False
except Exception, e:
try:
analysis_module.failed = datetime.utcnow().replace(tzinfo=utc)
analysis_module.save()
finally:
analysis_module._started = False
exc_type, exc_value, exc_traceback = sys.exc_info()
readable_exception = traceback.format_exception(
exc_type, exc_value, exc_traceback)
logger.error(readable_exception)
send_message_to_client(
user.id,
dict(event='postSavePublisherFailed',
job_id=str(job.hashid),
config_entity_id=config_entity.id,
ids=[analysis_module.id],
class_name='AnalysisModule',
key=analysis_module.key))
raise Exception(readable_exception)
def post_save_publishing(signal_path, config_entity, user, **kwargs):
"""
The initial entry point and recursive entry point for all post save publishing methods
:signal_path - the full module path of the signal that called this
:param kwargs:
signal_proportion_lookup - A dictionary of signal names to the proportion complete of the overall post save.
The signal matching signal_path will be sought in the dictionary
config_entity - The scope of whatever being post-saved, whether a config_entity or something within it
dependent_signal_paths - Full module signal paths called in sequentially by this publisher
crud_type - CrudKey.CREATE|CLONE|UPDATE|SYNC|DELETE
instance_class - Optional. Overrides the class of the instance for use in communicating with the client.
This is used when the client only cares about a base class, such as Feature or to for DbEntityInterest
to be a DbEntity
client_instance_path - Optional. Property path to resolve the instance to another instance for the client.
(this is only used to convert DbEntityInterest to DbEntity)
"""
api_key = ApiKey.objects.get(user=user).key
# Gather instance ids, class, and optional instance keys
bundle = InstanceBundle(**merge(kwargs, dict(user_id=user.id)))
# Pass the arguments to the task and run via celery. Note that kwargs is being treated
# as a dict here and passed along
logger.info("Django post save: %s" % unicode(bundle))
# Send the start event to the client if we aren't recursing.
if not kwargs.get('recurse', False):
event = 'postSavePublisherStarted'
logger.info("Sending start message %s to user %s with %s" % (event, user.username, unicode(bundle)))
send_message_to_client(
user.id,
dict(
event=event,
config_entity_id=config_entity and config_entity.id,
config_entity_class_name=config_entity and config_entity.__class__.__name__,
class_name=bundle.class_name_for_client,
# Always send 0 for initial
proportion=0,
ids=bundle.client_instance_ids,
keys=bundle.keys,
class_key=bundle.class_key
)
)
# Start Celery
logger.info("Starting post save publishing with signal path %s" % signal_path)
job = start_and_track_task(_post_save_publishing,
api_key,
config_entity,
user,
**merge(
remove_keys(kwargs, ['instance']),
dict(
# If we are recursing (already in a celery worker, don't start a new celery task
# When we get dependency order figured out, we can do this, but there's probably
# a better way via the Task object or something
current_job=kwargs.get('job', None),
signal_path=signal_path,
crud_type=kwargs.get('crud_type'),
bundle=bundle
)))
return HttpResponse(job.hashid)
def analysis_module_task(job, user, config_entity_id, key, kwargs):
config_entity = ConfigEntity.objects.get(id=config_entity_id).subclassed
analysis_module = AnalysisModule.objects.get(config_entity=config_entity, key=key)
# Set again for new instance
analysis_module._started = True
try:
# TODO progress calls should be moved to each module so the status bar increments on the client
# logger.info('AnalysisModule %s Started for ConfigEntity %s with kwarg keys' % (analysis_module.name, config_entity.name, ', '.join(kwargs or dict().keys())))
send_message_to_client(user.id, dict(
event='postSavePublisherStarted'.format(capitalize(key)),
job_id=str(job.hashid),
config_entity_id=config_entity.id,
ids=[analysis_module.id],
class_name='AnalysisModule',
key=analysis_module.key))
# Call each tool's update method
for analysis_tool in analysis_module.analysis_tools.all().select_subclasses():
updated_kwargs = deepcopy(kwargs)
updated_kwargs.update(dict(analysis_module=analysis_module, user=user, job=job, key=key))
analysis_tool.update(**updated_kwargs)
# Call the post save publisher
from footprint.main.publishing.config_entity_publishing import post_save_config_entity_analysis_module
post_save_config_entity_analysis_module.send(sender=config_entity.__class__, config_entity=config_entity, analysis_module=analysis_module)
logger.info('AnalysisModule %s Completed for ConfigEntity %s' % (analysis_module.name, config_entity.name))
logger.info('Sending message to client postSavePublisherCompleted to user %s for module %s and config entity %s' % \
(user.username, analysis_module.name, config_entity.name))
send_message_to_client(user.id,
dict(event='postSavePublisherCompleted',
job_id=str(job.hashid),
config_entity_id=config_entity.id,
ids=[analysis_module.id],
class_name='AnalysisModule',
key=analysis_module.key)
)
analysis_module.completed = datetime.utcnow().replace(tzinfo=utc)
analysis_module.save()
analysis_module._started = False
except Exception, e:
try:
analysis_module.failed = datetime.utcnow().replace(tzinfo=utc)
analysis_module.save()
finally:
analysis_module._started = False
exc_type, exc_value, exc_traceback = sys.exc_info()
readable_exception = traceback.format_exception(exc_type, exc_value, exc_traceback)
logger.error(readable_exception)
send_message_to_client(user.id,
dict(event='postSavePublisherFailed',
job_id=str(job.hashid),
config_entity_id=config_entity.id,
ids=[analysis_module.id],
class_name='AnalysisModule',
key=analysis_module.key
)
)
raise Exception(readable_exception)
# This creates an update statement to increment to the setup_percent_complete field
# by the given proportion
logger.info("Instance percent before %s" % instance.setup_percent_complete)
instance.setup_percent_complete = F('setup_percent_complete') + 100*publishing_info['proportion']
instance.save()
logger.info("Instance percent after %s" % instance.setup_percent_complete)
send_message_to_client(user.id, dict(
event=event,
job_id=str(job.hashid),
config_entity_id=config_entity and config_entity.id,
config_entity_class_name=config_entity and config_entity.__class__.__name__,
# Send the key since the id of new instances might be meaningless to the client
# If it hasn't updated the record's id yet
publisher_name=publishing_info['publisher_name'],
class_name=bundle.class_name_for_client,
ids=bundle.client_instance_ids,
keys=bundle.keys,
# Used for Features and other things that have a class scope key
class_key=bundle.class_key,
# Send the proportion of work that completing this signal signifies--0 to 1
proportion=publishing_info['proportion'],
# The client can display a this to describe the progress
progress_description=humanize(publishing_info['signal_path']),
))
# Find all dependent signals of this one and run each in parallel
for dependent_signal_path in publishing_info['dependent_signal_paths']:
# Recurse
post_save_publishing(
dependent_signal_path,
config_entity,
def create_db_entity(pg_dump_fpath, db_entity_key, db_entity_name, table_name,
layer_count, user, config_entity, **kwargs):
"""
Create a DbEntity and all associated layers, etc. The majority of the
processing occurs in the post-save methods on DbEntity objects, this method
simply gets objects in the necessary state to trigger it.
"""
logger.debug("Creating DbEntity %s with pg_dump file %s", db_entity_key,
pg_dump_fpath)
if 'upload_task' in kwargs:
upload_task = kwargs['upload_task']
else:
# we're calling this from the command line
# for testing purposes
upload_task = create_upload_task(
user,
pg_dump_fpath,
config_entity,
extra_data_dict={'X-Progress-ID': 'unused'})
# later post-save processes expect a zipped sql file
zipped_sql_fpath = "{}.zip".format(pg_dump_fpath)
with ZipFile(zipped_sql_fpath, 'w') as zipped_sql:
zipped_sql.write(pg_dump_fpath)
# the UploadDataset represents the processing of a single file
upload_dataset_task = UploadDatasetTask.objects.create(
upload_task=upload_task,
dataset_id=-1,
file_path=zipped_sql_fpath,
filename=db_entity_key,
progress=upload_task.progress,
status=upload_task.status,
extra=upload_task.extra)
# the schema metadata, has information necessary for Django to create
# new data models based on the upload. The DbEntity post-save
# logic uses this.
schema_metadata = get_schema_metadata(pg_dump_fpath, table_name)
upload_dataset_task.metadata = schema_metadata
logger.debug("Saving DbEntity %s and inititialzing post-save processing.",
db_entity_key)
upload_dataset_task.save()
db_entity = DbEntity(creator=user,
updater=user,
name=db_entity_name,
key=db_entity_key,
url='file://{}'.format(zipped_sql_fpath),
setup_percent_complete=0,
schema=config_entity.schema())
# setting `_config_entity` and then calling `save()` triggers
# the post-save processing flow, which, among other things,
# loads the data into the database, creates layers and runs
# required updates to other model objects to be aware of this
# layer.
db_entity._config_entity = config_entity
db_entity.save()
db_entity.categories.add(
Category.objects.get(key=DbEntityCategoryKey.KEY_CLASSIFICATION,
value=DbEntityCategoryKey.REFERENCE))
upload_dataset_task.progress = 100
upload_dataset_task.status = UploadDatasetTask.SUCCESS
upload_dataset_task.ended_on = timezone.now()
upload_dataset_task.save()
upload_dataset_task.send_progress()
finished_dataset_count = UploadDatasetTask.objects.filter(
upload_task=upload_task, status=UploadDatasetTask.SUCCESS).count()
if finished_dataset_count == layer_count:
upload_task.progress = 100
upload_task.status = UploadTask.SUCCESS
upload_task.ended_on = timezone.now()
upload_task.save()
upload_task.send_progress()
message_kwargs = dict(event="doCreateDbEntity",
id=db_entity.id,
name=db_entity_name,
key=db_entity_key,
config_entity=config_entity.id,
file_dataset=upload_dataset_task.id)
# send websockets `doCreateDbEntity` signal to the browser
send_message_to_client(user.id, message_kwargs)