def process_upload(self):
# save the file w/ soil
self.uploaded_file.file.seek(0)
saved_file = expose_download(self.uploaded_file.file.read(), expiry=BulkMultimediaStatusCache.cache_expiry)
processing_id = saved_file.download_id
status = BulkMultimediaStatusCache(processing_id)
status.save()
process_bulk_upload_zip.delay(processing_id, self.domain, self.app_id,
username=self.request.couch_user.username if self.request.couch_user else None,
share_media=self.share_media,
license_name=self.license_used, author=self.author,
attribution_notes=self.attribution_notes, replace_existing=self.replace_existing)
return status.get_response()
def post(self, request, *args, **kwargs):
if not self.processing_id:
return HttpResponseBadRequest("A processing_id is required.")
status = BulkMultimediaStatusCache.get(self.processing_id)
if status is None:
# No status could be retrieved from the cache
fake_status = BulkMultimediaStatusCache(self.processing_id)
fake_status.complete = True
fake_status.errors.append(_('There was an issue retrieving the status from the cache. '
'We are looking into it. Please try uploading again.'))
logging.error("[Multimedia Bulk Upload] Process ID #%s encountered an issue while retrieving "
"a status from the cache." % self.processing_id)
response = fake_status.get_response()
else:
response = status.get_response()
return HttpResponse(json.dumps(response))
def process_upload(self):
if hasattr(self.uploaded_file, 'temporary_file_path') and settings.SHARED_DRIVE_CONF.temp_dir:
processing_id = uuid.uuid4().hex
path = settings.SHARED_DRIVE_CONF.get_temp_file(suffix='.upload')
shutil.move(self.uploaded_file.temporary_file_path(), path)
status = BulkMultimediaStatusCacheNfs(processing_id, path)
status.save()
else:
self.uploaded_file.file.seek(0)
saved_file = expose_cached_download(
self.uploaded_file.file.read(),
expiry=BulkMultimediaStatusCache.cache_expiry,
file_extension=file_extention_from_filename(self.uploaded_file.name),
)
processing_id = saved_file.download_id
status = BulkMultimediaStatusCache(processing_id)
status.save()
process_bulk_upload_zip.delay(processing_id, self.domain, self.app_id,
username=self.username,
share_media=self.share_media,
license_name=self.license_used,
author=self.author,
attribution_notes=self.attribution_notes)
return status.get_response()
def post(self, request, *args, **kwargs):
if not self.processing_id:
return HttpResponseBadRequest("A processing_id is required.")
status = BulkMultimediaStatusCache.get(self.processing_id)
if status is None:
# No status could be retrieved from the cache
fake_status = BulkMultimediaStatusCache(self.processing_id)
fake_status.complete = True
fake_status.errors.append(_('There was an issue retrieving the status from the cache. '
'We are looking into it. Please try uploading again.'))
logging.error("[Multimedia Bulk Upload] Process ID #%s encountered an issue while retrieving "
"a status from the cache." % self.processing_id)
response = fake_status.get_response()
else:
response = status.get_response()
return HttpResponse(json.dumps(response))
def process_upload(self):
# save the file w/ soil
self.uploaded_file.file.seek(0)
saved_file = expose_download(self.uploaded_file.file.read(), expiry=BulkMultimediaStatusCache.cache_expiry)
processing_id = saved_file.download_id
status = BulkMultimediaStatusCache(processing_id)
status.save()
process_bulk_upload_zip.delay(processing_id, self.domain, self.app_id,
username=self.username,
share_media=self.share_media,
license_name=self.license_used,
author=self.author,
attribution_notes=self.attribution_notes,
replace_existing=self.replace_existing)
return status.get_response()
def process_bulk_upload_zip(processing_id,
domain,
app_id,
username=None,
share_media=False,
license_name=None,
author=None,
attribution_notes=None):
"""
Responsible for processing the uploaded zip from Bulk Upload.
"""
status = BulkMultimediaStatusCache.get(processing_id)
if not status:
# no download data available, abort
return
app = get_app(domain, app_id)
status.in_celery = True
status.save()
uploaded_zip = status.get_upload_zip()
if not uploaded_zip:
return
zipped_files = uploaded_zip.namelist()
status.total_files = len(zipped_files)
checked_paths = []
try:
for index, path in enumerate(zipped_files):
status.update_progress(len(checked_paths))
checked_paths.append(path)
file_name = os.path.basename(path)
try:
data = uploaded_zip.read(path)
except Exception as e:
status.add_unmatched_path(path,
_("Error reading file: %s" % e))
continue
media_class = CommCareMultimedia.get_class_by_data(data,
filename=path)
if not media_class:
status.add_skipped_path(path,
CommCareMultimedia.get_mime_type(data))
continue
app_paths = list(app.get_all_paths_of_type(media_class.__name__))
app_paths_lower = [p.lower() for p in app_paths]
form_path = media_class.get_form_path(path, lowercase=True)
if not form_path in app_paths_lower:
status.add_unmatched_path(
path,
_("Did not match any %s paths in application." %
media_class.get_nice_name()))
continue
index_of_path = app_paths_lower.index(form_path)
form_path = app_paths[
index_of_path] # this is the correct capitalization as specified in the form
multimedia = media_class.get_by_data(data)
if not multimedia:
status.add_unmatched_path(
path,
_("Matching path found, but could not save the data to couch."
))
continue
is_new = form_path not in list(app.multimedia_map)
is_updated = multimedia.attach_data(data,
original_filename=file_name,
username=username)
if not is_updated and not getattr(multimedia, '_id'):
status.add_unmatched_path(
form_path,
_("Matching path found, but didn't save new multimedia correctly."
))
continue
if is_updated or is_new:
multimedia.add_domain(domain, owner=True)
if share_media:
multimedia.update_or_add_license(
domain,
type=license_name,
author=author,
attribution_notes=attribution_notes)
app.create_mapping(multimedia, form_path)
media_info = multimedia.get_media_info(form_path,
is_updated=is_updated,
original_path=path)
status.add_matched_path(media_class, media_info)
status.update_progress(len(checked_paths))
except Exception as e:
status.mark_with_error(_("Error while processing zip: %s" % e))
uploaded_zip.close()
status.complete = True
status.save()
def process_bulk_upload_zip(processing_id, domain, app_id, username=None, share_media=False,
license_name=None, author=None, attribution_notes=None):
"""
Responsible for processing the uploaded zip from Bulk Upload.
"""
status = BulkMultimediaStatusCache.get(processing_id)
if not status:
# no download data available, abort
return
app = get_app(domain, app_id)
status.in_celery = True
status.save()
uploaded_zip = status.get_upload_zip()
if not uploaded_zip:
return
zipped_files = uploaded_zip.namelist()
status.total_files = len(zipped_files)
checked_paths = []
try:
for index, path in enumerate(zipped_files):
status.update_progress(len(checked_paths))
checked_paths.append(path)
file_name = os.path.basename(path)
try:
data = uploaded_zip.read(path)
except Exception as e:
status.add_unmatched_path(path, _("Error reading file: %s" % e))
continue
media_class = CommCareMultimedia.get_class_by_data(data, filename=path)
if not media_class:
status.add_skipped_path(path, CommCareMultimedia.get_mime_type(data))
continue
app_paths = list(app.get_all_paths_of_type(media_class.__name__))
app_paths_lower = [p.lower() for p in app_paths]
form_path = media_class.get_form_path(path, lowercase=True)
if not form_path in app_paths_lower:
status.add_unmatched_path(path,
_("Did not match any %s paths in application." % media_class.get_nice_name()))
continue
index_of_path = app_paths_lower.index(form_path)
form_path = app_paths[index_of_path] # this is the correct capitalization as specified in the form
multimedia = media_class.get_by_data(data)
if not multimedia:
status.add_unmatched_path(path,
_("Matching path found, but could not save the data to couch."))
continue
is_new = form_path not in list(app.multimedia_map)
is_updated = multimedia.attach_data(data,
original_filename=file_name,
username=username)
if not is_updated and not getattr(multimedia, '_id'):
status.add_unmatched_path(form_path,
_("Matching path found, but didn't save new multimedia correctly."))
continue
if is_updated or is_new:
multimedia.add_domain(domain, owner=True)
if share_media:
multimedia.update_or_add_license(domain, type=license_name, author=author,
attribution_notes=attribution_notes)
app.create_mapping(multimedia, form_path)
media_info = multimedia.get_media_info(form_path, is_updated=is_updated, original_path=path)
status.add_matched_path(media_class, media_info)
status.update_progress(len(checked_paths))
except Exception as e:
status.mark_with_error(_("Error while processing zip: %s" % e))
uploaded_zip.close()
status.complete = True
status.save()
def process_bulk_upload_zip(processing_id, domain, app_id, username=None, share_media=False,
license_name=None, author=None, attribution_notes=None, replace_existing=False):
"""
Responsible for processing the uploaded zip from Bulk Upload.
"""
status = BulkMultimediaStatusCache.get(processing_id)
if not status:
# no download data available, abort
return
app = get_app(domain, app_id)
status.in_celery = True
status.save()
try:
saved_file = StringIO.StringIO()
saved_ref = DownloadBase.get(processing_id)
data = saved_ref.get_content()
saved_file.write(data)
except Exception as e:
status.mark_with_error(_("Could not fetch cached bulk upload file. Error: %s." % e))
return
try:
saved_file.seek(0)
uploaded_zip = zipfile.ZipFile(saved_file)
except Exception as e:
status.mark_with_error(_("Error opening file as zip file: %s" % e))
return
if uploaded_zip.testzip():
status.mark_with_error(_("Error encountered processing Zip File. File doesn't look valid."))
return
zipped_files = uploaded_zip.namelist()
status.total_files = len(zipped_files)
checked_paths = []
try:
for index, path in enumerate(zipped_files):
status.update_progress(len(checked_paths))
checked_paths.append(path)
file_name = os.path.basename(path)
try:
data = uploaded_zip.read(path)
except Exception as e:
status.add_unmatched_path(path, _("Error reading file: %s" % e))
continue
media_class = CommCareMultimedia.get_class_by_data(data)
if not media_class:
# skip these...
continue
app_paths = app.get_all_paths_of_type(media_class.__name__)
form_path = media_class.get_form_path(path)
if not form_path in app_paths:
status.add_unmatched_path(path,
_("Did not match any %s paths in application." % media_class.get_nice_name()))
continue
multimedia = media_class.get_by_data(data)
if not multimedia:
status.add_unmatched_path(path,
_("Matching path found, but could not save the data to couch."))
continue
is_updated = multimedia.attach_data(data, original_filename=file_name, username=username,
replace_attachment=replace_existing)
if not is_updated and not getattr(multimedia, '_id'):
status.add_unmatched_path(form_path,
_("Matching path found, but didn't save new multimedia correctly."))
continue
if is_updated:
multimedia.add_domain(domain, owner=True)
if share_media:
multimedia.update_or_add_license(domain, type=license_name, author=author,
attribution_notes=attribution_notes)
app.create_mapping(multimedia, form_path)
media_info = multimedia.get_media_info(form_path, is_updated=is_updated, original_path=path)
status.add_matched_path(media_class, media_info)
status.update_progress(len(checked_paths))
except Exception as e:
status.mark_with_error(_("Error while processing zip: %s" % e))
uploaded_zip.close()
status.complete = True
status.save()
def post(self, request, *args, **kwargs):
if not self.processing_id:
return HttpResponseBadRequest("A processing_id is required.")
status = BulkMultimediaStatusCache.get(self.processing_id)
return HttpResponse(json.dumps(status.get_response()))