def rebuild_export(config, schema, output_dir, last_access_cutoff=None, filter=None):
if output_dir == "couch":
saved = get_saved_export_and_delete_copies(config.index)
if last_access_cutoff and saved and saved.last_accessed and \
saved.last_accessed < last_access_cutoff:
# ignore exports that haven't been accessed since last_access_cutoff
return
try:
files = schema.get_export_files(format=config.format, filter=filter)
except SchemaMismatchException:
# fire off a delayed force update to prevent this from happening again
rebuild_schemas.delay(config.index)
raise ExportRebuildError(u'Schema mismatch for {}. Rebuilding tables...'.format(config.filename))
with files:
payload = files.file.payload
if output_dir == "couch":
if not saved:
saved = SavedBasicExport(configuration=config)
else:
saved.configuration = config
if saved.last_accessed is None:
saved.last_accessed = datetime.utcnow()
saved.last_updated = datetime.utcnow()
saved.save()
saved.set_payload(payload)
else:
with open(os.path.join(output_dir, config.filename), "wb") as f:
f.write(payload)
def export_for_group(export_id_or_group, output_dir):
if isinstance(export_id_or_group, basestring):
try:
config = GroupExportConfiguration.get(export_id_or_group)
except ResourceNotFound:
raise Exception("Couldn't find an export with id %s" %
export_id_or_group)
else:
config = export_id_or_group
for config, schema in config.all_exports:
try:
files = schema.get_export_files(format=config.format)
except SchemaMismatchException, e:
# fire off a delayed force update to prevent this from happening again
rebuild_schemas.delay(config.index)
continue
payload = files.file.payload
if output_dir == "couch":
saved = SavedBasicExport.view("couchexport/saved_exports",
key=json.dumps(config.index),
include_docs=True,
reduce=False).one()
if not saved:
saved = SavedBasicExport(configuration=config)
else:
saved.configuration = config
saved.last_updated = datetime.utcnow()
saved.save()
saved.set_payload(payload)
else:
with open(os.path.join(output_dir, config.filename), "wb") as f:
f.write(payload)
def export_for_group(export_id_or_group, output_dir):
if isinstance(export_id_or_group, basestring):
try:
config = GroupExportConfiguration.get(export_id_or_group)
except ResourceNotFound:
raise Exception("Couldn't find an export with id %s" % export_id_or_group)
else:
config = export_id_or_group
for config, schema in config.all_exports:
try:
tmp, _ = schema.get_export_files(format=config.format)
except SchemaMismatchException, e:
# fire off a delayed force update to prevent this from happening again
rebuild_schemas.delay(config.index)
continue
payload = Temp(tmp).payload
if output_dir == "couch":
saved = SavedBasicExport.view("couchexport/saved_exports",
key=json.dumps(config.index),
include_docs=True,
reduce=False).one()
if not saved:
saved = SavedBasicExport(configuration=config)
else:
saved.configuration = config
saved.last_updated = datetime.utcnow()
saved.save()
saved.set_payload(payload)
else:
with open(os.path.join(output_dir, config.filename), "wb") as f:
f.write(payload)
def generate_bulk_file(self, update_progress=None):
update_progress = update_progress or (lambda x: x)
configs = list()
schemas = list()
checkpoints = list()
for export_object in self.export_objects:
config, schema, checkpoint = export_object.get_export_components(
filter=self.export_filter)
configs.append(config)
schemas.append(schema)
checkpoints.append(checkpoint)
writer = get_writer(self.format)
# generate the headers for the bulk excel file
headers = self.generate_table_headers(schemas, checkpoints)
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as tmp:
writer.open(headers, tmp)
# now that the headers are set, lets build the rows
for i, config in enumerate(configs):
try:
for doc in config.get_docs():
if self.export_objects[i].transform:
doc = self.export_objects[i].transform(doc)
table = get_formatted_rows(doc,
schemas[i],
separator=self.separator,
include_headers=isinstance(
self, CustomBulkExport))
if isinstance(self, CustomBulkExport):
table = self.export_objects[i].trim(table, doc)
if table and table[0]:
# if an export only contains data from repeats and a form has no repeats
# then the table list will be empty
table = self.export_objects[i].parse_tables(table)
writer.write(table)
except SchemaMismatchException:
# fire off a delayed force update to prevent this from happening again
rebuild_schemas.delay(self.export_objects[i].index)
writer.write([(self.export_objects[i].table_name, [
FormattedRow([
ugettext(
'There was an error generating this export. '
'If the problem persists please report an issue.'
)
],
separator=self.separator)
])])
update_progress(i + 1)
writer.close()
return path
def generate_bulk_file(self, update_progress=None):
update_progress = update_progress or (lambda x: x)
configs = list()
schemas = list()
checkpoints = list()
for export_object in self.export_objects:
config, schema, checkpoint = export_object.get_export_components(filter=self.export_filter)
configs.append(config)
schemas.append(schema)
checkpoints.append(checkpoint)
writer = get_writer(self.format)
# generate the headers for the bulk excel file
headers = self.generate_table_headers(schemas, checkpoints)
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as tmp:
writer.open(headers, tmp)
# now that the headers are set, lets build the rows
for i, config in enumerate(configs):
try:
for doc in config.get_docs():
if self.export_objects[i].transform:
doc = self.export_objects[i].transform(doc)
table = get_formatted_rows(
doc, schemas[i], separator=self.separator,
include_headers=isinstance(self, CustomBulkExport))
if isinstance(self, CustomBulkExport):
table = self.export_objects[i].trim(table, doc)
if table and table[0]:
# if an export only contains data from repeats and a form has no repeats
# then the table list will be empty
table = self.export_objects[i].parse_tables(table)
writer.write(table)
except SchemaMismatchException:
# fire off a delayed force update to prevent this from happening again
rebuild_schemas.delay(self.export_objects[i].index)
writer.write(
[(self.export_objects[i].table_name, [
FormattedRow([
ugettext(
'There was an error generating this export. '
'If the problem persists please report an issue.'
)],
separator=self.separator)
])]
)
update_progress(i+1)
writer.close()
return path
def rebuild_export(config, schema, last_access_cutoff=None, filter=None):
saved_export = get_saved_export_and_delete_copies(config.index)
if _should_not_rebuild_export(saved_export, last_access_cutoff):
return
try:
files = schema.get_export_files(format=config.format, filter=filter)
except SchemaMismatchException:
# fire off a delayed force update to prevent this from happening again
rebuild_schemas.delay(config.index)
raise ExportRebuildError('Schema mismatch for {}. Rebuilding tables...'.format(config.filename))
with files:
_save_export_payload(files, saved_export, config, is_safe=schema.is_safe)
def rebuild_export(config, schema, last_access_cutoff=None, filter=None):
saved_export = get_saved_export_and_delete_copies(config.index)
if _should_not_rebuild_export(saved_export, last_access_cutoff):
return
try:
files = schema.get_export_files(format=config.format, filter=filter)
except SchemaMismatchException:
# fire off a delayed force update to prevent this from happening again
rebuild_schemas.delay(config.index)
raise ExportRebuildError('Schema mismatch for {}. Rebuilding tables...'.format(config.filename))
with files:
_save_export_payload(files, saved_export, config, is_safe=schema.is_safe)
if async:
return export_object.export_data_async(
filter=filter,
filename=filename,
previous_export_id=previous_export_id,
format=format,
max_column_size=max_column_size,
)
else:
if not next:
next = export.ExcelExportReport.get_url(domain=domain)
try:
resp = export_object.download_data(format, filter=filter, limit=limit)
except SchemaMismatchException, e:
rebuild_schemas.delay(export_object.index)
messages.error(
request,
"Sorry, the export failed for %s, please try again later" \
% export_object.name
)
return HttpResponseRedirect(next)
if resp:
return resp
else:
messages.error(request, "Sorry, there was no data found for the tag '%s'." % export_object.name)
return HttpResponseRedirect(next)
@login_or_digest
@require_form_export_permission
@require_GET
return export_object.export_data_async(
filter=filter,
filename=filename,
previous_export_id=previous_export_id,
format=format,
max_column_size=max_column_size,
)
else:
if not next:
next = export.ExcelExportReport.get_url(domain=domain)
try:
resp = export_object.download_data(format,
filter=filter,
limit=limit)
except SchemaMismatchException, e:
rebuild_schemas.delay(export_object.index)
messages.error(
request,
"Sorry, the export failed for %s, please try again later" \
% export_object.name
)
return HttpResponseRedirect(next)
if resp:
return resp
else:
messages.error(
request, "Sorry, there was no data found for the tag '%s'." %
export_object.name)
return HttpResponseRedirect(next)
def _export_default_or_custom_data(request, domain, export_id=None, bulk_export=False, safe_only=False):
async = request.GET.get('async') == 'true'
next = request.GET.get("next", "")
format = request.GET.get("format", "")
export_type = request.GET.get("type", "form")
previous_export_id = request.GET.get("previous_export", None)
filename = request.GET.get("filename", None)
max_column_size = int(request.GET.get("max_column_size", 2000))
filter = util.create_export_filter(request, domain, export_type=export_type)
if bulk_export:
try:
is_custom = json.loads(request.GET.get("is_custom", "false"))
export_tags = json.loads(request.GET.get("export_tags", "null") or "null")
except ValueError:
return HttpResponseBadRequest()
export_helper = (CustomBulkExportHelper if is_custom else ApplicationBulkExportHelper)(
domain=domain,
safe_only=safe_only
)
return export_helper.prepare_export(export_tags, filter)
elif export_id:
# this is a custom export
try:
export_object = CustomExportHelper.make(request, export_type, domain, export_id).custom_export
if safe_only and not export_object.is_safe:
return HttpResponseForbidden()
except ResourceNotFound:
raise Http404()
elif safe_only:
return HttpResponseForbidden()
else:
if not async:
# this function doesn't support synchronous export without a custom export object
# if we ever want that (i.e. for HTML Preview) then we just need to give
# FakeSavedExportSchema a download_data function (called below)
return HttpResponseBadRequest()
try:
export_tag = json.loads(request.GET.get("export_tag", "null") or "null")
except ValueError:
return HttpResponseBadRequest()
assert(export_tag[0] == domain)
# hack - also filter instances here rather than mess too much with trying to make this
# look more like a FormExportSchema
if export_type == 'form':
filter &= SerializableFunction(instances)
export_object = FakeSavedExportSchema(index=export_tag)
if not filename:
filename = export_object.name
filename += ' ' + date.today().isoformat()
if async:
return export_object.export_data_async(
filter=filter,
filename=filename,
previous_export_id=previous_export_id,
format=format,
max_column_size=max_column_size,
)
else:
if not next:
next = export.ExcelExportReport.get_url(domain=domain)
try:
resp = export_object.download_data(format, filter=filter)
except SchemaMismatchException, e:
rebuild_schemas.delay(export_object.index)
messages.error(
request,
"Sorry, the export failed for %s, please try again later" \
% export_object.name
)
return HttpResponseRedirect(next)
if resp:
return resp
else:
messages.error(request, "Sorry, there was no data found for the tag '%s'." % export_object.name)
return HttpResponseRedirect(next)
