def stamp(survey, output_filename, cmdline):
# copy questionnaire_ids
# get number of sheets to create
if cmdline['file'] or cmdline['random'] or cmdline['existing']:
if not survey.defs.print_questionnaire_id:
log.error(_("You may not specify the number of sheets for this survey. All questionnaires will be identical as the survey has been configured to not use questionnaire IDs for each sheet."))
return 1
if cmdline['existing']:
questionnaire_ids = survey.questionnaire_ids
elif cmdline['file']:
if cmdline['file'] == '-':
fd = sys.stdin
else:
fd = codecs.open(cmdline['file'], 'r', encoding="utf-8")
questionnaire_ids = list()
for line in fd.readlines():
# Only strip newline/linefeed not spaces
line = line.strip('\n\r')
# Skip empty lines
if line == "":
continue
questionnaire_ids.append(survey.validate_questionnaire_id(line))
else:
# Create random IDs
max = pow(2, 16)
min = max - 50000
questionnaire_ids = range(min, max)
# Remove any id that has already been used.
for id in survey.questionnaire_ids:
if type(id) != int:
continue
questionnaire_ids[id - min] = 0
questionnaire_ids = [id for id in questionnaire_ids if id > min]
random.shuffle(questionnaire_ids)
questionnaire_ids = questionnaire_ids[:cmdline['random']]
else:
if survey.defs.print_questionnaire_id:
log.error(_("This survey has been configured to use questionnaire IDs. Each questionnaire will be unique. You need to use on of the options to add new IDs or use the existing ones."))
return 1
questionnaire_ids = None
if questionnaire_ids is not None:
survey.questionnaire_ids.extend(questionnaire_ids)
if os.path.exists(survey.path('questionnaire.tex')):
# use the LaTeX stamper
from sdaps.stamp.latex import create_stamp_pdf
else:
from sdaps.stamp.generic import create_stamp_pdf
create_stamp_pdf(survey, output_filename, questionnaire_ids)
survey.save()
def stamp(survey, output_filename, cmdline):
# copy questionnaire_ids
# get number of sheets to create
if cmdline['file'] or cmdline['random'] or cmdline['existing']:
if not survey.defs.print_questionnaire_id:
log.error(_("You may not specify the number of sheets for this survey. All questionnaires will be identical as the survey has been configured to not use questionnaire IDs for each sheet."))
return 1
if cmdline['existing']:
questionnaire_ids = survey.questionnaire_ids
elif cmdline['file']:
if cmdline['file'] == '-':
fd = sys.stdin
else:
fd = codecs.open(cmdline['file'], 'r', encoding="utf-8")
questionnaire_ids = list()
for line in fd.readlines():
# Only strip newline/linefeed not spaces
line = line.strip('\n\r')
# Skip empty lines
if line == "":
continue
questionnaire_ids.append(survey.validate_questionnaire_id(line))
else:
# Create random IDs
max = pow(2, 16)
min = max - 50000
questionnaire_ids = range(min, max)
# Remove any id that has already been used.
for id in survey.questionnaire_ids:
if type(id) != int:
continue
questionnaire_ids[id - min] = 0
questionnaire_ids = [id for id in questionnaire_ids if id > min]
random.shuffle(questionnaire_ids)
questionnaire_ids = questionnaire_ids[:cmdline['random']]
else:
if survey.defs.print_questionnaire_id:
log.error(_("This survey has been configured to use questionnaire IDs. Each questionnaire will be unique. You need to use on of the options to add new IDs or use the existing ones."))
return 1
questionnaire_ids = None
if questionnaire_ids is not None and not cmdline['existing']:
survey.questionnaire_ids.extend(questionnaire_ids)
if os.path.exists(survey.path('questionnaire.tex')):
# use the LaTeX stamper
from sdaps.stamp.latex import create_stamp_pdf
else:
from sdaps.stamp.generic import create_stamp_pdf
create_stamp_pdf(survey, output_filename, questionnaire_ids)
survey.save()
def create_stamp_pdf(survey, output_filename, questionnaire_ids):
if questionnaire_ids is None:
log.warn(_("There should be no need to stamp a SDAPS Project that uses LaTeX and does not have different questionnaire IDs printed on each sheet.\nI am going to do so anyways."))
# Temporary directory for TeX files.
tmpdir = tempfile.mkdtemp(prefix='sdaps-stamp-')
try:
latex.write_override(survey, os.path.join(tmpdir, 'sdaps.opt'), questionnaire_ids=questionnaire_ids)
print(_("Running %s now multiple times to generate the stamped questionnaire.") % survey.defs.engine)
latex.compile(survey.defs.engine, 'questionnaire.tex', tmpdir, inputs=[os.path.abspath(survey.path())])
if not os.path.exists(os.path.join(tmpdir, 'questionnaire.pdf')):
log.error(_("Error running \"%s\" to compile the LaTeX file.") % defs.latex_engine)
raise AssertionError('PDF file not generated')
shutil.move(os.path.join(tmpdir, 'questionnaire.pdf'), output_filename)
except:
log.error(_("An error occured during creation of the report. Temporary files left in '%s'." % tmpdir))
raise
shutil.rmtree(tmpdir)
def add(cmdline):
import sys
from sdaps.add import add_image, check_image
from sdaps import image
error = False
survey = model.survey.Survey.load(cmdline['project'])
filelist = []
deletelist = []
if not cmdline['convert']:
for file in cmdline['images']:
filelist.append(file)
if not check_image(survey,
file,
cmdline['duplex'],
cmdline['force'],
message=True):
error = True
if error:
return
else:
if not cmdline['copy']:
log.error(
_("The --no-copy option is not compatible with --convert!"))
return 1
try:
from sdaps.convert import convert_images
except:
log.error(
"Need to convert the images to monochrome TIFF, however the conversion module cannot be imported. You are likely missing the OpenCV dependency."
)
return 1
print _("Converting input files into a single temporary file.")
tmp = tempfile.mktemp(suffix='.tif', prefix='sdaps-convert-')
deletelist.append(tmp)
filelist.append(tmp)
# Run conversion
# TODO: Allow 3D transformation here!
try:
convert_images(cmdline['images'], tmp, survey.defs.paper_width,
survey.defs.paper_height, cmdline['transform'])
if not check_image(survey, tmp, cmdline['duplex'],
cmdline['force']):
log.error(
_("The page count of the created temporary file does not work with this survey."
))
raise AssertionError()
except Exception, e:
log.error(str(e))
log.error(_("Running the conversion failed."))
error = True
def create_stamp_pdf(survey, output_filename, questionnaire_ids):
if questionnaire_ids is None:
log.warn(
_("There should be no need to stamp a SDAPS Project that uses LaTeX and does not have different questionnaire IDs printed on each sheet.\nI am going to do so anyways."
))
# Temporary directory for TeX files.
tmpdir = tempfile.mkdtemp()
try:
# Similar to setuptex/setup.py, but we also set questionnaire IDs
latex_override = open(os.path.join(tmpdir, 'sdaps.opt'), 'w')
latex_override.write(
'% This file exists to force the latex document into "final" mode.\n'
)
latex_override.write(
'% It is parsed after the setup phase of the SDAPS class.\n\n')
latex_override.write('\setcounter{surveyidlshw}{%i}\n' %
(survey.survey_id % (2**16)))
latex_override.write('\setcounter{surveyidmshw}{%i}\n' %
(survey.survey_id / (2**16)))
latex_override.write('\def\surveyid{%i}\n' % (survey.survey_id))
latex_override.write('\def\globalid{%s}\n' %
(tex_quote_braces(survey.global_id)) if survey.
global_id is not None else '')
latex_override.write('\\@STAMPtrue\n')
latex_override.write('\\@PAGEMARKtrue\n')
latex_override.write('\\@sdaps@draftfalse\n')
if questionnaire_ids is not None:
quoted_ids = [
tex_quote_braces(str(id)) for id in questionnaire_ids
]
latex_override.write('\def\questionnaireids{{%s}}\n' %
'},{'.join(quoted_ids))
latex_override.close()
print _("Running %s now twice to generate the stamped questionnaire."
) % defs.latex_engine
latex.compile('questionnaire.tex',
tmpdir,
inputs=[os.path.abspath(survey.path())])
if not os.path.exists(os.path.join(tmpdir, 'questionnaire.pdf')):
log.error(
_("Error running \"%s\" to compile the LaTeX file.") %
defs.latex_engine)
raise AssertionError('PDF file not generated')
shutil.move(os.path.join(tmpdir, 'questionnaire.pdf'), output_filename)
except:
log.error(
_("An error occured during creation of the report. Temporary files left in '%s'."
% tmpdir))
raise
shutil.rmtree(tmpdir)
def convert(cmdline):
from sdaps.convert import convert_images
if cmdline['output'] is None:
log.error(_("No output filename specified!"))
sys.exit(1)
# We need a survey only for the paper size!
survey = model.survey.Survey.load(cmdline['project'])
convert_images(cmdline['images'], cmdline['output'], survey.defs.paper_width, survey.defs.paper_height, cmdline['transform'])
def convert(cmdline):
from sdaps.convert import convert_images
if cmdline['output'] is None:
log.error(_("No output filename specified!"))
sys.exit(1)
# We need a survey only for the paper size!
survey = model.survey.Survey.load(cmdline['project'])
convert_images(cmdline['images'], cmdline['output'], survey.defs.paper_width, survey.defs.paper_height, cmdline['transform'])
def convert(cmdline):
import sys
if cmdline["output"] is None:
log.error(_("No output filename specified!"))
sys.exit(1)
# We need a survey only for the paper size!
survey = model.survey.Survey.load(cmdline["project"])
convert_images(
cmdline["images"], cmdline["output"], survey.defs.paper_width, survey.defs.paper_height, cmdline["transform"]
)
def __init__(self, *args):
model.buddy.Buddy.__init__(self, *args)
if self.obj.sheet.survey.defs.style == "classic":
import classic
elif self.obj.sheet.survey.defs.style == "code128":
import code128
elif self.obj.sheet.survey.defs.style == "custom":
if not hasattr(self.obj, "style"):
import sys
log.error(_("No style buddy loaded. This needs to be done for the \"custom\" style!"))
sys.exit(1)
else:
raise AssertionError
def convert_images(images, outfile, paper_width, paper_height, transform=False):
for i, (img, filename, page) in enumerate(opencv.iter_images_and_pages(images)):
img = opencv.ensure_portrait(img)
img = opencv.sharpen(img)
if transform:
try:
img = opencv.transform_using_corners(img, paper_width, paper_height)
except AssertionError:
log.error(_("Could not apply 3D-transformation to image '%s', page %i!") % (filename, page))
mono = opencv.convert_to_monochrome(img)
image.write_a1_to_tiff(outfile, opencv.to_a1_surf(mono))
def gui(survey, cmdline):
filter = clifilter.clifilter(survey, cmdline['filter'])
provider = Provider(survey, filter)
if not provider.images:
log.error(_("The survey does not have any images! Please add images (and run recognize) before using the GUI."))
return 1
try:
# Exit the mainloop if Ctrl+C is pressed in the terminal.
GLib.unix_signal_add_full(GLib.PRIORITY_HIGH, signal.SIGINT, lambda *args : Gtk.main_quit(), None)
except AttributeError:
# Whatever, it is only to enable Ctrl+C anyways
pass
MainWindow(provider).run()
def gui(survey, cmdline):
filter = clifilter.clifilter(survey, cmdline['filter'])
provider = Provider(survey, filter)
if not provider.images:
log.error(_("The survey does not have any images! Please add images (and run recognize) before using the GUI."))
return 1
try:
# Exit the mainloop if Ctrl+C is pressed in the terminal.
GLib.unix_signal_add_full(GLib.PRIORITY_HIGH, signal.SIGINT, lambda *args : Gtk.main_quit(), None)
except AttributeError:
# Whatever, it is only to enable Ctrl+C anyways
pass
MainWindow(provider).run()
def __init__(self, *args):
model.buddy.Buddy.__init__(self, *args)
if self.obj.sheet.survey.defs.style == "classic":
import classic
elif self.obj.sheet.survey.defs.style == "code128":
import code128
elif self.obj.sheet.survey.defs.style == "qr":
import qrcode
elif self.obj.sheet.survey.defs.style == "custom":
if not hasattr(self.obj, "style"):
import sys
log.error(_("No style buddy loaded. This needs to be done for the \"custom\" style!"))
sys.exit(1)
else:
raise AssertionError
def validate_questionnaire_id(self, qid):
if self.defs.style == "classic":
# The ID needs to be an integer
try:
return int(qid)
except ValueError:
log.error(_("IDs need to be integers in \"classic\" style!"))
sys.exit(1)
elif self.defs.style == "code128":
# Check each character for validity
for c in unicode(qid):
if not c in defs.c128_chars:
log.error(_("Invalid character %s in questionnaire ID \"%s\" in \"code128\" style!") % (c, qid))
sys.exit(1)
return qid
else:
AssertionError()
def create_stamp_pdf(survey, output_filename, questionnaire_ids):
if questionnaire_ids is None:
log.warn(
_(
"There should be no need to stamp a SDAPS Project that uses LaTeX and does not have different questionnaire IDs printed on each sheet.\nI am going to do so anyways."
)
)
# Temporary directory for TeX files.
tmpdir = tempfile.mkdtemp()
try:
# Similar to setuptex/setup.py, but we also set questionnaire IDs
latex_override = open(os.path.join(tmpdir, "sdaps.opt"), "w")
latex_override.write('% This file exists to force the latex document into "final" mode.\n')
latex_override.write("% It is parsed after the setup phase of the SDAPS class.\n\n")
latex_override.write("\setcounter{surveyidlshw}{%i}\n" % (survey.survey_id % (2 ** 16)))
latex_override.write("\setcounter{surveyidmshw}{%i}\n" % (survey.survey_id / (2 ** 16)))
latex_override.write("\def\surveyid{%i}\n" % (survey.survey_id))
latex_override.write(
"\def\globalid{%s}\n" % (tex_quote_braces(survey.global_id)) if survey.global_id is not None else ""
)
latex_override.write("\\@STAMPtrue\n")
latex_override.write("\\@PAGEMARKtrue\n")
latex_override.write("\\@sdaps@draftfalse\n")
if questionnaire_ids is not None:
quoted_ids = [tex_quote_braces(str(id)) for id in questionnaire_ids]
latex_override.write("\def\questionnaireids{{%s}}\n" % "},{".join(quoted_ids))
latex_override.close()
print _("Running %s now twice to generate the stamped questionnaire.") % defs.latex_engine
latex.compile("questionnaire.tex", tmpdir, inputs=[os.path.abspath(survey.path())])
if not os.path.exists(os.path.join(tmpdir, "questionnaire.pdf")):
log.error(_('Error running "%s" to compile the LaTeX file.') % defs.latex_engine)
raise AssertionError("PDF file not generated")
shutil.move(os.path.join(tmpdir, "questionnaire.pdf"), output_filename)
except:
log.error(_("An error occured during creation of the report. Temporary files left in '%s'." % tmpdir))
raise
shutil.rmtree(tmpdir)
def validate_questionnaire_id(self, qid):
"""Do style specific sanity checks on the questionnaire ID."""
if self.defs.style == "classic":
# The ID needs to be an integer
try:
return int(qid)
except ValueError:
log.error(_("IDs need to be integers in \"classic\" style!"))
sys.exit(1)
elif self.defs.style == "code128":
# Check each character for validity
for c in str(qid):
if not c in defs.c128_chars:
log.error(
_("Invalid character %s in questionnaire ID \"%s\" in \"code128\" style!"
) % (c, qid))
sys.exit(1)
return qid
elif self.defs.style == "custom":
log.error(
_("SDAPS cannot draw a questionnaire ID with the \"custom\" style. Do this yourself somehow!"
))
sys.exit(1)
elif self.defs.style == "qr":
return qid
else:
AssertionError()
def create_stamp_pdf(survey, output_filename, questionnaire_ids):
if questionnaire_ids is None:
log.warn(
_("There should be no need to stamp a SDAPS Project that uses LaTeX and does not have different questionnaire IDs printed on each sheet.\nI am going to do so anyways."
))
# Temporary directory for TeX files.
tmpdir = tempfile.mkdtemp(prefix='sdaps-stamp-')
try:
latex.write_override(survey,
os.path.join(tmpdir, 'sdaps.opt'),
questionnaire_ids=questionnaire_ids)
print(
_("Running %s now multiple times to generate the stamped questionnaire."
) % survey.defs.engine)
latex.compile(survey.defs.engine,
'questionnaire.tex',
tmpdir,
inputs=[os.path.abspath(survey.path())])
if not os.path.exists(os.path.join(tmpdir, 'questionnaire.pdf')):
log.error(
_("Error running \"%s\" to compile the LaTeX file.") %
defs.latex_engine)
raise AssertionError('PDF file not generated')
shutil.move(os.path.join(tmpdir, 'questionnaire.pdf'), output_filename)
except:
log.error(
_("An error occured during creation of the report. Temporary files left in '%s'."
% tmpdir))
raise
shutil.rmtree(tmpdir)
def convert_images(images,
outfile,
paper_width,
paper_height,
transform=False):
from sdaps import image
from sdaps.utils import opencv
for i, (img, filename,
page) in enumerate(opencv.iter_images_and_pages(images)):
img = opencv.ensure_portrait(img)
img = opencv.sharpen(img)
if transform:
try:
img = opencv.transform_using_corners(img, paper_width,
paper_height)
except AssertionError:
log.error(
_("Could not apply 3D-transformation to image '%s', page %i!"
) % (filename, page))
mono = opencv.convert_to_monochrome(img)
image.write_a1_to_tiff(outfile, opencv.to_a1_surf(mono))
def validate_questionnaire_id(self, qid):
"""Do style specific sanity checks on the questionnaire ID."""
if self.defs.style == "classic":
# The ID needs to be an integer
try:
return int(qid)
except ValueError:
log.error(_("IDs need to be integers in \"classic\" style!"))
sys.exit(1)
elif self.defs.style == "code128":
# Check each character for validity
for c in unicode(qid):
if not c in defs.c128_chars:
log.error(_("Invalid character %s in questionnaire ID \"%s\" in \"code128\" style!") % (c, qid))
sys.exit(1)
return qid
elif self.defs.style == "custom":
log.error(_("SDAPS cannot draw a questionnaire ID with the \"custom\" style. Do this yourself somehow!"))
sys.exit(1)
else:
AssertionError()
def setup(survey, cmdline):
if os.access(survey.path(), os.F_OK):
log.error(_('The survey directory already exists.'))
return 1
questionnaire_tex = cmdline['questionnaire.tex']
additionalqobjects = cmdline['additional_questions']
mime = mimetype(questionnaire_tex)
if mime != 'text/x-tex' and mime != '':
log.warn(_('Unknown file type (%s). questionnaire_tex should be of type text/x-tex.') % mime)
log.warn(_('Will keep going, but expect failure!'))
if additionalqobjects is not None:
mime = mimetype(additionalqobjects)
if mime != 'text/plain' and mime != '':
log.error(_('Unknown file type (%s). additionalqobjects should be text/plain.') % mime)
return 1
# Add the new questionnaire
survey.add_questionnaire(model.questionnaire.Questionnaire())
# Create the survey directory, and copy the tex file.
os.mkdir(survey.path())
try:
shutil.copy(questionnaire_tex, survey.path('questionnaire.tex'))
write_latex_override_file(survey, draft=True)
# Copy class and dictionary files
if paths.local_run:
cls_file = os.path.join(paths.source_dir, 'tex', 'sdaps.cls')
code128_file = os.path.join(paths.source_dir, 'tex', 'code128.tex')
dict_files = os.path.join(paths.build_dir, 'tex', '*.dict')
dict_files = glob.glob(dict_files)
else:
cls_file = os.path.join(paths.prefix, 'share', 'sdaps', 'tex', 'sdaps.cls')
code128_file = os.path.join(paths.prefix, 'share', 'sdaps', 'tex', 'code128.tex')
dict_files = os.path.join(paths.prefix, 'share', 'sdaps', 'tex', '*.dict')
dict_files = glob.glob(dict_files)
shutil.copyfile(cls_file, survey.path('sdaps.cls'))
shutil.copyfile(code128_file, survey.path('code128.tex'))
for dict_file in dict_files:
shutil.copyfile(dict_file, survey.path(os.path.basename(dict_file)))
for add_file in cmdline['add']:
shutil.copyfile(add_file, survey.path(os.path.basename(add_file)))
print _("Running %s now twice to generate the questionnaire.") % defs.latex_engine
subprocess.call([defs.latex_engine, '-halt-on-error',
'-interaction', 'batchmode', 'questionnaire.tex'],
cwd=survey.path())
# And again, without the draft mode
subprocess.call([defs.latex_engine, '-halt-on-error',
'-interaction', 'batchmode', 'questionnaire.tex'],
cwd=survey.path())
if not os.path.exists(survey.path('questionnaire.pdf')):
print _("Error running \"%s\" to compile the LaTeX file.") % defs.latex_engine
raise AssertionError('PDF file not generated')
survey.defs.print_questionnaire_id = False
survey.defs.print_survey_id = True
# Parse qobjects
try:
sdapsfileparser.parse(survey)
except Exception, e:
log.error(_("Caught an Exception while parsing the SDAPS file. The current state is:"))
print >>sys.stderr, unicode(survey.questionnaire)
print >>sys.stderr, "------------------------------------"
raise e
# Parse additionalqobjects
if additionalqobjects:
additionalparser.parse(survey, additionalqobjects)
# Last but not least calculate the survey id
survey.calculate_survey_id()
if not survey.check_settings():
log.error(_("Some combination of options and project properties do not work. Aborted Setup."))
shutil.rmtree(survey.path())
return 1
# We need to now rebuild everything so that the correct ID is at the bottom
write_latex_override_file(survey)
print _("Running %s now twice to generate the questionnaire.") % defs.latex_engine
os.remove(survey.path('questionnaire.pdf'))
subprocess.call([defs.latex_engine, '-halt-on-error',
'-interaction', 'batchmode', 'questionnaire.tex'],
cwd=survey.path())
# And again, without the draft mode
subprocess.call([defs.latex_engine, '-halt-on-error',
'-interaction', 'batchmode', 'questionnaire.tex'],
cwd=survey.path())
if not os.path.exists(survey.path('questionnaire.pdf')):
print _("Error running \"%s\" to compile the LaTeX file.") % defs.latex_engine
raise AssertionError('PDF file not generated')
# Print the result
print survey.title
for item in survey.info.items():
print u'%s: %s' % item
print unicode(survey.questionnaire)
log.logfile.open(survey.path('log'))
survey.save()
log.logfile.close()
write_latex_override_file(survey)
print _("Running %s now twice to generate the questionnaire.") % defs.latex_engine
os.remove(survey.path('questionnaire.pdf'))
subprocess.call([defs.latex_engine, '-halt-on-error',
'-interaction', 'batchmode', 'questionnaire.tex'],
cwd=survey.path())
# And again, without the draft mode
subprocess.call([defs.latex_engine, '-halt-on-error',
'-interaction', 'batchmode', 'questionnaire.tex'],
cwd=survey.path())
if not os.path.exists(survey.path('questionnaire.pdf')):
print _("Error running \"%s\" to compile the LaTeX file.") % defs.latex_engine
raise AssertionError('PDF file not generated')
# Print the result
print survey.title
for item in survey.info.items():
print u'%s: %s' % item
print unicode(survey.questionnaire)
log.logfile.open(survey.path('log'))
survey.save()
log.logfile.close()
except:
log.error(_("An error occured in the setup routine. The survey directory still exists. You can for example check the questionnaire.log file for LaTeX compile errors."))
raise
def recognize(self):
global warned_multipage_not_correctly_scanned
self.obj.valid = 1
duplex_mode = self.obj.survey.defs.duplex
# Load all images of this sheet
for image in self.obj.images:
if not image.ignored:
image.rotated = 0
image.surface.load()
failed_pages = set()
# Matrix recognition for all of them
matrix_errors = set()
for page, image in enumerate(self.obj.images):
try:
image.recognize.calculate_matrix()
except RecognitionError:
matrix_errors.add(page)
# We need to check the matrix_errors. Some are expected in simplex mode
for page in matrix_errors:
# in simplex mode every page will have a matrix; it might be a None
# matrix though
log.warn(_('%s, %i: Matrix not recognized.') % (self.obj.images[page].filename, self.obj.images[page].tiff_page))
failed_pages.add(page)
# Rotation for all of them
for page, image in enumerate(self.obj.images):
try:
# This may set the rotation to "None" for unknown
image.recognize.calculate_rotation()
except RecognitionError:
log.warn(_('%s, %i: Rotation not found.') % (image.filename, image.tiff_page))
failed_pages.add(page)
# Copy the rotation over (if required) and print warning if the rotation is unknown
self.duplex_copy_image_attr(failed_pages, 'rotated', _("Neither %s, %i or %s, %i has a known rotation!"))
# Reload any image that is rotated.
for page, image in enumerate(self.obj.images):
if image.rotated and not image.ignored:
image.surface.load()
# And redo the whole matrix stuff ...
# XXX: It would be better to manipulate the matrix instead.
try:
image.recognize.calculate_matrix()
except RecognitionError:
if duplex_mode:
log.warn(_('%s, %i: Matrix not recognized (again).') % (image.filename, image.tiff_page))
failed_pages.add(page)
############
# At this point we can extract the page numbers and IDs as neccessary.
############
# Figure out the page numbers
# ***************************
for page, image in enumerate(self.obj.images):
try:
# This may set the page_number to "None" for unknown
image.recognize.calculate_page_number()
except RecognitionError:
log.warn(_('%s, %i: Could not get page number.') % (image.filename, image.tiff_page))
image.page_number = None
failed_pages.add(page)
i = 0
while i < len(self.obj.images):
# We try to recover at least the page number of failed pages
# this way.
# NOTE: In simplex mode dummy pages will be inserted, so one page
# always has no page number, and the other one has one.
# This is exactly what we want, so we don't need to do anything
# (except warn if we did not find any page!)
failed = (i in failed_pages or i + 1 in failed_pages)
first = self.obj.images[i]
second = self.obj.images[i + 1]
if first.page_number is None and second.page_number is None:
if not failed:
# Whoa, that should not happen.
log.warn(_("Neither %s, %i or %s, %i has a known page number!" %
(first.filename, first.tiff_page, second.filename, second.tiff_page)))
failed_pages.add(i)
failed_pages.add(i + 1)
elif duplex_mode == False:
# Simplex mode is special, as we know that one has to be unreadable
# we need to ensure one of the page numbers is None
if first.page_number is not None and second.page_number is not None:
# We don't touch the ignore flag in this case
# Simply print a message as this should *never* happen
log.error(_("Got a simplex document where two adjacent pages had a known page number. This should never happen as even simplex scans are converted to duplex by inserting dummy pages. Maybe you did a simplex scan but added it in duplex mode? The pages in question are %s, %i and %s, %i.") % (first.filename, first.tiff_page, second.filename, second.tiff_page))
# Set the ignored flag for the unreadable page. This is a valid
# operation as the back side of a readable page is known to be
# empty.
elif first.page_number is None:
first.ignored = True
else:
second.ignored = True
elif first.page_number is None:
# One based, odd -> +1, even -> -1
first.page_number = second.page_number - 1 + 2 * (second.page_number % 2)
elif second.page_number is None:
second.page_number = first.page_number - 1 + 2 * (first.page_number % 2)
elif first.page_number != (second.page_number - 1 + 2 * (second.page_number % 2)):
if not failed:
log.warn(_("Images %s, %i and %s, %i do not have consecutive page numbers!" %
(first.filename, first.tiff_page, second.filename, second.tiff_page)))
failed_pages.add(i)
failed_pages.add(i + 1)
i += 2
# Check that every page has a non None value, and each page exists once.
pages = set()
for i, image in enumerate(self.obj.images):
# Ignore known blank pages
if image.ignored:
continue
if image.page_number is None:
log.warn(_("No page number for page %s, %i exists." % (image.filename, image.tiff_page)))
failed_pages.add(i)
continue
if image.page_number in pages:
log.warn(_("Page number for page %s, %i already used by another image.") %
(image.filename, image.tiff_page))
failed_pages.add(i)
continue
if image.page_number <= 0 or image.page_number > self.obj.survey.questionnaire.page_count:
log.warn(_("Page number %i for page %s, %i is out of range.") %
(image.page_number, image.filename, image.tiff_page))
failed_pages.add(i)
continue
pages.add(image.page_number)
# Figure out the suvey ID if neccessary
# *************************************
if self.obj.survey.defs.print_survey_id:
for page, image in enumerate(self.obj.images):
try:
if not duplex_mode or (image.page_number is not None and image.page_number % 2 == 0):
image.recognize.calculate_survey_id()
else:
image.survey_id = None
except RecognitionError:
log.warn(_('%s, %i: Could not read survey ID, but should be able to.') %
(image.filename, image.tiff_page))
failed_pages.add(page)
self.duplex_copy_image_attr(failed_pages, "survey_id", _("Could not read survey ID of either %s, %i or %s, %i!"))
# Simply use the survey ID from the first image globally
self.obj.survey_id = self.obj.images[0].survey_id
if self.obj.survey_id != self.obj.survey.survey_id:
# Broken survey ID ...
log.warn(_("Got a wrong survey ID (%s, %i)! It is %s, but should be %i.") %
(self.obj.images[0].filename,
self.obj.images[0].tiff_page,
self.obj.survey_id,
self.obj.survey.survey_id))
self.obj.valid = 0
else:
# Assume that the data is from the correct survey
self.obj.survey_id = self.obj.survey.survey_id
for image in self.obj.images:
image.survey_id = self.obj.survey.survey_id
# Figure out the questionnaire ID if neccessary
# *********************************************
if self.obj.survey.defs.print_questionnaire_id:
questionnaire_ids = []
for page, image in enumerate(self.obj.images):
try:
if not duplex_mode or (image.page_number is not None and image.page_number % 2 == 0):
image.recognize.calculate_questionnaire_id()
except RecognitionError:
log.warn(_('%s, %i: Could not read questionnaire ID, but should be able to.') % \
(image.filename, image.tiff_page))
failed_pages.add(page)
if image.questionnaire_id is not None:
questionnaire_ids.append(image.questionnaire_id)
self.duplex_copy_image_attr(failed_pages, "questionnaire_id", _("Could not read questionnaire ID of either %s, %i or %s, %i!"))
if len(questionnaire_ids):
self.obj.questionnaire_id = questionnaire_ids
else:
self.obj.questionnaire_id
# Try to load the global ID. If it does not exist we will get None, if
# it does, then it will be non-None. We don't care much about it
# internally anyways.
# However, we do want to ensure that it is the same everywhere if it
# can be read in.
# *********************************************
for page, image in enumerate(self.obj.images):
try:
if not duplex_mode or (image.page_number is not None and image.page_number % 2 == 0):
image.recognize.calculate_global_id()
except RecognitionError:
pass
self.duplex_copy_image_attr(failed_pages, "global_id")
self.obj.global_id = self.obj.images[0].global_id
for image in self.obj.images:
if self.obj.global_id != image.global_id or \
self.obj.survey_id != image.survey_id or \
self.obj.questionnaire_id != image.questionnaire_id:
if not warned_multipage_not_correctly_scanned:
log.warn(_("Got different IDs on different pages for at least one sheet! Do *NOT* try to use filters with this survey! You have to run a \"reorder\" step for this to work properly!"))
warned_multipage_not_correctly_scanned = True
# Done
if failed_pages:
self.obj.valid = 0
def setup(survey, cmdline):
if os.access(survey.path(), os.F_OK):
log.error(_('The survey directory already exists.'))
return 1
questionnaire_tex = cmdline['questionnaire.tex']
additionalqobjects = cmdline['additional_questions']
mime = mimetype(questionnaire_tex)
if mime != 'text/x-tex' and mime != '':
log.warn(
_('Unknown file type (%s). questionnaire_tex should be of type text/x-tex.'
) % mime)
log.warn(_('Will keep going, but expect failure!'))
if additionalqobjects is not None:
mime = mimetype(additionalqobjects)
if mime != 'text/plain' and mime != '':
log.error(
_('Unknown file type (%s). additionalqobjects should be text/plain.'
) % mime)
return 1
# Add the new questionnaire
survey.add_questionnaire(model.questionnaire.Questionnaire())
# Create the survey directory, and copy the tex file.
os.mkdir(survey.path())
try:
shutil.copy(questionnaire_tex, survey.path('questionnaire.tex'))
write_latex_override_file(survey, draft=True)
# Copy class and dictionary files
if paths.local_run:
cls_file = os.path.join(paths.source_dir, 'tex', 'sdaps.cls')
code128_file = os.path.join(paths.source_dir, 'tex', 'code128.tex')
dict_files = os.path.join(paths.build_dir, 'tex', '*.dict')
dict_files = glob.glob(dict_files)
else:
cls_file = os.path.join(paths.prefix, 'share', 'sdaps', 'tex',
'sdaps.cls')
code128_file = os.path.join(paths.prefix, 'share', 'sdaps', 'tex',
'code128.tex')
dict_files = os.path.join(paths.prefix, 'share', 'sdaps', 'tex',
'*.dict')
dict_files = glob.glob(dict_files)
shutil.copyfile(cls_file, survey.path('sdaps.cls'))
shutil.copyfile(code128_file, survey.path('code128.tex'))
for dict_file in dict_files:
shutil.copyfile(dict_file,
survey.path(os.path.basename(dict_file)))
for add_file in cmdline['add']:
shutil.copyfile(add_file, survey.path(os.path.basename(add_file)))
print _("Running %s now twice to generate the questionnaire."
) % defs.latex_engine
subprocess.call([
defs.latex_engine, '-halt-on-error', '-interaction', 'batchmode',
'questionnaire.tex'
],
cwd=survey.path())
# And again, without the draft mode
subprocess.call([
defs.latex_engine, '-halt-on-error', '-interaction', 'batchmode',
'questionnaire.tex'
],
cwd=survey.path())
if not os.path.exists(survey.path('questionnaire.pdf')):
print _("Error running \"%s\" to compile the LaTeX file."
) % defs.latex_engine
raise AssertionError('PDF file not generated')
survey.defs.print_questionnaire_id = False
survey.defs.print_survey_id = True
# Parse qobjects
try:
sdapsfileparser.parse(survey)
except Exception, e:
log.error(
_("Caught an Exception while parsing the SDAPS file. The current state is:"
))
print >> sys.stderr, unicode(survey.questionnaire)
print >> sys.stderr, "------------------------------------"
raise e
# Parse additionalqobjects
if additionalqobjects:
additionalparser.parse(survey, additionalqobjects)
# Last but not least calculate the survey id
survey.calculate_survey_id()
if not survey.check_settings():
log.error(
_("Some combination of options and project properties do not work. Aborted Setup."
))
shutil.rmtree(survey.path())
return 1
# We need to now rebuild everything so that the correct ID is at the bottom
write_latex_override_file(survey)
print _("Running %s now twice to generate the questionnaire."
) % defs.latex_engine
os.remove(survey.path('questionnaire.pdf'))
subprocess.call([
defs.latex_engine, '-halt-on-error', '-interaction', 'batchmode',
'questionnaire.tex'
],
cwd=survey.path())
# And again, without the draft mode
subprocess.call([
defs.latex_engine, '-halt-on-error', '-interaction', 'batchmode',
'questionnaire.tex'
],
cwd=survey.path())
if not os.path.exists(survey.path('questionnaire.pdf')):
print _("Error running \"%s\" to compile the LaTeX file."
) % defs.latex_engine
raise AssertionError('PDF file not generated')
# Print the result
print survey.title
for item in survey.info.items():
print u'%s: %s' % item
print unicode(survey.questionnaire)
log.logfile.open(survey.path('log'))
survey.save()
log.logfile.close()
import os
import sys
from sdaps import paths
from sdaps import defs
from sdaps import log
from sdaps.utils.ugettext import ugettext, ungettext
_ = ugettext
if paths.local_run:
# image.so liegt in lib_build_dir/image/
__path__.append(os.path.join(paths.lib_build_dir, 'image'))
# If SDAPS is installed, then the image.so file is in the current directory.
# Simply importing it without changes to the paths will work.
try:
from image import *
except ImportError, e:
print e
log.error(_("It appears you have not build the C extension. Please run \"./setup.py build\" in the toplevel directory."))
sys.exit(1)
set_magic_values(defs.corner_mark_min_length,
defs.corner_mark_max_length,
defs.image_line_width,
defs.corner_mark_search_distance,
defs.image_line_coverage)
def create_stamp_pdf(survey, output_filename, questionnaire_ids):
if questionnaire_ids is None:
log.warn(
_(
"There should be no need to stamp a SDAPS Project that uses LaTeX and does not have different questionnaire IDs printed on each sheet.\nI am going to do so anyways."
)
)
# Temporary directory for TeX files.
tmpdir = tempfile.mkdtemp()
try:
# Copy class and dictionary files
tex_file = survey.path("questionnaire.tex")
code128_file = survey.path("code128.tex")
cls_file = survey.path("sdaps.cls")
dict_files = survey.path("*.dict")
dict_files = glob.glob(dict_files)
shutil.copyfile(tex_file, os.path.join(tmpdir, "questionnaire.tex"))
shutil.copyfile(code128_file, os.path.join(tmpdir, "code128.tex"))
shutil.copyfile(cls_file, os.path.join(tmpdir, "sdaps.cls"))
for dict_file in dict_files:
shutil.copyfile(dict_file, os.path.join(tmpdir, os.path.basename(dict_file)))
latex_override = open(os.path.join(tmpdir, "report.tex"), "w")
# Similar to setuptex/setup.py, but we also set questionnaire IDs
latex_override = open(os.path.join(tmpdir, "sdaps.opt"), "w")
latex_override.write('% This file exists to force the latex document into "final" mode.\n')
latex_override.write("% It is parsed after the setup phase of the SDAPS class.\n\n")
latex_override.write("\setcounter{surveyidlshw}{%i}\n" % (survey.survey_id % (2 ** 16)))
latex_override.write("\setcounter{surveyidmshw}{%i}\n" % (survey.survey_id / (2 ** 16)))
latex_override.write("\def\surveyid{%i}\n" % (survey.survey_id))
latex_override.write(
"\def\globalid{%s}\n" % (tex_quote_braces(survey.global_id)) if survey.global_id is not None else ""
)
latex_override.write("\\@STAMPtrue\n")
latex_override.write("\\@PAGEMARKtrue\n")
latex_override.write("\\@sdaps@draftfalse\n")
if questionnaire_ids is not None:
quoted_ids = [tex_quote_braces(str(id)) for id in questionnaire_ids]
latex_override.write("\def\questionnaireids{{%s}}\n" % "},{".join(quoted_ids))
latex_override.close()
print _("Running %s now twice to generate the stamped questionnaire.") % defs.latex_engine
os.environ["TEXINPUTS"] = ":" + os.path.abspath(survey.path())
subprocess.call(
[
defs.latex_engine,
"-halt-on-error",
"-interaction",
"batchmode",
os.path.join(tmpdir, "questionnaire.tex"),
],
cwd=tmpdir,
)
# And again
subprocess.call(
[
defs.latex_engine,
"-halt-on-error",
"-interaction",
"batchmode",
os.path.join(tmpdir, "questionnaire.tex"),
],
cwd=tmpdir,
)
if not os.path.exists(os.path.join(tmpdir, "questionnaire.pdf")):
log.error(_('Error running "%s" to compile the LaTeX file.') % defs.latex_engine)
raise AssertionError("PDF file not generated")
shutil.move(os.path.join(tmpdir, "questionnaire.pdf"), output_filename)
except:
log.error(_("An error occured during creation of the report. Temporary files left in '%s'." % tmpdir))
raise
shutil.rmtree(tmpdir)
def create_stamp_pdf(survey, output_filename, questionnaire_ids):
sheets = 1 if questionnaire_ids is None else len(questionnaire_ids)
questionnaire_length = survey.questionnaire.page_count
have_pdftk = False
# Test if pdftk is present, if it is we can use it to be faster
try:
result = subprocess.Popen(['pdftk', '--version'], stdout=subprocess.PIPE)
# Just assume pdftk is there, if it was executed sucessfully
if result is not None:
have_pdftk = True
except OSError:
pass
if not have_pdftk:
try:
import pyPdf
except:
log.error(_(u'You need to have either pdftk or pyPdf installed. pdftk is the faster method.'))
sys.exit(1)
# Write the "stamp" out to tmp.pdf if are using pdftk.
if have_pdftk:
stampsfile = file(survey.path('tmp.pdf'), 'wb')
else:
stampsfile = StringIO.StringIO()
canvas = \
reportlab.pdfgen.canvas.Canvas(stampsfile,
bottomup=False,
pagesize=(survey.defs.paper_width * mm,
survey.defs.paper_height * mm))
# bottomup = False =>(0, 0) is the upper left corner
print ungettext(u'Creating stamp PDF for %i sheet', u'Creating stamp PDF for %i sheets', sheets) % sheets
log.progressbar.start(sheets)
for i in range(sheets):
id = questionnaire_ids.pop(0)
for j in range(questionnaire_length):
if survey.defs.style == "classic":
draw_corner_marks(survey, canvas)
draw_corner_boxes(survey, canvas, j)
if not survey.defs.duplex or j % 2:
if questionnaire_ids is not None:
draw_questionnaire_id(canvas, survey, id)
if survey.defs.print_survey_id:
draw_survey_id(canvas, survey)
elif survey.defs.style == "code128":
draw_corner_marks(survey, canvas)
if not survey.defs.duplex or j % 2:
if questionnaire_ids is not None:
draw_code128_questionnaire_id(canvas, survey, id)
# Survey ID has to be printed in CODE128 mode, because it
# contains the page number and rotation.
draw_code128_sdaps_info(canvas, survey, j + 1)
if survey.global_id is not None:
draw_code128_global_id(canvas, survey)
else:
raise AssertionError()
canvas.showPage()
log.progressbar.update(i + 1)
canvas.save()
print ungettext(u'%i sheet; %f seconds per sheet', u'%i sheet; %f seconds per sheet', log.progressbar.max_value) % (
log.progressbar.max_value,
float(log.progressbar.elapsed_time) /
float(log.progressbar.max_value)
)
if have_pdftk:
stampsfile.close()
# Merge using pdftk
print _("Stamping using pdftk")
tmp_dir = tempfile.mkdtemp()
for page in xrange(1, questionnaire_length + 1):
print ungettext(u"pdftk: Splitting out page %d of each sheet.", u"pdftk: Splitting out page %d of each sheet.", page) % page
args = []
args.append('pdftk')
args.append(survey.path('tmp.pdf'))
args.append('cat')
cur = page
for i in range(sheets):
args.append('%d' % cur)
cur += questionnaire_length
args.append('output')
args.append(os.path.join(tmp_dir, 'stamp-%d.pdf' % page))
subprocess.call(args)
print _(u"pdftk: Splitting the questionnaire for watermarking.")
subprocess.call(['pdftk', survey.path('questionnaire.pdf'),
'dump_data', 'output',
os.path.join(tmp_dir, 'doc_data.txt')])
for page in xrange(1, questionnaire_length + 1):
subprocess.call(['pdftk', survey.path('questionnaire.pdf'), 'cat',
'%d' % page, 'output',
os.path.join(tmp_dir, 'watermark-%d.pdf' % page)])
for page in xrange(1, questionnaire_length + 1):
print ungettext(u"pdftk: Watermarking page %d of all sheets.", u"pdftk: Watermarking page %d of all sheets.", page) % page
subprocess.call(['pdftk',
os.path.join(tmp_dir, 'stamp-%d.pdf' % page),
'background',
os.path.join(tmp_dir, 'watermark-%d.pdf' % page),
'output',
os.path.join(tmp_dir, 'watermarked-%d.pdf' % page)])
args = []
args.append('pdftk')
for page in xrange(1, questionnaire_length + 1):
char = chr(ord('A') + page - 1)
args.append('%s=' % char + os.path.join(tmp_dir, 'watermarked-%d.pdf' % page))
args.append('cat')
for i in range(sheets):
for page in xrange(1, questionnaire_length + 1):
char = chr(ord('A') + page - 1)
args.append('%s%d' % (char, i + 1))
args.append('output')
args.append(os.path.join(tmp_dir, 'final.pdf'))
print _(u"pdftk: Assembling everything into the final PDF.")
subprocess.call(args)
subprocess.call(['pdftk', os.path.join(tmp_dir, 'final.pdf'),
'update_info', os.path.join(tmp_dir, 'doc_data.txt'),
'output', output_filename])
# Remove tmp.pdf
os.unlink(survey.path('tmp.pdf'))
# Remove all the temporary files
shutil.rmtree(tmp_dir)
else:
# Merge using pyPdf
stamped = pyPdf.PdfFileWriter()
stamped._info.getObject().update({
pyPdf.generic.NameObject('/Producer'): pyPdf.generic.createStringObject(u'sdaps'),
pyPdf.generic.NameObject('/Title'): pyPdf.generic.createStringObject(survey.title),
})
subject = []
for key, value in survey.info.iteritems():
subject.append(u'%(key)s: %(value)s' % {'key': key, 'value': value})
subject = u'\n'.join(subject)
stamped._info.getObject().update({
pyPdf.generic.NameObject('/Subject'): pyPdf.generic.createStringObject(subject),
})
stamps = pyPdf.PdfFileReader(stampsfile)
del stampsfile
questionnaire = pyPdf.PdfFileReader(
file(survey.path('questionnaire.pdf'), 'rb')
)
print _(u'Stamping using pyPdf. For faster stamping, install pdftk.')
log.progressbar.start(sheets)
for i in range(sheets):
for j in range(questionnaire_length):
s = stamps.getPage(i * questionnaire_length + j)
if not have_pdftk:
q = questionnaire.getPage(j)
s.mergePage(q)
stamped.addPage(s)
log.progressbar.update(i + 1)
stamped.write(open(output_filename, 'wb'))
print ungettext(u'%i sheet; %f seconds per sheet', u'%i sheet; %f seconds per sheet',
log.progressbar.max_value) % (
log.progressbar.max_value,
float(log.progressbar.elapsed_time) /
float(log.progressbar.max_value))
def setup(survey_dir, questionnaire_tex, engine, additionalqobjects=None, extra_files=[]):
if os.access(survey_dir, os.F_OK):
log.error(_('The survey directory already exists.'))
return 1
mime = mimetype(questionnaire_tex)
if mime != 'text/x-tex' and mime != '':
log.warn(_('Unknown file type (%s). questionnaire_tex should be of type text/x-tex.') % mime)
log.warn(_('Will keep going, but expect failure!'))
if additionalqobjects is not None:
mime = mimetype(additionalqobjects)
if mime != 'text/plain' and mime != '':
log.error(_('Unknown file type (%s). additionalqobjects should be text/plain.') % mime)
return 1
# Create the survey directory, and copy the tex file.
survey = model.survey.Survey.new(survey_dir)
survey.defs.engine = engine
# Add the new questionnaire
survey.add_questionnaire(model.questionnaire.Questionnaire())
try:
shutil.copy(questionnaire_tex, survey.path('questionnaire.tex'))
latex.write_override(survey, survey.path('sdaps.opt'), draft=True)
# Copy class and dictionary files
if paths.local_run:
cls_extra_files = os.path.join(paths.source_dir, 'tex', '*.cls')
cls_files = os.path.join(paths.source_dir, 'tex', 'class', 'build', 'local', '*.cls')
tex_files = os.path.join(paths.source_dir, 'tex', 'class', 'build', 'local', '*.tex')
sty_files = os.path.join(paths.source_dir, 'tex', 'class', 'build', 'local', '*.sty')
dict_files = os.path.join(paths.build_dir, 'tex', '*.dict')
else:
cls_extra_files = None
cls_files = os.path.join(paths.prefix, 'share', 'sdaps', 'tex', '*.cls')
tex_files = os.path.join(paths.prefix, 'share', 'sdaps', 'tex', '*.tex')
sty_files = os.path.join(paths.prefix, 'share', 'sdaps', 'tex', '*.sty')
dict_files = os.path.join(paths.prefix, 'share', 'sdaps', 'tex', '*.dict')
def copy_to_survey(files_glob):
files = glob.glob(files_glob)
for file in files:
shutil.copyfile(file, survey.path(os.path.basename(file)))
if cls_extra_files is not None:
copy_to_survey(cls_extra_files)
copy_to_survey(cls_files)
copy_to_survey(tex_files)
copy_to_survey(sty_files)
copy_to_survey(dict_files)
for add_file in extra_files:
if os.path.isdir(add_file):
shutil.copytree(add_file, survey.path(os.path.basename(add_file)))
else:
shutil.copyfile(add_file, survey.path(os.path.basename(add_file)))
print(_("Running %s now multiple times to generate the questionnaire.") % survey.defs.engine)
latex.compile(survey.defs.engine, 'questionnaire.tex', cwd=survey.path())
if not os.path.exists(survey.path('questionnaire.pdf')):
print(_("Error running \"%s\" to compile the LaTeX file.") % defs.latex_engine)
raise AssertionError('PDF file not generated')
survey.defs.print_questionnaire_id = False
survey.defs.print_survey_id = True
# Parse qobjects
try:
sdapsfileparser.parse(survey)
for qobject in survey.questionnaire.qobjects:
qobject.setup.setup()
qobject.setup.validate()
except:
log.error(_("Caught an Exception while parsing the SDAPS file. The current state is:"))
print(str(survey.questionnaire), file=sys.stderr)
print("------------------------------------", file=sys.stderr)
raise
# Parse additionalqobjects
if additionalqobjects:
additionalparser.parse(survey, additionalqobjects)
# Last but not least calculate the survey id
survey.calculate_survey_id()
if not survey.check_settings():
log.error(_("Some combination of options and project properties do not work. Aborted Setup."))
shutil.rmtree(survey.path())
return 1
# We need to now rebuild everything so that the correct ID is at the bottom
# Dissable draft mode if the survey doesn't have questionnaire IDs
latex.write_override(survey, survey.path('sdaps.opt'), draft=survey.defs.print_questionnaire_id)
print(_("Running %s now multiple imes to generate the questionnaire.") % survey.defs.engine)
os.remove(survey.path('questionnaire.pdf'))
latex.compile(survey.defs.engine, 'questionnaire.tex', survey.path())
if not os.path.exists(survey.path('questionnaire.pdf')):
print(_("Error running \"%s\" to compile the LaTeX file.") % survey.defs.engine)
raise AssertionError('PDF file not generated')
# Print the result
print(survey.title)
for item in list(survey.info.items()):
print('%s: %s' % item)
log.logfile.open(survey.path('log'))
survey.save()
log.logfile.close()
except:
log.error(_("An error occured in the setup routine. The survey directory still exists. You can for example check the questionnaire.log file for LaTeX compile errors."))
raise
def setup(survey, questionnaire_tex, additionalqobjects=None, extra_files=[]):
if os.access(survey.path(), os.F_OK):
log.error(_('The survey directory already exists.'))
return 1
mime = mimetype(questionnaire_tex)
if mime != 'text/x-tex' and mime != '':
log.warn(
_('Unknown file type (%s). questionnaire_tex should be of type text/x-tex.'
) % mime)
log.warn(_('Will keep going, but expect failure!'))
if additionalqobjects is not None:
mime = mimetype(additionalqobjects)
if mime != 'text/plain' and mime != '':
log.error(
_('Unknown file type (%s). additionalqobjects should be text/plain.'
) % mime)
return 1
# Add the new questionnaire
survey.add_questionnaire(model.questionnaire.Questionnaire())
# Create the survey directory, and copy the tex file.
os.makedirs(survey.path())
try:
shutil.copy(questionnaire_tex, survey.path('questionnaire.tex'))
latex.write_override(survey, survey.path('sdaps.opt'), draft=True)
# Copy class and dictionary files
if paths.local_run:
cls_extra_files = os.path.join(paths.source_dir, 'tex', '*.cls')
cls_files = os.path.join(paths.source_dir, 'tex', 'class', 'build',
'local', '*.cls')
tex_files = os.path.join(paths.source_dir, 'tex', 'class', 'build',
'local', '*.tex')
sty_files = os.path.join(paths.source_dir, 'tex', 'class', 'build',
'local', '*.sty')
dict_files = os.path.join(paths.build_dir, 'tex', '*.dict')
else:
cls_extra_files = None
cls_files = os.path.join(paths.prefix, 'share', 'sdaps', 'tex',
'*.cls')
tex_files = os.path.join(paths.prefix, 'share', 'sdaps', 'tex',
'*.tex')
sty_files = os.path.join(paths.prefix, 'share', 'sdaps', 'tex',
'*.sty')
dict_files = os.path.join(paths.prefix, 'share', 'sdaps', 'tex',
'*.dict')
def copy_to_survey(files_glob):
files = glob.glob(files_glob)
for file in files:
shutil.copyfile(file, survey.path(os.path.basename(file)))
if cls_extra_files is not None:
copy_to_survey(cls_extra_files)
copy_to_survey(cls_files)
copy_to_survey(tex_files)
copy_to_survey(sty_files)
copy_to_survey(dict_files)
for add_file in extra_files:
if os.path.isdir(add_file):
shutil.copytree(add_file,
survey.path(os.path.basename(add_file)))
else:
shutil.copyfile(add_file,
survey.path(os.path.basename(add_file)))
print _("Running %s now twice to generate the questionnaire."
) % defs.latex_engine
latex.compile('questionnaire.tex', cwd=survey.path())
if not os.path.exists(survey.path('questionnaire.pdf')):
print _("Error running \"%s\" to compile the LaTeX file."
) % defs.latex_engine
raise AssertionError('PDF file not generated')
survey.defs.print_questionnaire_id = False
survey.defs.print_survey_id = True
# Parse qobjects
try:
sdapsfileparser.parse(survey)
for qobject in survey.questionnaire.qobjects:
qobject.setup.setup()
qobject.setup.validate()
except:
log.error(
_("Caught an Exception while parsing the SDAPS file. The current state is:"
))
print >> sys.stderr, unicode(survey.questionnaire)
print >> sys.stderr, "------------------------------------"
raise
# Parse additionalqobjects
if additionalqobjects:
additionalparser.parse(survey, additionalqobjects)
# Last but not least calculate the survey id
survey.calculate_survey_id()
if not survey.check_settings():
log.error(
_("Some combination of options and project properties do not work. Aborted Setup."
))
shutil.rmtree(survey.path())
return 1
# We need to now rebuild everything so that the correct ID is at the bottom
# Dissable draft mode if the survey doesn't have questionnaire IDs
latex.write_override(survey,
survey.path('sdaps.opt'),
draft=survey.defs.print_questionnaire_id)
print _("Running %s now twice to generate the questionnaire."
) % defs.latex_engine
os.remove(survey.path('questionnaire.pdf'))
latex.compile('questionnaire.tex', survey.path())
if not os.path.exists(survey.path('questionnaire.pdf')):
print _("Error running \"%s\" to compile the LaTeX file."
) % defs.latex_engine
raise AssertionError('PDF file not generated')
# Print the result
print survey.title
for item in survey.info.items():
print u'%s: %s' % item
print unicode(survey.questionnaire)
log.logfile.open(survey.path('log'))
survey.save()
log.logfile.close()
except:
log.error(
_("An error occured in the setup routine. The survey directory still exists. You can for example check the questionnaire.log file for LaTeX compile errors."
))
raise
def setup(survey, cmdline):
if os.access(survey.path(), os.F_OK):
log.error(_('The survey directory already exists.'))
return 1
questionnaire_odt = cmdline['questionnaire.odt']
questionnaire_pdf = cmdline['questionnaire.pdf']
additionalqobjects = cmdline['additional_questions']
mime = mimetype(questionnaire_odt)
if mime != 'application/vnd.oasis.opendocument.text' and mime != '':
log.error(
_('Unknown file type (%s). questionnaire_odt should be application/vnd.oasis.opendocument.text.'
) % mime)
return 1
mime = mimetype(questionnaire_pdf)
if mime != 'application/pdf' and mime != '':
log.error(
_('Unknown file type (%s). questionnaire_pdf should be application/pdf.'
) % mime)
return 1
if additionalqobjects is not None:
mime = mimetype(additionalqobjects)
if mime != 'text/plain' and mime != '':
log.error(
_('Unknown file type (%s). additionalqobjects should be text/plain.'
) % mime)
return 1
# Add the new questionnaire
survey.add_questionnaire(model.questionnaire.Questionnaire())
# Parse the box objects into a cache
boxes, page_count = boxesparser.parse(questionnaire_pdf)
survey.questionnaire.page_count = page_count
# Get the papersize
doc = pdffile.PDFDocument(questionnaire_pdf)
page = doc.read_page(1)
survey.defs.paper_width = abs(page.MediaBox[0] -
page.MediaBox[2]) / 72.0 * 25.4
survey.defs.paper_height = abs(page.MediaBox[1] -
page.MediaBox[3]) / 72.0 * 25.4
survey.defs.print_questionnaire_id = cmdline['print_questionnaire_id']
survey.defs.print_survey_id = cmdline['print_survey_id']
survey.defs.style = cmdline['style']
# Force simplex if page count is one.
survey.defs.duplex = False if page_count == 1 else cmdline['duplex']
survey.global_id = cmdline['global_id']
# Parse qobjects
try:
qobjectsparser.parse(survey, questionnaire_odt, boxes)
except:
log.error(
_("Caught an Exception while parsing the ODT file. The current state is:"
))
print unicode(survey.questionnaire)
print "------------------------------------"
print _(
"If the dependencies for the \"annotate\" command are installed, then an annotated version will be created next to the original PDF file."
)
print "------------------------------------"
# Try to make an annotation
try:
if questionnaire_pdf.lower().endswith('.pdf'):
annotated_pdf = questionnaire_pdf[:-4] + '_annotated.pdf'
else:
# No .pdf ending? Just append the _annotated.pdf.
annotated_pdf = questionnaire_pdf + '_annotated.pdf'
import sdaps.annotate.annotate as annotate
annotate.annotate(survey, questionnaire_pdf, annotated_pdf)
except:
# Well, whatever
pass
raise
# Parse additionalqobjects
if additionalqobjects:
additionalparser.parse(survey, additionalqobjects)
# Parse Metadata
metaparser.parse(survey, questionnaire_odt)
# Last but not least calculate the survey id
survey.calculate_survey_id()
if not survey.check_settings():
log.error(
_("Some combination of options and project properties do not work. Aborted Setup."
))
return 1
# Print the result
print survey.title
for item in survey.info.items():
print u'%s: %s' % item
print unicode(survey.questionnaire)
# Create the survey
os.mkdir(survey.path())
log.logfile.open(survey.path('log'))
shutil.copy(questionnaire_odt, survey.path('questionnaire.odt'))
shutil.copy(questionnaire_pdf, survey.path('questionnaire.pdf'))
survey.save()
log.logfile.close()
def setup(survey, questionnaire_odt, questionnaire_pdf, additionalqobjects, options):
if os.access(survey.path(), os.F_OK):
log.error(_('The survey directory already exists.'))
return 1
mime = mimetype(questionnaire_odt)
if mime != 'application/vnd.oasis.opendocument.text' and mime not in ['', 'binary', 'application/octet-stream']:
log.error(_('Unknown file type (%s). questionnaire_odt should be application/vnd.oasis.opendocument.text.') % mime)
return 1
mime = mimetype(questionnaire_pdf)
if mime != 'application/pdf' and mime != '':
log.error(_('Unknown file type (%s). questionnaire_pdf should be application/pdf.') % mime)
return 1
if additionalqobjects is not None:
mime = mimetype(additionalqobjects)
if mime != 'text/plain' and mime != '':
log.error(_('Unknown file type (%s). additionalqobjects should be text/plain.') % mime)
return 1
# Add the new questionnaire
survey.add_questionnaire(model.questionnaire.Questionnaire())
# Parse the box objects into a cache
boxes, page_count = boxesparser.parse(questionnaire_pdf)
survey.questionnaire.page_count = page_count
# Get the papersize
doc = pdffile.PDFDocument(questionnaire_pdf)
page = doc.read_page(1)
survey.defs.paper_width = abs(page.MediaBox[0] - page.MediaBox[2]) / 72.0 * 25.4
survey.defs.paper_height = abs(page.MediaBox[1] - page.MediaBox[3]) / 72.0 * 25.4
survey.defs.print_questionnaire_id = options['print_questionnaire_id']
survey.defs.print_survey_id = options['print_survey_id']
survey.defs.style = options['style']
# Force simplex if page count is one.
survey.defs.duplex = False if page_count == 1 else options['duplex']
survey.global_id = options['global_id']
# Parse qobjects
try:
qobjectsparser.parse(survey, questionnaire_odt, boxes)
except:
log.error(_("Caught an Exception while parsing the ODT file. The current state is:"))
print unicode(survey.questionnaire)
print "------------------------------------"
print _("If the dependencies for the \"annotate\" command are installed, then an annotated version will be created next to the original PDF file.")
print "------------------------------------"
# Try to make an annotation
try:
if questionnaire_pdf.lower().endswith('.pdf'):
annotated_pdf = questionnaire_pdf[:-4] + '_annotated.pdf'
else:
# No .pdf ending? Just append the _annotated.pdf.
annotated_pdf = questionnaire_pdf + '_annotated.pdf'
import sdaps.annotate.annotate as annotate
annotate.annotate(survey, questionnaire_pdf, annotated_pdf)
except:
# Well, whatever
pass
raise
# Parse additionalqobjects
if additionalqobjects:
additionalparser.parse(survey, additionalqobjects)
# Parse Metadata
metaparser.parse(survey, questionnaire_odt)
# Last but not least calculate the survey id
survey.calculate_survey_id()
if not survey.check_settings():
log.error(_("Some combination of options and project properties do not work. Aborted Setup."))
return 1
# Print the result
print survey.title
for item in survey.info.items():
print u'%s: %s' % item
print unicode(survey.questionnaire)
# Create the survey
os.mkdir(survey.path())
log.logfile.open(survey.path('log'))
shutil.copy(questionnaire_odt, survey.path('questionnaire.odt'))
shutil.copy(questionnaire_pdf, survey.path('questionnaire.pdf'))
survey.save()
log.logfile.close()
def add(cmdline):
import sys
from sdaps.add import add_image, check_image
from sdaps import image
error = False
survey = model.survey.Survey.load(cmdline['project'])
filelist = []
deletelist = []
if not cmdline['convert']:
for file in cmdline['images']:
filelist.append(file)
if not check_image(survey, file, cmdline['duplex'], cmdline['force'], message=True):
error=True
if error:
return
else:
if not cmdline['copy']:
log.error(_("The --no-copy option is not compatible with --convert!"))
return 1
try:
from sdaps.convert import convert_images
except:
log.error("Need to convert the images to monochrome TIFF, however the conversion module cannot be imported. You are likely missing the OpenCV dependency.")
return 1
print(_("Converting input files into a single temporary file."))
tmp = tempfile.mktemp(suffix='.tif', prefix='sdaps-convert-')
deletelist.append(tmp)
filelist.append(tmp)
# Run conversion
# TODO: Allow 3D transformation here!
try:
convert_images(cmdline['images'], tmp, survey.defs.paper_width, survey.defs.paper_height, cmdline['transform'])
if not check_image(survey, tmp, cmdline['duplex'], cmdline['force']):
log.error(_("The page count of the created temporary file does not work with this survey."))
raise AssertionError()
except Exception as e:
log.error(str(e))
log.error(_("Running the conversion failed."))
error = True
raise
if not error:
for file in filelist:
print(_('Processing %s') % file)
add_image(survey, file, cmdline['duplex'], cmdline['force'], cmdline['copy'])
print(_('Done'))
for file in deletelist:
try:
os.unlink(file)
except OSError:
pass
if error:
return 1
else:
survey.save()
return 0
def create_stamp_pdf(survey, output_filename, questionnaire_ids):
sheets = 1 if questionnaire_ids is None else len(questionnaire_ids)
questionnaire_length = survey.questionnaire.page_count
have_pdftk = False
# Test if pdftk is present, if it is we can use it to be faster
try:
result = subprocess.Popen(['pdftk', '--version'],
stdout=subprocess.PIPE)
# Just assume pdftk is there, if it was executed sucessfully
if result is not None:
have_pdftk = True
except OSError:
pass
if not have_pdftk:
try:
import pyPdf
except:
log.error(
_(u'You need to have either pdftk or pyPdf installed. pdftk is the faster method.'
))
sys.exit(1)
# Write the "stamp" out to tmp.pdf if are using pdftk.
if have_pdftk:
stampsfile = file(survey.path('tmp.pdf'), 'wb')
else:
stampsfile = StringIO.StringIO()
canvas = \
reportlab.pdfgen.canvas.Canvas(stampsfile,
bottomup=False,
pagesize=(survey.defs.paper_width * mm,
survey.defs.paper_height * mm))
# bottomup = False =>(0, 0) is the upper left corner
print ungettext(u'Creating stamp PDF for %i sheet',
u'Creating stamp PDF for %i sheets', sheets) % sheets
log.progressbar.start(sheets)
for i in range(sheets):
if questionnaire_ids is not None:
id = questionnaire_ids.pop(0)
for j in range(questionnaire_length):
if survey.defs.style == "classic":
draw_corner_marks(survey, canvas)
draw_corner_boxes(survey, canvas, j)
if not survey.defs.duplex or j % 2:
if questionnaire_ids is not None:
draw_questionnaire_id(canvas, survey, id)
if survey.defs.print_survey_id:
draw_survey_id(canvas, survey)
elif survey.defs.style == "code128":
draw_corner_marks(survey, canvas)
if not survey.defs.duplex or j % 2:
if questionnaire_ids is not None:
draw_code128_questionnaire_id(canvas, survey, id)
# Survey ID has to be printed in CODE128 mode, because it
# contains the page number and rotation.
draw_code128_sdaps_info(canvas, survey, j + 1)
if survey.global_id is not None:
draw_code128_global_id(canvas, survey)
elif survey.defs.style == "custom":
# Only draw corner marker
draw_corner_marks(survey, canvas)
pass
else:
raise AssertionError()
canvas.showPage()
log.progressbar.update(i + 1)
canvas.save()
print ungettext(
u'%i sheet; %f seconds per sheet', u'%i sheet; %f seconds per sheet',
log.progressbar.max_value) % (log.progressbar.max_value,
float(log.progressbar.elapsed_time) /
float(log.progressbar.max_value))
if have_pdftk:
stampsfile.close()
# Merge using pdftk
print _("Stamping using pdftk")
tmp_dir = tempfile.mkdtemp()
for page in xrange(1, questionnaire_length + 1):
print ungettext(u"pdftk: Splitting out page %d of each sheet.",
u"pdftk: Splitting out page %d of each sheet.",
page) % page
args = []
args.append('pdftk')
args.append(survey.path('tmp.pdf'))
args.append('cat')
cur = page
for i in range(sheets):
args.append('%d' % cur)
cur += questionnaire_length
args.append('output')
args.append(os.path.join(tmp_dir, 'stamp-%d.pdf' % page))
subprocess.call(args)
print _(u"pdftk: Splitting the questionnaire for watermarking.")
subprocess.call([
'pdftk',
survey.path('questionnaire.pdf'), 'dump_data', 'output',
os.path.join(tmp_dir, 'doc_data.txt')
])
for page in xrange(1, questionnaire_length + 1):
subprocess.call([
'pdftk',
survey.path('questionnaire.pdf'), 'cat',
'%d' % page, 'output',
os.path.join(tmp_dir, 'watermark-%d.pdf' % page)
])
for page in xrange(1, questionnaire_length + 1):
print ungettext(u"pdftk: Watermarking page %d of all sheets.",
u"pdftk: Watermarking page %d of all sheets.",
page) % page
subprocess.call([
'pdftk',
os.path.join(tmp_dir, 'stamp-%d.pdf' % page), 'background',
os.path.join(tmp_dir, 'watermark-%d.pdf' % page), 'output',
os.path.join(tmp_dir, 'watermarked-%d.pdf' % page)
])
args = []
args.append('pdftk')
for page in xrange(1, questionnaire_length + 1):
char = chr(ord('A') + page - 1)
args.append('%s=' % char +
os.path.join(tmp_dir, 'watermarked-%d.pdf' % page))
args.append('cat')
for i in range(sheets):
for page in xrange(1, questionnaire_length + 1):
char = chr(ord('A') + page - 1)
args.append('%s%d' % (char, i + 1))
args.append('output')
args.append(os.path.join(tmp_dir, 'final.pdf'))
print _(u"pdftk: Assembling everything into the final PDF.")
subprocess.call(args)
subprocess.call([
'pdftk',
os.path.join(tmp_dir, 'final.pdf'), 'update_info',
os.path.join(tmp_dir, 'doc_data.txt'), 'output', output_filename
])
# Remove tmp.pdf
os.unlink(survey.path('tmp.pdf'))
# Remove all the temporary files
shutil.rmtree(tmp_dir)
else:
# Merge using pyPdf
stamped = pyPdf.PdfFileWriter()
stamped._info.getObject().update({
pyPdf.generic.NameObject('/Producer'):
pyPdf.generic.createStringObject(u'sdaps'),
pyPdf.generic.NameObject('/Title'):
pyPdf.generic.createStringObject(survey.title),
})
subject = []
for key, value in survey.info.iteritems():
subject.append(u'%(key)s: %(value)s' % {
'key': key,
'value': value
})
subject = u'\n'.join(subject)
stamped._info.getObject().update({
pyPdf.generic.NameObject('/Subject'):
pyPdf.generic.createStringObject(subject),
})
stamps = pyPdf.PdfFileReader(stampsfile)
del stampsfile
questionnaire = pyPdf.PdfFileReader(
file(survey.path('questionnaire.pdf'), 'rb'))
print _(u'Stamping using pyPdf. For faster stamping, install pdftk.')
log.progressbar.start(sheets)
for i in range(sheets):
for j in range(questionnaire_length):
s = stamps.getPage(i * questionnaire_length + j)
if not have_pdftk:
q = questionnaire.getPage(j)
s.mergePage(q)
stamped.addPage(s)
log.progressbar.update(i + 1)
stamped.write(open(output_filename, 'wb'))
print ungettext(
u'%i sheet; %f seconds per sheet',
u'%i sheet; %f seconds per sheet',
log.progressbar.max_value) % (log.progressbar.max_value,
float(log.progressbar.elapsed_time) /
float(log.progressbar.max_value))
def build_survey(self, djsurvey_id):
"""Creates the SDAPS project and database for the survey.
This process should be run on an already initialized survey that
has a questionnaire written to it."""
djsurvey = get_object_or_404(models.Survey, pk=djsurvey_id)
assert (djsurvey.initialized == False)
lock_id = ('%s_build_survey' % djsurvey.id)
with task_lock(lock_id, self.app.oid) as acquired:
if acquired:
import sdaps.setuptex as setup
from sdaps.utils import latex
from sdaps.setuptex import sdapsfileparser
survey = model.survey.Survey.new(djsurvey.path)
latex.write_override(survey, survey.path('sdaps.opt'), draft=True)
if not utils.atomic_latex_compile(
djsurvey.path, 'questionnaire.tex', need_sdaps=True):
# XXX: The sqlite file should not be created immediately!
os.unlink(survey.path('survey.sqlite'))
return False
# We now have the .sdaps file that can be parsed
# Defaults
survey.defs.print_questionnaire_id = False
survey.defs.print_survey_id = True
survey.defs.engine = defs.latex_engine
survey.add_questionnaire(model.questionnaire.Questionnaire())
# Parse qobjects
try:
sdapsfileparser.parse(survey)
for qobject in survey.questionnaire.qobjects:
qobject.setup.setup()
qobject.setup.validate()
except:
log.error(
"Caught an Exception while parsing the SDAPS file. The current state is:"
)
print(str(survey.questionnaire), file=sys.stderr)
print("------------------------------------", file=sys.stderr)
raise AssertionError("Exception while parsing the SDAPS file.")
# Last but not least calculate the survey id
survey.calculate_survey_id()
if not survey.check_settings():
log.error(
"Some combination of options and project properties do not work. Aborted Setup."
)
os.unlink(survey.path('survey.sqlite'))
return False
latex.write_override(survey, survey.path('sdaps.opt'), draft=False)
if not utils.atomic_latex_compile(
djsurvey.path, 'questionnaire.tex', need_sdaps=True):
os.unlink(survey.path('survey.sqlite'))
return False
# TODO: If something goes wrong while initializing the survey,
# there should be an option to delete the files.
survey.save()
djsurvey.initialized = True
djsurvey.title = survey.title
if 'Author' in survey.info:
djsurvey.author = survey.info['Author']
djsurvey.save()
log.logfile.close()
import os
import sys
from sdaps import paths
from sdaps import defs
from sdaps import log
from sdaps.utils.ugettext import ugettext, ungettext
_ = ugettext
if paths.local_run:
# image.so liegt in lib_build_dir/image/
__path__.append(os.path.join(paths.lib_build_dir, 'image'))
# If SDAPS is installed, then the image.so file is in the current directory.
# Simply importing it without changes to the paths will work.
try:
from .image import *
except ImportError as e:
print(e)
log.error(
_("It appears you have not build the C extension. Please run \"./setup.py build\" in the toplevel directory."
))
sys.exit(1)
set_magic_values(defs.corner_mark_min_length, defs.corner_mark_max_length,
defs.image_line_width, defs.corner_mark_search_distance,
defs.image_line_coverage)
def create_stamp_pdf(survey, output_filename, questionnaire_ids):
sheets = 1 if questionnaire_ids is None else len(questionnaire_ids)
questionnaire_length = survey.questionnaire.page_count
have_pdftk = False
# Test if pdftk is present, if it is we can use it to be faster
try:
result = subprocess.Popen(["pdftk", "--version"], stdout=subprocess.PIPE)
# Just assume pdftk is there, if it was executed sucessfully
if result is not None:
have_pdftk = True
except OSError:
pass
if not have_pdftk:
try:
import pyPdf
except:
log.error(_(u"You need to have either pdftk or pyPdf installed. pdftk is the faster method."))
sys.exit(1)
# Write the "stamp" out to tmp.pdf if are using pdftk.
if have_pdftk:
stampsfile = file(survey.path("tmp.pdf"), "wb")
else:
stampsfile = StringIO.StringIO()
canvas = reportlab.pdfgen.canvas.Canvas(
stampsfile, bottomup=False, pagesize=(survey.defs.paper_width * mm, survey.defs.paper_height * mm)
)
# bottomup = False =>(0, 0) is the upper left corner
print ungettext(u"Creating stamp PDF for %i sheet", u"Creating stamp PDF for %i sheets", sheets) % sheets
log.progressbar.start(sheets)
for i in range(sheets):
if questionnaire_ids is not None:
id = questionnaire_ids.pop(0)
for j in range(questionnaire_length):
if survey.defs.style == "classic":
draw_corner_marks(survey, canvas)
draw_corner_boxes(survey, canvas, j)
if not survey.defs.duplex or j % 2:
if questionnaire_ids is not None:
draw_questionnaire_id(canvas, survey, id)
if survey.defs.print_survey_id:
draw_survey_id(canvas, survey)
elif survey.defs.style == "code128":
draw_corner_marks(survey, canvas)
if not survey.defs.duplex or j % 2:
if questionnaire_ids is not None:
draw_code128_questionnaire_id(canvas, survey, id)
# Survey ID has to be printed in CODE128 mode, because it
# contains the page number and rotation.
draw_code128_sdaps_info(canvas, survey, j + 1)
if survey.global_id is not None:
draw_code128_global_id(canvas, survey)
elif survey.defs.style == "custom":
# Only draw corner marker
draw_corner_marks(survey, canvas)
pass
else:
raise AssertionError()
canvas.showPage()
log.progressbar.update(i + 1)
canvas.save()
print ungettext(u"%i sheet; %f seconds per sheet", u"%i sheet; %f seconds per sheet", log.progressbar.max_value) % (
log.progressbar.max_value,
float(log.progressbar.elapsed_time) / float(log.progressbar.max_value),
)
if have_pdftk:
stampsfile.close()
# Merge using pdftk
print _("Stamping using pdftk")
tmp_dir = tempfile.mkdtemp()
if sheets == 1:
# Shortcut if we only have one sheet.
# In this case form data in the PDF will *not* break, in
# the other code path it *will* break.
print _(u"pdftk: Overlaying the original PDF with the markings.")
subprocess.call(
["pdftk", survey.path("questionnaire.pdf"), "stamp", survey.path("tmp.pdf"), "output", output_filename]
)
else:
for page in xrange(1, questionnaire_length + 1):
print ungettext(
u"pdftk: Splitting out page %d of each sheet.", u"pdftk: Splitting out page %d of each sheet.", page
) % page
args = []
args.append("pdftk")
args.append(survey.path("tmp.pdf"))
args.append("cat")
cur = page
for i in range(sheets):
args.append("%d" % cur)
cur += questionnaire_length
args.append("output")
args.append(os.path.join(tmp_dir, "stamp-%d.pdf" % page))
subprocess.call(args)
print _(u"pdftk: Splitting the questionnaire for watermarking.")
subprocess.call(
[
"pdftk",
survey.path("questionnaire.pdf"),
"dump_data",
"output",
os.path.join(tmp_dir, "doc_data.txt"),
]
)
for page in xrange(1, questionnaire_length + 1):
subprocess.call(
[
"pdftk",
survey.path("questionnaire.pdf"),
"cat",
"%d" % page,
"output",
os.path.join(tmp_dir, "watermark-%d.pdf" % page),
]
)
if sheets == 1:
for page in xrange(1, questionnaire_length + 1):
print ungettext(
u"pdftk: Watermarking page %d of all sheets.",
u"pdftk: Watermarking page %d of all sheets.",
page,
) % page
subprocess.call(
[
"pdftk",
os.path.join(tmp_dir, "stamp-%d.pdf" % page),
"background",
os.path.join(tmp_dir, "watermark-%d.pdf" % page),
"output",
os.path.join(tmp_dir, "watermarked-%d.pdf" % page),
]
)
else:
for page in xrange(1, questionnaire_length + 1):
print ungettext(
u"pdftk: Watermarking page %d of all sheets.",
u"pdftk: Watermarking page %d of all sheets.",
page,
) % page
subprocess.call(
[
"pdftk",
os.path.join(tmp_dir, "stamp-%d.pdf" % page),
"background",
os.path.join(tmp_dir, "watermark-%d.pdf" % page),
"output",
os.path.join(tmp_dir, "watermarked-%d.pdf" % page),
]
)
args = []
args.append("pdftk")
for page in xrange(1, questionnaire_length + 1):
char = chr(ord("A") + page - 1)
args.append("%s=" % char + os.path.join(tmp_dir, "watermarked-%d.pdf" % page))
args.append("cat")
for i in range(sheets):
for page in xrange(1, questionnaire_length + 1):
char = chr(ord("A") + page - 1)
args.append("%s%d" % (char, i + 1))
args.append("output")
args.append(os.path.join(tmp_dir, "final.pdf"))
print _(u"pdftk: Assembling everything into the final PDF.")
subprocess.call(args)
subprocess.call(
[
"pdftk",
os.path.join(tmp_dir, "final.pdf"),
"update_info",
os.path.join(tmp_dir, "doc_data.txt"),
"output",
output_filename,
]
)
# Remove tmp.pdf
os.unlink(survey.path("tmp.pdf"))
# Remove all the temporary files
shutil.rmtree(tmp_dir)
else:
# Merge using pyPdf
stamped = pyPdf.PdfFileWriter()
stamped._info.getObject().update(
{
pyPdf.generic.NameObject("/Producer"): pyPdf.generic.createStringObject(u"sdaps"),
pyPdf.generic.NameObject("/Title"): pyPdf.generic.createStringObject(survey.title),
}
)
subject = []
for key, value in survey.info.iteritems():
subject.append(u"%(key)s: %(value)s" % {"key": key, "value": value})
subject = u"\n".join(subject)
stamped._info.getObject().update(
{pyPdf.generic.NameObject("/Subject"): pyPdf.generic.createStringObject(subject)}
)
stamps = pyPdf.PdfFileReader(stampsfile)
del stampsfile
questionnaire = pyPdf.PdfFileReader(file(survey.path("questionnaire.pdf"), "rb"))
print _(u"Stamping using pyPdf. For faster stamping, install pdftk.")
log.progressbar.start(sheets)
for i in range(sheets):
for j in range(questionnaire_length):
s = stamps.getPage(i * questionnaire_length + j)
if not have_pdftk:
q = questionnaire.getPage(j)
s.mergePage(q)
stamped.addPage(s)
log.progressbar.update(i + 1)
stamped.write(open(output_filename, "wb"))
print ungettext(
u"%i sheet; %f seconds per sheet", u"%i sheet; %f seconds per sheet", log.progressbar.max_value
) % (log.progressbar.max_value, float(log.progressbar.elapsed_time) / float(log.progressbar.max_value))
def create_stamp_pdf(survey, output_filename, questionnaire_ids):
if questionnaire_ids is None:
log.warn(_("There should be no need to stamp a SDAPS Project that uses LaTeX and does not have different questionnaire IDs printed on each sheet.\nI am going to do so anyways."))
# Temporary directory for TeX files.
tmpdir = tempfile.mkdtemp()
try:
# Copy class and dictionary files
tex_file = survey.path('questionnaire.tex')
code128_file = survey.path('code128.tex')
cls_file = survey.path('sdaps.cls')
dict_files = survey.path('*.dict')
dict_files = glob.glob(dict_files)
shutil.copyfile(tex_file, os.path.join(tmpdir, 'questionnaire.tex'))
shutil.copyfile(code128_file, os.path.join(tmpdir, 'code128.tex'))
shutil.copyfile(cls_file, os.path.join(tmpdir, 'sdaps.cls'))
for dict_file in dict_files:
shutil.copyfile(dict_file, os.path.join(tmpdir, os.path.basename(dict_file)))
latex_override = open(os.path.join(tmpdir, 'report.tex'), 'w')
# Similar to setuptex/setup.py, but we also set questionnaire IDs
latex_override = open(os.path.join(tmpdir, 'sdaps.opt'), 'w')
latex_override.write('% This file exists to force the latex document into "final" mode.\n')
latex_override.write('% It is parsed after the setup phase of the SDAPS class.\n\n')
latex_override.write('\setcounter{surveyidlshw}{%i}\n' % (survey.survey_id % (2 ** 16)))
latex_override.write('\setcounter{surveyidmshw}{%i}\n' % (survey.survey_id / (2 ** 16)))
latex_override.write('\def\surveyid{%i}\n' % (survey.survey_id))
latex_override.write('\def\globalid{%s}\n' % (tex_quote_braces(survey.global_id)) if survey.global_id is not None else '')
latex_override.write('\\@STAMPtrue\n')
latex_override.write('\\@PAGEMARKtrue\n')
latex_override.write('\\@sdaps@draftfalse\n')
if questionnaire_ids is not None:
quoted_ids = [tex_quote_braces(str(id)) for id in questionnaire_ids]
latex_override.write('\def\questionnaireids{{%s}}\n' % '},{'.join(quoted_ids))
latex_override.close()
print _("Running %s now twice to generate the stamped questionnaire.") % defs.latex_engine
os.environ['TEXINPUTS'] = ':' + os.path.abspath(survey.path())
subprocess.call([defs.latex_engine, '-halt-on-error',
'-interaction', 'batchmode',
os.path.join(tmpdir, 'questionnaire.tex')],
cwd=tmpdir)
# And again
subprocess.call([defs.latex_engine, '-halt-on-error',
'-interaction', 'batchmode',
os.path.join(tmpdir, 'questionnaire.tex')],
cwd=tmpdir)
if not os.path.exists(os.path.join(tmpdir, 'questionnaire.pdf')):
log.error(_("Error running \"%s\" to compile the LaTeX file.") % defs.latex_engine)
raise AssertionError('PDF file not generated')
shutil.move(os.path.join(tmpdir, 'questionnaire.pdf'), output_filename)
except:
log.error(_("An error occured during creation of the report. Temporary files left in '%s'." % tmpdir))
raise
shutil.rmtree(tmpdir)
def recognize(self):
global warned_multipage_not_correctly_scanned
self.obj.valid = 1
duplex_mode = self.obj.survey.defs.duplex
# Load all images of this sheet
for image in self.obj.images:
if not image.ignored:
image.rotated = 0
image.surface.load()
failed_pages = set()
# Matrix recognition for all of them
matrix_errors = set()
for page, image in enumerate(self.obj.images):
try:
image.recognize.calculate_matrix()
except RecognitionError:
matrix_errors.add(page)
# We need to check the matrix_errors. Some are expected in simplex mode
for page in matrix_errors:
# in simplex mode every page will have a matrix; it might be a None
# matrix though
log.warn(
_('%s, %i: Matrix not recognized.') %
(self.obj.images[page].filename,
self.obj.images[page].tiff_page))
failed_pages.add(page)
# Rotation for all of them
for page, image in enumerate(self.obj.images):
try:
# This may set the rotation to "None" for unknown
image.recognize.calculate_rotation()
except RecognitionError:
log.warn(
_('%s, %i: Rotation not found.') %
(image.filename, image.tiff_page))
failed_pages.add(page)
# Copy the rotation over (if required) and print warning if the rotation is unknown
self.duplex_copy_image_attr(
failed_pages, 'rotated',
_("Neither %s, %i or %s, %i has a known rotation!"))
# Reload any image that is rotated.
for page, image in enumerate(self.obj.images):
if image.rotated and not image.ignored:
image.surface.load()
# And redo the whole matrix stuff ...
# XXX: It would be better to manipulate the matrix instead.
try:
image.recognize.calculate_matrix()
except RecognitionError:
if duplex_mode:
log.warn(
_('%s, %i: Matrix not recognized (again).') %
(image.filename, image.tiff_page))
failed_pages.add(page)
############
# At this point we can extract the page numbers and IDs as neccessary.
############
# Figure out the page numbers
# ***************************
for page, image in enumerate(self.obj.images):
try:
# This may set the page_number to "None" for unknown
image.recognize.calculate_page_number()
except RecognitionError:
log.warn(
_('%s, %i: Could not get page number.') %
(image.filename, image.tiff_page))
image.page_number = None
failed_pages.add(page)
i = 0
while i < len(self.obj.images):
# We try to recover at least the page number of failed pages
# this way.
# NOTE: In simplex mode dummy pages will be inserted, so one page
# always has no page number, and the other one has one.
# This is exactly what we want, so we don't need to do anything
# (except warn if we did not find any page!)
failed = (i in failed_pages or i + 1 in failed_pages)
first = self.obj.images[i]
second = self.obj.images[i + 1]
if first.page_number is None and second.page_number is None:
if not failed:
# Whoa, that should not happen.
log.warn(
_("Neither %s, %i or %s, %i has a known page number!" %
(first.filename, first.tiff_page, second.filename,
second.tiff_page)))
failed_pages.add(i)
failed_pages.add(i + 1)
elif duplex_mode == False:
# Simplex mode is special, as we know that one has to be unreadable
# we need to ensure one of the page numbers is None
if first.page_number is not None and second.page_number is not None:
# We don't touch the ignore flag in this case
# Simply print a message as this should *never* happen
log.error(
_("Got a simplex document where two adjacent pages had a known page number. This should never happen as even simplex scans are converted to duplex by inserting dummy pages. Maybe you did a simplex scan but added it in duplex mode? The pages in question are %s, %i and %s, %i."
) % (first.filename, first.tiff_page,
second.filename, second.tiff_page))
# Set the ignored flag for the unreadable page. This is a valid
# operation as the back side of a readable page is known to be
# empty.
elif first.page_number is None:
first.ignored = True
else:
second.ignored = True
elif first.page_number is None:
# One based, odd -> +1, even -> -1
first.page_number = second.page_number - 1 + 2 * (
second.page_number % 2)
elif second.page_number is None:
second.page_number = first.page_number - 1 + 2 * (
first.page_number % 2)
elif first.page_number != (second.page_number - 1 + 2 *
(second.page_number % 2)):
if not failed:
log.warn(
_("Images %s, %i and %s, %i do not have consecutive page numbers!"
% (first.filename, first.tiff_page, second.filename,
second.tiff_page)))
failed_pages.add(i)
failed_pages.add(i + 1)
i += 2
# Check that every page has a non None value, and each page exists once.
pages = set()
for i, image in enumerate(self.obj.images):
# Ignore known blank pages
if image.ignored:
continue
if image.page_number is None:
log.warn(
_("No page number for page %s, %i exists." %
(image.filename, image.tiff_page)))
failed_pages.add(i)
continue
if image.page_number in pages:
log.warn(
_("Page number for page %s, %i already used by another image."
) % (image.filename, image.tiff_page))
failed_pages.add(i)
continue
if image.page_number <= 0 or image.page_number > self.obj.survey.questionnaire.page_count:
log.warn(
_("Page number %i for page %s, %i is out of range.") %
(image.page_number, image.filename, image.tiff_page))
failed_pages.add(i)
continue
pages.add(image.page_number)
# Figure out the suvey ID if neccessary
# *************************************
if self.obj.survey.defs.print_survey_id:
for page, image in enumerate(self.obj.images):
try:
if not duplex_mode or (image.page_number is not None
and image.page_number % 2 == 0):
image.recognize.calculate_survey_id()
else:
image.survey_id = None
except RecognitionError:
log.warn(
_('%s, %i: Could not read survey ID, but should be able to.'
) % (image.filename, image.tiff_page))
failed_pages.add(page)
self.duplex_copy_image_attr(
failed_pages, "survey_id",
_("Could not read survey ID of either %s, %i or %s, %i!"))
# Simply use the survey ID from the first image globally
self.obj.survey_id = self.obj.images[0].survey_id
if self.obj.survey_id != self.obj.survey.survey_id:
# Broken survey ID ...
log.warn(
_("Got a wrong survey ID (%s, %i)! It is %s, but should be %i."
) %
(self.obj.images[0].filename, self.obj.images[0].tiff_page,
self.obj.survey_id, self.obj.survey.survey_id))
self.obj.valid = 0
else:
# Assume that the data is from the correct survey
self.obj.survey_id = self.obj.survey.survey_id
for image in self.obj.images:
image.survey_id = self.obj.survey.survey_id
# Figure out the questionnaire ID if neccessary
# *********************************************
if self.obj.survey.defs.print_questionnaire_id:
questionnaire_ids = []
for page, image in enumerate(self.obj.images):
try:
if not duplex_mode or (image.page_number is not None
and image.page_number % 2 == 0):
image.recognize.calculate_questionnaire_id()
except RecognitionError:
log.warn(_('%s, %i: Could not read questionnaire ID, but should be able to.') % \
(image.filename, image.tiff_page))
failed_pages.add(page)
if image.questionnaire_id is not None:
questionnaire_ids.append(image.questionnaire_id)
self.duplex_copy_image_attr(
failed_pages, "questionnaire_id",
_("Could not read questionnaire ID of either %s, %i or %s, %i!"
))
if len(questionnaire_ids):
self.obj.questionnaire_id = questionnaire_ids
else:
self.obj.questionnaire_id
# Try to load the global ID. If it does not exist we will get None, if
# it does, then it will be non-None. We don't care much about it
# internally anyways.
# However, we do want to ensure that it is the same everywhere if it
# can be read in.
# *********************************************
for page, image in enumerate(self.obj.images):
try:
if not duplex_mode or (image.page_number is not None
and image.page_number % 2 == 0):
image.recognize.calculate_global_id()
except RecognitionError:
pass
self.duplex_copy_image_attr(failed_pages, "global_id")
self.obj.global_id = self.obj.images[0].global_id
for image in self.obj.images:
if self.obj.global_id != image.global_id or \
self.obj.survey_id != image.survey_id or \
self.obj.questionnaire_id != image.questionnaire_id:
if not warned_multipage_not_correctly_scanned:
log.warn(
_("Got different IDs on different pages for at least one sheet! Do *NOT* try to use filters with this survey! You have to run a \"reorder\" step for this to work properly!"
))
warned_multipage_not_correctly_scanned = True
# Done
if failed_pages:
self.obj.valid = 0
def build_survey(self, djsurvey_id):
"""Creates the SDAPS project and database for the survey.
This process should be run on an already initialized survey that
has a questionnaire written to it."""
djsurvey = get_object_or_404(models.Survey, pk=djsurvey_id)
assert(djsurvey.initialized == False)
lock_id = ('%s_build_survey' % djsurvey.id)
with task_lock(lock_id, self.app.oid) as acquired:
if acquired:
import sdaps.setuptex as setup
from sdaps.utils import latex
from sdaps.setuptex import sdapsfileparser
survey = model.survey.Survey.new(djsurvey.path)
latex.write_override(survey, survey.path('sdaps.opt'), draft=True)
if not utils.atomic_latex_compile(djsurvey.path, 'questionnaire.tex', need_sdaps=True):
# XXX: The sqlite file should not be created immediately!
os.unlink(survey.path('survey.sqlite'))
return False
# We now have the .sdaps file that can be parsed
# Defaults
survey.defs.print_questionnaire_id = False
survey.defs.print_survey_id = True
survey.defs.engine = defs.latex_engine
survey.add_questionnaire(model.questionnaire.Questionnaire())
# Parse qobjects
try:
sdapsfileparser.parse(survey)
for qobject in survey.questionnaire.qobjects:
qobject.setup.setup()
qobject.setup.validate()
except:
log.error("Caught an Exception while parsing the SDAPS file. The current state is:")
print(str(survey.questionnaire), file=sys.stderr)
print("------------------------------------", file=sys.stderr)
raise AssertionError("Exception while parsing the SDAPS file.")
# Last but not least calculate the survey id
survey.calculate_survey_id()
if not survey.check_settings():
log.error("Some combination of options and project properties do not work. Aborted Setup.")
os.unlink(survey.path('survey.sqlite'))
return False
latex.write_override(survey, survey.path('sdaps.opt'), draft=False)
if not utils.atomic_latex_compile(djsurvey.path, 'questionnaire.tex', need_sdaps=True):
os.unlink(survey.path('survey.sqlite'))
return False
# TODO: If something goes wrong while initializing the survey,
# there should be an option to delete the files.
survey.save()
djsurvey.initialized = True
djsurvey.title = survey.title
if 'Author' in survey.info:
djsurvey.author = survey.info['Author']
djsurvey.save()
log.logfile.close()
def setup(survey, cmdline):
if os.access(survey.path(), os.F_OK):
log.error(_('The survey directory already exists.'))
return 1
questionnaire_odt = cmdline['questionnaire.odt']
questionnaire_pdf = cmdline['questionnaire.pdf']
additionalqobjects = cmdline['additional_questions']
mimetype = utils.mimetype(questionnaire_odt)
if mimetype != 'application/vnd.oasis.opendocument.text' and mimetype != '':
log.error(_('Unknown file type (%s). questionnaire_odt should be application/vnd.oasis.opendocument.text.') % mimetype)
return 1
mimetype = utils.mimetype(questionnaire_pdf)
if mimetype != 'application/pdf' and mimetype != '':
log.error(_('Unknown file type (%s). questionnaire_pdf should be application/pdf.') % mimetype)
return 1
if additionalqobjects is not None:
mimetype = utils.mimetype(additionalqobjects)
if mimetype != 'text/plain' and mimetype != '':
log.error(_('Unknown file type (%s). additionalqobjects should be text/plain.') % mimetype)
return 1
# Add the new questionnaire
survey.add_questionnaire(model.questionnaire.Questionnaire())
# Parse the box objects into a cache
boxes, page_count = boxesparser.parse(questionnaire_pdf)
survey.questionnaire.page_count = page_count
# Get the papersize
doc = pdffile.PDFDocument(questionnaire_pdf)
page = doc.read_page(1)
survey.defs.paper_width = abs(page.MediaBox[0] - page.MediaBox[2]) / 72.0 * 25.4
survey.defs.paper_height = abs(page.MediaBox[1] - page.MediaBox[3]) / 72.0 * 25.4
survey.defs.print_questionnaire_id = cmdline['print_questionnaire_id']
survey.defs.print_survey_id = cmdline['print_survey_id']
survey.defs.style = cmdline['style']
# Force simplex if page count is one.
survey.defs.duplex = False if page_count == 1 else cmdline['duplex']
survey.global_id = cmdline['global_id']
# Parse qobjects
try:
qobjectsparser.parse(survey, questionnaire_odt, boxes)
except:
log.error(_("Caught an Exception while parsing the ODT file. The current state is:"))
print unicode(survey.questionnaire)
print "------------------------------------"
raise
# Parse additionalqobjects
if additionalqobjects:
additionalparser.parse(survey, additionalqobjects)
# Parse Metadata
metaparser.parse(survey, questionnaire_odt)
# Last but not least calculate the survey id
survey.calculate_survey_id()
if not survey.check_settings():
log.error(_("Some combination of options and project properties do not work. Aborted Setup."))
return 1
# Print the result
print survey.title
for item in survey.info.items():
print u'%s: %s' % item
print unicode(survey.questionnaire)
# Create the survey
os.mkdir(survey.path())
log.logfile.open(survey.path('log'))
shutil.copy(questionnaire_odt, survey.path('questionnaire.odt'))
shutil.copy(questionnaire_pdf, survey.path('questionnaire.pdf'))
survey.save()
log.logfile.close()
],
cwd=survey.path())
# And again, without the draft mode
subprocess.call([
defs.latex_engine, '-halt-on-error', '-interaction', 'batchmode',
'questionnaire.tex'
],
cwd=survey.path())
if not os.path.exists(survey.path('questionnaire.pdf')):
print _("Error running \"%s\" to compile the LaTeX file."
) % defs.latex_engine
raise AssertionError('PDF file not generated')
# Print the result
print survey.title
for item in survey.info.items():
print u'%s: %s' % item
print unicode(survey.questionnaire)
log.logfile.open(survey.path('log'))
survey.save()
log.logfile.close()
except:
log.error(
_("An error occured in the setup routine. The survey directory still exists. You can for example check the questionnaire.log file for LaTeX compile errors."
))
raise