def test_1n1_associated_file_cleanup(session):
item = MediaItem(description="Foo")
item.content = MediaItemFile(id='abcd', name='bar')
item.content.reference = as_fileintent(b'bar', 'bar')
session.add(item)
session.flush()
item = session.query(MediaItem).one()
folder = Path(item.content.reference.file._metadata_path).parent.parent
assert session.query(File).count() == 1
assert item.content.reference.file.read() == b'bar'
assert sum(1 for p in folder.iterdir()) == 1
item.content = MediaItemFile(id='abcd', name='baz')
item.content.reference = as_fileintent(b'baz', 'baz')
session.flush()
transaction.commit()
assert session.query(File).count() == 1
assert item.content.reference.file.read() == b'baz'
assert sum(1 for p in folder.iterdir()) == 1 # 2
def upload_attachment(self, request):
""" Upload an attachment and add it to the notice.
Raises a HTTP 405 (Metho not Allowed) for non-admins if the notice has
already been accepted.
Raises a HTTP 415 (Unsupported Media Type) if the file format is not
supported.
"""
if self.state == 'accepted' or self.state == 'published':
if not request.is_secret(self):
raise exc.HTTPMethodNotAllowed()
request.assert_valid_csrf_token()
attachment = GazetteNoticeFile(id=random_token())
attachment.name = request.params['file'].filename
attachment.reference = as_fileintent(request.params['file'].file,
request.params['file'].filename)
if attachment.reference.content_type != 'application/pdf':
raise exc.HTTPUnsupportedMediaType()
self.files.append(attachment)
self.add_change(request, _("Attachment added."))
request.message(_("Attachment added."), 'success')
return redirect(request.link(self, 'attachments'))
def pdf_file(self, value):
""" Sets the PDF content for the agency (and all its
suborganizations). Automatically sets a nice filename. Replaces only
the reference, if possible.
"""
filename = '{}.pdf'.format(normalize_for_url(self.title))
if self.pdf:
self.pdf.reference = as_fileintent(value, filename)
self.pdf.name = filename
else:
pdf = AgencyPdf(id=random_token())
pdf.reference = as_fileintent(value, filename)
pdf.name = filename
self.pdf = pdf
def test_page_file(swissvotes_app):
session = swissvotes_app.session()
page = TranslatablePage(id='page',
title_translations={
'de_CH': "Titel",
'en': "Title"
},
content_translations={
'de_CH': "Inhalt",
'en': "Content"
})
session.add(page)
session.flush()
assert page.files == []
attachment = TranslatablePageFile(id=random_token())
attachment.name = 'de_CH-test.txt'
attachment.reference = as_fileintent(BytesIO(b'test'), 'test.txt')
page.files.append(attachment)
session.flush()
file = session.query(TranslatablePage).one().files[0]
assert file.name == 'de_CH-test.txt'
assert file.filename == 'test.txt'
assert file.locale == 'de_CH'
def replace(self, file, content):
""" Replaces the content of the given file with the new content. """
if not self.allow_duplicates:
self.assert_not_duplicate(content)
file.reference = as_fileintent(content, file.name)
self.session.flush()
def pdf(self, value):
filename = '{}.pdf'.format(self.name)
pdf = self.pdf or IssuePdfFile(id=random_token())
pdf.name = filename
pdf.reference = as_fileintent(value, filename)
if not self.pdf:
self.files.append(pdf)
def handle_import(request, app):
temp = TemporaryDirectory()
path = Path(temp.name)
data = {}
zip = ZipFile(import_file)
zip.extractall(path)
with (path / 'data.json').open('r') as f:
data = json.load(f)
existing = {
str(v.id): v for v in request.session.query(MissionReportVehicle)
}
for id in data:
if id in existing and replace:
request.session.delete(existing[id])
request.session.flush()
if id in existing and not replace:
continue
vehicle = None
if match:
vehicle = request.session.query(MissionReportVehicle)\
.filter_by(name=data[id]['name'])\
.first()
if not vehicle:
vehicle = MissionReportVehicle(id=id)
vehicle.name = data[id]['name']
vehicle.description = data[id]['description']
vehicle.website = data[id]['website']
symbol_path = path / 'symbols' / str(id)
if symbol_path.exists():
filename = data[id]['filename']
with symbol_path.open('rb') as f:
vehicle.symbol = MissionReportFile(
id=random_token(),
name=filename,
reference=as_fileintent(
content=f,
filename=filename
)
)
request.session.add(vehicle)
def _import(request, app):
votes = SwissVoteCollection(app)
attachments = {}
for name in os.listdir(folder):
if (os.path.isdir(os.path.join(folder, name)) and isinstance(
SwissVote.__dict__.get(name), LocalizedFile)):
attachments[name] = os.path.join(folder, name)
else:
click.secho(f"Ignoring /{name}", fg='yellow')
for attachment, attachment_folder in attachments.items():
locales = {}
for name in os.listdir(attachment_folder):
if (os.path.isdir(os.path.join(attachment_folder, name))
and name in app.locales):
locales[name] = os.path.join(attachment_folder, name)
else:
click.secho(f"Ignoring /{attachment}/{name}", fg='yellow')
for locale, locale_folder in locales.items():
for name in sorted(os.listdir(locale_folder)):
if not (name.endswith('.pdf') or name.endswith('.xlsx')):
click.secho(f"Ignoring {attachment}/{locale}/{name}",
fg='yellow')
continue
try:
bfs_number = Decimal(
name.replace('.pdf', '').replace('.xlsx', ''))
except InvalidOperation:
click.secho(
f"Invalid name {attachment}/{locale}/{name}",
fg='red')
continue
vote = votes.by_bfs_number(bfs_number)
if not vote:
click.secho(
f"No matching vote {bfs_number} for "
f"{attachment}/{locale}/{name}",
fg='red')
continue
file = SwissVoteFile(id=random_token())
with open(os.path.join(locale_folder, name), 'rb') as f:
file.reference = as_fileintent(
f, f'{attachment}-{locale}')
vote.__class__.__dict__[attachment].__set_by_locale__(
vote, file, locale)
click.secho(f"Added {attachment}/{locale}/{name}",
fg='green')
def pdf_attachment(content):
pdf = BytesIO()
inline = Pdf(pdf)
inline.init_report()
inline.p(content)
inline.generate()
pdf.seek(0)
attachment = GazetteNoticeFile(id=random_token())
attachment.name = 'file.pdf'
attachment.reference = as_fileintent(pdf, 'file.pdf')
return attachment
def test_issue_file(gazette_app, session):
session.add(
IssuePdfFile(id='abcd',
name='test.txt',
reference=as_fileintent('Test text.'.encode('utf-8'),
'test.txt')))
session.flush()
file = session.query(IssuePdfFile).one()
assert file.id == 'abcd'
assert file.name == 'test.txt'
assert file.type == 'gazette_issue'
assert file.reference.file.read().decode('utf-8') == 'Test text.'
def update_model(self, model):
locale = self.request.locale
for field in self:
name = field.name
action = getattr(field, 'action', '')
if action == 'delete':
delattr(model, name)
if action == 'replace':
if field.data:
file = SwissVoteFile(id=random_token())
file.reference = as_fileintent(field.raw_data[-1].file,
f'{name}-{locale}')
setattr(model, name, file)
def test_file_cleanup(session):
session.add(File(name='readme.txt', reference=b'foo'))
transaction.commit()
readme = session.query(File).first()
folder = Path(readme.reference.file._metadata_path).parent.parent
assert sum(1 for f in folder.iterdir()) == 1
readme.reference = as_fileintent(b'bar', 'readme.txt')
transaction.commit()
assert sum(1 for f in folder.iterdir()) == 1
assert session.query(File).first().reference.file.read() == b'bar'
def attachments(swissvotes_app):
result = {}
for name, content in (
('ad_analysis', "Inserateanalyse"),
('brief_description', "Kurschbeschreibung"),
('federal_council_message', "Message du Conseil fédéral"),
('parliamentary_debate', "Parlamentdebatte"),
('realization', "Réalisation"),
('resolution', "Arrêté constatant le résultat"),
('voting_booklet', "Brochure explicative"),
('voting_text', "Abstimmungstext"),
):
file = BytesIO()
pdf = Pdf(file)
pdf.init_report()
pdf.p(content)
pdf.generate()
file.seek(0)
attachment = SwissVoteFile(id=random_token())
attachment.reference = as_fileintent(file, name)
result[name] = attachment
for name in ('results_by_domain', ):
file = BytesIO()
workbook = Workbook(file)
worksheet = workbook.add_worksheet('DATA')
worksheet.write_row(0, 0, ['a', 'b'])
worksheet.write_row(1, 0, [100, 200])
workbook.close()
file.seek(0)
attachment = SwissVoteFile(id=random_token())
attachment.reference = as_fileintent(file, name)
result[name] = attachment
yield result
def upload_page_attachment(self, request):
""" Upload an attachment and add it to the page.
Raises a HTTP 415 (Unsupported Media Type) if the file format is not
supported.
"""
request.assert_valid_csrf_token()
attachment = TranslatablePageFile(id=random_token())
attachment.name = '{}-{}'.format(request.locale,
request.params['file'].filename)
attachment.reference = as_fileintent(request.params['file'].file,
request.params['file'].filename)
self.files.append(attachment)
request.message(_("Attachment added."), 'success')
return redirect(request.link(self, 'attachments'))
def test_notice_file(gazette_app, session):
session.add(GazetteNotice(title='notice'))
session.flush()
notice = session.query(GazetteNotice).one()
notice.files.append(
GazetteNoticeFile(id='abcd',
name='test.txt',
reference=as_fileintent('Test text.'.encode('utf-8'),
'test.txt')))
session.flush()
file = session.query(GazetteNoticeFile).one()
assert file.id == 'abcd'
assert file.name == 'test.txt'
assert file.type == 'gazette_notice'
assert file.reference.file.read().decode('utf-8') == 'Test text.'
assert notice.files[0] == file
assert file.linked_official_notices[0] == notice
def test_layout_delete_page_attachment(swissvotes_app):
request = DummyRequest()
model = TranslatablePageFile(id=random_token())
model.name = 'attachment-name'
model.reference = as_fileintent(BytesIO(b'test'), 'filename')
layout = DeletePageAttachmentLayout(model, request)
assert layout.title == "Delete attachment"
assert layout.editbar_links == []
assert path(layout.breadcrumbs) == 'DummyPrincipal/TranslatablePageFile/#'
# Log in as editor
request.roles = ['editor']
layout = DeletePageAttachmentLayout(model, request)
assert layout.editbar_links == []
# Log in as admin
request.roles = ['admin']
layout = DeletePageAttachmentLayout(model, request)
assert layout.editbar_links == []
def add(self, filename, content, note=None, published=True,
publish_date=None):
""" Adds a file with the given filename. The content maybe either
in bytes or a file object.
"""
if not self.allow_duplicates:
self.assert_not_duplicate(content)
type = self.type != '*' and self.type or None
file = File.get_polymorphic_class(type, File)()
file.name = filename
file.note = note
file.type = type
file.published = published
file.publish_date = publish_date
file.reference = as_fileintent(content, filename)
self.session.add(file)
self.session.flush()
return file
def test_reindex(session_manager, temporary_directory, redis_url):
cfg_path = os.path.join(temporary_directory, 'onegov.yml')
write_config(cfg_path, session_manager.dsn, temporary_directory, redis_url)
result = run_command(cfg_path, 'govikon', ['add'])
assert result.exit_code == 0
assert "Instance was created successfully" in result.output
result = run_command(cfg_path, 'govikon', ['reindex'])
assert result.exit_code == 0
# Add vote
vote = SwissVote(
id=1,
bfs_number=Decimal(1),
date=date(1990, 6, 2),
legislation_number=4,
legislation_decade=NumericRange(1990, 1994),
title_de="Vote",
title_fr="Vote",
short_title_de="Vote",
short_title_fr="Vote",
votes_on_same_day=3,
_legal_form=1,
)
file = BytesIO()
pdf = Pdf(file)
pdf.init_report()
pdf.p("Abstimmungstext")
pdf.generate()
file.seek(0)
attachment = SwissVoteFile(id=random_token())
attachment.reference = as_fileintent(file, 'voting_text')
vote.voting_text = attachment
session_manager.ensure_schema_exists('onegov_swissvotes-govikon')
session_manager.set_current_schema('onegov_swissvotes-govikon')
session = session_manager.session()
session.add(vote)
session.flush()
commit()
result = run_command(cfg_path, 'govikon', ['reindex'])
assert result.exit_code == 0
assert "Reindexed vote 1.00" in result.output
vote = session.query(SwissVote).one()
assert "abstimmungstex" in vote.searchable_text_de_CH
with open(vote.voting_text.reference.file._file_path, 'wb') as file:
pdf = Pdf(file)
pdf.init_report()
pdf.p("Realisation")
pdf.generate()
vote = session.query(SwissVote).one()
assert "abstimmungstex" in vote.searchable_text_de_CH
result = run_command(cfg_path, 'govikon', ['reindex'])
assert result.exit_code == 0
assert "Reindexed vote 1.00" in result.output
vote = session.query(SwissVote).one()
assert "realisa" in vote.searchable_text_de_CH
