def queue_index_actions(blog):
'''
Pushes to the publishing queue all the index pages for a given blog
that are marked for Immediate publishing.
'''
try:
templates = Template.select().where(
Template.blog == blog,
Template.template_type == template_type.index,
Template.publishing_mode == publishing_mode.immediate)
if templates.count() == 0:
raise Template.DoesNotExist
except Template.DoesNotExist:
raise Template.DoesNotExist(
"No index templates exist for blog {}.".format(blog.for_log))
else:
mappings = TemplateMapping.select().where(
TemplateMapping.template << templates)
fileinfos = FileInfo.select().where(
FileInfo.template_mapping << mappings)
for f in fileinfos:
push_to_queue(job_type=job_type.index,
priority=1,
blog=blog,
site=blog.site,
data_integer=f.id)
def build_page(queue_entry, async_write=False):
try:
fileinfo = FileInfo.get(FileInfo.id == queue_entry.data_integer)
blog = queue_entry.blog
page_tags = generate_page_tags(fileinfo, blog)
file_page_text = generate_page_text(fileinfo, page_tags)
if async_write:
if not t.is_alive:
t.target = write_file_queue
t.args = (write_queue, )
t.start()
write_queue.put_nowait(
(file_page_text, blog.path, fileinfo.file_path))
else:
write_file(file_page_text, blog.path, fileinfo.file_path)
except FileInfo.DoesNotExist as e:
raise Exception('''Fileinfo {} could not be found in the system.
It may refer to a fileinfo that was deleted by another action. ({})'''.format(
queue_entry.data_integer, e))
except NoArchiveForFileInfo:
logger.info(
"Fileinfo {} has no corresponding pages. File {} removed.".format(
fileinfo.id, fileinfo.file_path))
delete_fileinfo_files((fileinfo, ))
# fileinfo.delete_instance(recursive=True)
# FIXME: for now we leave this out
# because deletes do not coalesce properly in the queue (I think)
except Exception as e:
context_list = [(f.object, f.ref) for f in fileinfo.context]
raise Exception('Error building fileinfo {} ({},{},{}): {}'.format(
fileinfo.id, fileinfo.page, context_list, fileinfo.file_path, e))
def queue_index_actions(blog, include_manual=False):
'''
Pushes to the publishing queue all the index pages for a given blog
that are marked for Immediate publishing.
:param blog:
The blog object whose index templates will be pushed to the queue.
:param include_manual:
If set to True, all templates, including those set to the Manual publishing mode,
will be pushed to the queue. Default is False, since those templates are not
pushed in most publishing actions.
'''
templates = blog.index_templates.select().where(
Template.publishing_mode != publishing_mode.do_not_publish)
if include_manual is False:
templates = templates.select().where(
Template.publishing_mode == publishing_mode.immediate)
if templates.count() == 0:
raise Template.DoesNotExist(
"No valid index templates exist for blog {}.".format(blog.for_log))
mappings = TemplateMapping.select().where(
TemplateMapping.template << templates)
fileinfos = FileInfo.select().where(FileInfo.template_mapping << mappings)
for f in fileinfos:
Queue.push(job_type=job_type.index,
priority=1,
blog=blog,
site=blog.site,
data_integer=f.id)
def delete_page_fileinfo(page):
'''
Deletes the fileinfo entry associated with a specific page.
This does not delete fileinfos that are general archives
for that page, only fileinfos associated with page templates.
:param page:
The page object to remove from the fileinfo index.
'''
# We should probably move this to models.Page
from core.models import Queue
from core.cms.queue import job_type
fileinfos_to_delete = FileInfo.select().where(FileInfo.page == page)
# We don't use page.fileinfos because that automatically
# regenerates any missing fileinfos.
# TODO: have an option, false by default, to suppress that
for n in fileinfos_to_delete:
Queue.delete().where(
Queue.job_type == job_type.page,
Queue.data_integer == n.id,
Queue.blog == page.blog,
).execute()
n.delete_instance()
def queue_index_actions(blog, include_manual=False):
'''
Pushes to the publishing queue all the index pages for a given blog
that are marked for Immediate publishing.
:param blog:
The blog object whose index templates will be pushed to the queue.
:param include_manual:
If set to True, all templates, including those set to the Manual publishing mode,
will be pushed to the queue. Default is False, since those templates are not
pushed in most publishing actions.
'''
templates = blog.index_templates.select().where(
Template.publishing_mode != publishing_mode.do_not_publish)
if include_manual is False:
templates = templates.select().where(
Template.publishing_mode == publishing_mode.immediate)
if templates.count() == 0:
raise Template.DoesNotExist("No valid index templates exist for blog {}.".format(
blog.for_log))
mappings = TemplateMapping.select().where(TemplateMapping.template << templates)
fileinfos = FileInfo.select().where(FileInfo.template_mapping << mappings)
for f in fileinfos:
Queue.push(job_type=job_type.index,
priority=1,
blog=blog,
site=blog.site,
data_integer=f.id)
def preview(path):
page = FileInfo.get(FileInfo.url == path)
# return template mapping if no page found
# prefix urls in preview for virtual filesystem movement
return ui.page_preview(page.page.id)
def build_page(q):
'''
Builds the file for a single blog page, q, based on its fileinfo data.
'''
fileinfo = FileInfo.get(FileInfo.id == q.data_integer)
try:
build_file(fileinfo, q.blog)
except BaseException:
raise
def delete(template):
t0 = FileInfo.delete().where(
FileInfo.template_mapping << template.mappings)
t0.execute()
t1 = TemplateMapping.delete().where(
TemplateMapping.id << template.mappings)
t1.execute()
t2 = Template.delete().where(Template.id == template.id)
t2.execute()
def delete_page_fileinfo(page):
'''
Deletes the fileinfo entry associated with a specific page.
This does not perform any security checks.
This also does not delete anything from the filesystem.
'''
fileinfo_to_delete = FileInfo.delete().where(FileInfo.page == page)
return fileinfo_to_delete.execute()
def purge_fileinfos(fileinfos):
'''
Takes a collection of fileinfos in the form of a model
and removes them from the fileinfo list.
Returns how many entries were purged.
No security checks are performed.
'''
context_purge = FileInfoContext.delete().where(FileInfoContext.fileinfo << fileinfos)
n = context_purge.execute()
purge = FileInfo.delete().where(FileInfo.id << fileinfos)
m = purge.execute()
return m, n
def push_to_queue(**ka):
'''
Inserts a single job item into the work queue.
:param job_type:
A string representing the type of job to be inserted.
'Page','Index', eventually 'Fileinfo'
:param data_integer:
Any integer data passed along with the job. For a job control item, this
is the number of items remaining for that particular job.
:param blog:
The blog object associated with the job.
:param site:
The site object associated with the job.
:param priority:
An integer, from 0-9, representing the processing priority associated with the job.
Higher-priority jobs are processed first. Most individual pages are given a high
priority; indexes are lower.
'''
try:
queue_job = Queue.get(Queue.job_type == ka['job_type'],
Queue.data_integer == ka['data_integer'],
Queue.blog == ka['blog'],
Queue.site == ka['site'])
except Queue.DoesNotExist:
queue_job = Queue()
else:
return
queue_job.job_type = ka['job_type']
queue_job.data_integer = int(ka.get('data_integer', None))
queue_job.blog = ka.get('blog', Blog()).id
queue_job.site = ka.get('site', Site()).id
queue_job.priority = ka.get('priority', 9)
queue_job.is_control = ka.get('is_control', False)
if queue_job.is_control:
queue_job.data_string = (queue_job.job_type +
": Blog {}".format(queue_job.blog.for_log))
else:
queue_job.data_string = (
queue_job.job_type + ": " +
FileInfo.get(FileInfo.id == queue_job.data_integer).file_path)
queue_job.date_touched = datetime.datetime.now()
queue_job.save()
def add_page_fileinfo(page,
template_mapping,
file_path,
url,
sitewide_file_path,
mapping_sort=None):
'''
Add a given page (could also be an index) to the fileinfo index.
Called by the page builder routines.
'''
try:
existing_fileinfo = FileInfo.get(
FileInfo.sitewide_file_path == sitewide_file_path,
FileInfo.template_mapping == template_mapping)
except FileInfo.DoesNotExist:
new_fileinfo = FileInfo.create(page=page,
template_mapping=template_mapping,
file_path=file_path,
sitewide_file_path=sitewide_file_path,
url=url,
mapping_sort=mapping_sort)
fileinfo = new_fileinfo
else:
existing_fileinfo.file_path = file_path
existing_fileinfo.sitewide_file_path = sitewide_file_path
existing_fileinfo.url = url
existing_fileinfo.modified_date = datetime.datetime.now()
existing_fileinfo.mapping_sort = mapping_sort
existing_fileinfo.save()
fileinfo = existing_fileinfo
return fileinfo
def queue_page_archive_actions(page):
'''
Pushes to the publishing queue all the page archives for a given page object.
:param page:
The page object whose archives will be pushed to the publishing queue.
'''
#===========================================================================
# NOTE: I tried to speed this up by checking the list of fileinfos
# related to mappings for the page (if any), and then pushing those
# if they exist, but I haven't seen evidence it does anything tangible
# for performance.
# I need to double-check that old mappings are in fact invalidated
# when they are changed.
#===========================================================================
archive_templates = page.blog.archive_templates
tags = template_tags(page=page)
for n in archive_templates:
try:
if n.publishing_mode != publishing_mode.do_not_publish:
fileinfo_mappings = FileInfo.select().where(
FileInfo.page == page,
FileInfo.template_mapping << n.mappings)
if fileinfo_mappings.count() == 0:
fileinfo_mappings = build_archives_fileinfos_by_mappings(
n, (page, ))
if len(fileinfo_mappings) == 0:
logger.info(
'No archive fileinfos could be built for page {} with template {}'
.format(page.for_log, n.for_log))
else:
for fileinfo_mapping in fileinfo_mappings:
Queue.push(job_type=job_type.archive,
blog=page.blog,
site=page.blog.site,
priority=7,
data_integer=fileinfo_mapping.id)
except Exception as e:
from core.error import QueueAddError
raise QueueAddError(
'Archive template {} for page {} could not be queued: '.format(
n, page.for_log, e))
def queue_page_archive_actions(page):
'''
Pushes to the publishing queue all the page archives for a given page object.
:param page:
The page object whose archives will be pushed to the publishing queue.
'''
#===========================================================================
# NOTE: I tried to speed this up by checking the list of fileinfos
# related to mappings for the page (if any), and then pushing those
# if they exist, but I haven't seen evidence it does anything tangible
# for performance.
# I need to double-check that old mappings are in fact invalidated
# when they are changed.
#===========================================================================
archive_templates = page.blog.archive_templates
tags = template_tags(page=page)
for n in archive_templates:
try:
if n.publishing_mode != publishing_mode.do_not_publish:
fileinfo_mappings = FileInfo.select().where(FileInfo.page == page,
FileInfo.template_mapping << n.mappings)
if fileinfo_mappings.count() == 0:
fileinfo_mappings=build_archives_fileinfos_by_mappings(n,(page,))
if len(fileinfo_mappings)==0:
logger.info('No archive fileinfos could be built for page {} with template {}'.format(
page.for_log,
n.for_log))
else:
for fileinfo_mapping in fileinfo_mappings:
Queue.push(job_type=job_type.archive,
blog=page.blog,
site=page.blog.site,
priority=7,
data_integer=fileinfo_mapping.id)
except Exception as e:
from core.error import QueueAddError
raise QueueAddError('Archive template {} for page {} could not be queued: '.format(
n,
page.for_log,
e))
def queue_archive_template_fast(template_id, action, pass_id=0):
from core.models import Template, Queue
template = Template.load(template_id)
blog = template.blog
from core import cms
from core.libs.bottle import HTTPResponse
r = HTTPResponse()
fileinfos = FileInfo.select().where(FileInfo.template_mapping << template.mappings).paginate(pass_id, 50)
# TODO: if action is fast and no fileinfos present, redirect to full rebuild?
if fileinfos.count() > 0:
r.body = "Adding {}".format(pass_id * 50)
for f in fileinfos:
Queue.push(job_type=cms.queue.job_type.archive,
blog=blog,
site=blog.site,
data_integer=f.id)
pass_id += 1
r.add_header('Refresh', "0;{}/template/{}/{}/fast/{}".format(
BASE_PATH,
template_id,
action,
pass_id))
else:
r.body = "Queue insertion finished."
if action == 'publish':
redir = 'publish'
else:
redir = 'queue'
r.add_header('Refresh', "0;{}/blog/{}/{}".format(
BASE_PATH,
blog.id,
redir))
return r
def queue_archive_template_fast(template_id, action, pass_id=0):
from core.models import Template, Queue
template = Template.load(template_id)
blog = template.blog
from core import cms
from core.libs.bottle import HTTPResponse
r = HTTPResponse()
fileinfos = FileInfo.select().where(
FileInfo.template_mapping << template.mappings).paginate(pass_id, 50)
# TODO: if action is fast and no fileinfos present, redirect to full rebuild?
if fileinfos.count() > 0:
r.body = "Adding {}".format(pass_id * 50)
for f in fileinfos:
Queue.push(job_type=cms.queue.job_type.archive,
blog=blog,
site=blog.site,
data_integer=f.id)
pass_id += 1
r.add_header(
'Refresh',
"0;{}/template/{}/{}/fast/{}".format(BASE_PATH, template_id,
action, pass_id))
else:
r.body = "Queue insertion finished."
if action == 'publish':
redir = 'publish'
else:
redir = 'queue'
r.add_header('Refresh',
"0;{}/blog/{}/{}".format(BASE_PATH, blog.id, redir))
return r
def queue_page_archive_actions(page):
'''
Pushes to the publishing queue all the page archives for a given page object.
'''
archive_templates = page.blog.archive_templates
tags = template_tags(page_id=page.id)
for n in archive_templates:
for m in n.mappings:
file_path = (page.blog.path + '/' + generate_date_mapping(
page.publication_date.date(), tags, m.path_string))
fileinfo_mapping = FileInfo.get(
FileInfo.sitewide_file_path == file_path)
push_to_queue(job_type=job_type.archive,
blog=page.blog,
site=page.blog.site,
data_integer=fileinfo_mapping.id)
def build_page(queue_entry, async_write=False):
try:
fileinfo = FileInfo.get(FileInfo.id == queue_entry.data_integer)
blog = queue_entry.blog
page_tags = generate_page_tags(fileinfo, blog)
file_page_text = generate_page_text(fileinfo, page_tags)
if async_write:
if not t.is_alive:
t.target = write_file_queue
t.args = (write_queue,)
t.start()
write_queue.put_nowait((file_page_text, blog.path, fileinfo.file_path))
else:
write_file(file_page_text, blog.path, fileinfo.file_path)
except FileInfo.DoesNotExist as e:
raise Exception('''Fileinfo {} could not be found in the system.
It may refer to a fileinfo that was deleted by another action. ({})'''.format(queue_entry.data_integer, e))
except NoArchiveForFileInfo:
logger.info("Fileinfo {} has no corresponding pages. File {} removed.".format(
fileinfo.id,
fileinfo.file_path)
)
delete_fileinfo_files((fileinfo,))
# fileinfo.delete_instance(recursive=True)
# FIXME: for now we leave this out
# because deletes do not coalesce properly in the queue (I think)
except Exception as e:
context_list = [(f.object, f.ref) for f in fileinfo.context]
raise Exception('Error building fileinfo {} ({},{},{}): {}'.format(
fileinfo.id,
fileinfo.page,
context_list,
fileinfo.file_path,
e))
def preview(path):
from core.ui import page
preview_page = FileInfo.get(FileInfo.url == path)
return page.page_preview(preview_page.page.id)
def add_page_fileinfo(page, template_mapping, file_path,
url, sitewide_file_path, mapping_sort=None):
'''
Add a given page (could also be an index) to the fileinfo index.
If the page already exists, then the existing fileinfo
is updated with the new information.
Called by the page builder routines.
:param page:
The page object to add to the fileinfo index.
:param template_mapping:
The template mapping to use for creating the page's fileinfo(s).
:param file_path:
The file path to use for the fileinfo.
:param url:
The URL to associate with the fileinfo.
:param sitewide_file_path:
The sitewide file path to use for the fileinfo.
:param mapping_sort:
Sort order for the mapping, if used.
'''
try:
existing_fileinfo = FileInfo.get(
FileInfo.sitewide_file_path == sitewide_file_path,
FileInfo.template_mapping == template_mapping
)
except FileInfo.DoesNotExist:
try:
new_fileinfo = FileInfo.create(page=page,
template_mapping=template_mapping,
file_path=file_path,
sitewide_file_path=sitewide_file_path,
url=url,
mapping_sort=mapping_sort)
fileinfo = new_fileinfo
except IntegrityError:
from core.error import FileInfoCollision
collision = FileInfo.get(
FileInfo.sitewide_file_path == sitewide_file_path)
raise FileInfoCollision('''
Template mapping #{}, {}, for template #{},
yields a path that already exists in the system: {}
This appears to be a collision with mapping {} in template {}'''.format(
template_mapping.id,
template_mapping.path_string,
template_mapping.template.id,
sitewide_file_path,
collision.template_mapping.path_string,
collision.template_mapping.template.for_log))
else:
existing_fileinfo.file_path = file_path
existing_fileinfo.sitewide_file_path = sitewide_file_path
existing_fileinfo.url = url
existing_fileinfo.modified_date = datetime.datetime.utcnow()
existing_fileinfo.mapping_sort = mapping_sort
existing_fileinfo.save()
fileinfo = existing_fileinfo
return fileinfo
def queue_page_actions(page, no_neighbors=False, no_archive=False):
'''
Pushes a Page object along with all its related items into the queue for publication.
This includes any archive indices associated with the page, and the page's next and
previous entries in its respective categories.
Note that this will only queue items that are actually set to be published.
:param page:
The Page object whose actions are to be queued.
:param no_neighbors:
Set to True to suppress generation of next/previous posts. Useful if you've loaded
all the posts for a blog into a queue.
:param no_archive:
Set to True to suppress generation of archive pages associated with this page. Also
useful for mass-queued actions.
'''
# fileinfos = FileInfo.select().where(FileInfo.page == page)
fileinfos = page.fileinfos
blog = page.blog
site = page.blog.site
for f in fileinfos:
push_to_queue(job_type=job_type.page,
blog=blog,
site=site,
data_integer=f.id)
if no_archive is False:
queue_page_archive_actions(page)
if no_neighbors is False:
next_page = page.next_page
previous_page = page.previous_page
# Next and previous across categories should be done through this
# mechanism somehow
if next_page is not None:
fileinfos_next = FileInfo.select().where(
FileInfo.page == next_page)
for f in fileinfos_next:
push_to_queue(job_type=job_type.page,
blog=blog,
site=site,
data_integer=f.id)
queue_page_archive_actions(next_page)
if previous_page is not None:
fileinfos_previous = FileInfo.select().where(
FileInfo.page == previous_page)
for f in fileinfos_previous:
push_to_queue(job_type=job_type.page,
blog=blog,
site=site,
data_integer=f.id)
queue_page_archive_actions(previous_page)
def preview(path):
from core.ui import page
preview_page = FileInfo.get(
FileInfo.url == path)
return page.page_preview(preview_page.page.id)