def blog_publish(blog_id): user = auth.is_logged_in(request) blog = Blog.load(blog_id) permission = auth.is_blog_publisher(user, blog) queue_length = Queue.job_counts(blog=blog) if queue_length > 0: start_message = template('queue/queue_run_include', queue=Queue.jobs(blog), percentage_complete=0, blog=blog, break_path='{}/blog/{}/publish/break'.format( BASE_URL, blog.id)) Queue.start(blog, queue_length) else: start_message = "Queue empty." tags = template_tags(blog_id=blog.id, user=user) # return template( 'queue/queue_run_ui', start=queue_length, start_message=start_message, action_url="../../blog/{}/publish/progress/{}".format( blog.id, queue_length), title='Publishing queue progress', # search_context=(search_context['blog_queue'], blog), menu=generate_menu('blog_queue', blog), **tags.__dict__)
def blog_publish_process(blog_id): user = auth.is_logged_in(request) blog = Blog.load(blog_id) permission = auth.is_blog_publisher(user, blog) control_jobs = Queue.control_jobs(blog) if control_jobs.count() > 0: queue_count = q_p(blog) # queue_count = transaction(queue.process_queue)(blog) time.sleep(RETRY_INTERVAL * 5) else: jobs = Queue.jobs(blog) if jobs.count() > 0: queue_count = jobs.count() Queue.start(blog, queue_count) queue_count = q_p(blog) # queue_count = transaction(queue.process_queue)(blog) time.sleep(RETRY_INTERVAL * 5) else: queue_count = 0 import settings return template('queue/queue_counter_include', blog=blog, settings=settings, queue_count=queue_count)
def queue_index_actions(blog, include_manual=False): ''' Pushes to the publishing queue all the index pages for a given blog that are marked for Immediate publishing. :param blog: The blog object whose index templates will be pushed to the queue. :param include_manual: If set to True, all templates, including those set to the Manual publishing mode, will be pushed to the queue. Default is False, since those templates are not pushed in most publishing actions. ''' templates = blog.index_templates.select().where( Template.publishing_mode != publishing_mode.do_not_publish) if include_manual is False: templates = templates.select().where( Template.publishing_mode == publishing_mode.immediate) if templates.count() == 0: raise Template.DoesNotExist("No valid index templates exist for blog {}.".format( blog.for_log)) mappings = TemplateMapping.select().where(TemplateMapping.template << templates) fileinfos = FileInfo.select().where(FileInfo.template_mapping << mappings) for f in fileinfos: Queue.push(job_type=job_type.index, priority=1, blog=blog, site=blog.site, data_integer=f.id)
def queue_index_actions(blog, include_manual=False): ''' Pushes to the publishing queue all the index pages for a given blog that are marked for Immediate publishing. :param blog: The blog object whose index templates will be pushed to the queue. :param include_manual: If set to True, all templates, including those set to the Manual publishing mode, will be pushed to the queue. Default is False, since those templates are not pushed in most publishing actions. ''' templates = blog.index_templates.select().where( Template.publishing_mode != publishing_mode.do_not_publish) if include_manual is False: templates = templates.select().where( Template.publishing_mode == publishing_mode.immediate) if templates.count() == 0: raise Template.DoesNotExist( "No valid index templates exist for blog {}.".format(blog.for_log)) mappings = TemplateMapping.select().where( TemplateMapping.template << templates) fileinfos = FileInfo.select().where(FileInfo.template_mapping << mappings) for f in fileinfos: Queue.push(job_type=job_type.index, priority=1, blog=blog, site=blog.site, data_integer=f.id)
def blog_publish(blog_id): user = auth.is_logged_in(request) blog = Blog.load(blog_id) permission = auth.is_blog_publisher(user, blog) queue_length = Queue.job_counts(blog=blog) if queue_length > 0: start_message = template('queue/queue_run_include', queue=Queue.jobs(blog), percentage_complete=0, blog=blog, break_path='{}/blog/{}/publish/break'.format(BASE_URL, blog.id) ) Queue.start(blog, queue_length) else: start_message = "Queue empty." tags = template_tags(blog_id=blog.id, user=user) # return template('queue/queue_run_ui', start=queue_length, start_message=start_message, action_url="../../blog/{}/publish/progress/{}".format(blog.id, queue_length), title='Publishing queue progress', # search_context=(search_context['blog_queue'], blog), menu=generate_menu('blog_queue', blog), **tags.__dict__)
def blog_publish_process(blog_id): user = auth.is_logged_in(request) blog = Blog.load(blog_id) permission = auth.is_blog_publisher(user, blog) control_jobs = Queue.control_jobs(blog) if control_jobs.count() > 0: queue_count = q_p(blog) # queue_count = transaction(queue.process_queue)(blog) time.sleep(RETRY_INTERVAL * 5) else: jobs = Queue.jobs(blog) if jobs.count() > 0: queue_count = jobs.count() Queue.start(blog, queue_count) queue_count = q_p(blog) # queue_count = transaction(queue.process_queue)(blog) time.sleep(RETRY_INTERVAL * 5) else: queue_count = 0 import settings return template('queue/queue_counter_include', blog=blog, settings=settings, queue_count=queue_count)
def delete_page_fileinfo(page): ''' Deletes the fileinfo entry associated with a specific page. This does not delete fileinfos that are general archives for that page, only fileinfos associated with page templates. :param page: The page object to remove from the fileinfo index. ''' # We should probably move this to models.Page from core.models import Queue from core.cms.queue import job_type fileinfos_to_delete = FileInfo.select().where(FileInfo.page == page) # We don't use page.fileinfos because that automatically # regenerates any missing fileinfos. # TODO: have an option, false by default, to suppress that for n in fileinfos_to_delete: Queue.delete().where( Queue.job_type == job_type.page, Queue.data_integer == n.id, Queue.blog == page.blog, ).execute() n.delete_instance()
def blog_queue_clear(blog_id): ''' Clear all pending jobs out of the queue ''' user = auth.is_logged_in(request) blog = Blog.load(blog_id) permission = auth.is_blog_publisher(user, blog) Queue.clear(blog)
def blog_queue_clear(blog_id): ''' Clear all pending jobs out of the queue ''' user = auth.is_logged_in(request) blog = Blog.load(blog_id) permission = auth.is_blog_publisher(user, blog) Queue.clear(blog)
def process_queue(blog): ''' Processes the jobs currently in the queue for the selected blog. ''' with db.atomic(): queue_control = publishing_lock(blog, True) if queue_control is None: return 0 queue_control.data_string = 'Running' queue_control.save() queue = Queue.select().order_by( Queue.priority.desc(), Queue.date_touched.desc()).where( Queue.blog == blog, Queue.is_control == False).limit(MAX_BATCH_OPS) queue_length = queue.count() if queue_length > 0: logger.info( "Queue job #{} @ {} (blog #{}, {} items) started.".format( queue_control.id, date_format(queue_control.date_touched), queue_control.blog.id, queue_length)) for q in queue: try: job_type.action[q.job_type](q) except BaseException: raise else: remove_from_queue(q.id) queue_control = Queue.get(Queue.blog == blog, Queue.is_control == True) queue_control.data_integer -= queue_length if queue_control.data_integer <= 0: queue_control.delete_instance() logger.info("Queue job #{} @ {} (blog #{}) finished.".format( queue_control.id, date_format(queue_control.date_touched), queue_control.blog.id)) else: queue_control.data_string = 'Paused' queue_control.save() logger.info( "Queue job #{} @ {} (blog #{}) continuing with {} items left.". format(queue_control.id, date_format(queue_control.date_touched), queue_control.blog.id, queue_length)) return queue_control.data_integer
def process_queue(blog): ''' Processes the jobs currently in the queue for the selected blog. ''' queue_control = Queue.acquire(blog, True) if queue_control is None: return 0 if queue_control.job_type == job_type.control: process_queue_publish(queue_control, blog) elif queue_control.job_type == job_type.insert: process_queue_insert(queue_control, blog) return Queue.job_counts(blog=blog)
def process_queue(blog): ''' Processes the jobs currently in the queue for the selected blog. ''' queue_control = Queue.acquire(blog, True) if queue_control is None: return 0 if queue_control.job_type == job_type.control: process_queue_publish(queue_control, blog) elif queue_control.job_type == job_type.insert: process_queue_insert(queue_control, blog) return Queue.job_counts(blog=blog)
def blog_publish_progress(blog_id, original_queue_length): user = auth.is_logged_in(request) blog = Blog.load(blog_id) permission = auth.is_blog_publisher(user, blog) queue_count = 0 control_jobs = Queue.control_jobs(blog) if control_jobs.count() > 0: # queue_count = queue.process_queue(blog) # queue_count = transaction(queue.process_queue)(blog) queue_count = q_p(blog) time.sleep(RETRY_INTERVAL * 5) else: queue_count = 0 percentage_complete = int( (1 - (int(queue_count) / int(original_queue_length))) * 100) import settings return template('queue/queue_run_include', queue_count=queue_count, blog=blog, break_path='{}/blog/{}/publish/break'.format( BASE_URL, blog.id), settings=settings, percentage_complete=percentage_complete)
def blog_publish_progress(blog_id, original_queue_length): user = auth.is_logged_in(request) blog = Blog.load(blog_id) permission = auth.is_blog_publisher(user, blog) queue_count = 0 control_jobs = Queue.control_jobs(blog) if control_jobs.count() > 0: # queue_count = queue.process_queue(blog) # queue_count = transaction(queue.process_queue)(blog) queue_count = q_p(blog) time.sleep(RETRY_INTERVAL * 5) else: queue_count = 0 percentage_complete = int((1 - (int(queue_count) / int(original_queue_length))) * 100) import settings return template('queue/queue_run_include', queue_count=queue_count, blog=blog, break_path='{}/blog/{}/publish/break'.format(BASE_URL, blog.id), settings=settings, percentage_complete=percentage_complete)
def system_queue(): user = auth.is_logged_in(request) permission = auth.is_sys_admin(user) queue = Queue.select().order_by(Queue.site.asc(), Queue.blog.asc(), Queue.job_type.asc(), Queue.date_touched.desc()) return listing(request, None, queue, 'queue', 'system_queue', user=user)
def queue_page_archive_actions(page): ''' Pushes to the publishing queue all the page archives for a given page object. :param page: The page object whose archives will be pushed to the publishing queue. ''' #=========================================================================== # NOTE: I tried to speed this up by checking the list of fileinfos # related to mappings for the page (if any), and then pushing those # if they exist, but I haven't seen evidence it does anything tangible # for performance. # I need to double-check that old mappings are in fact invalidated # when they are changed. #=========================================================================== archive_templates = page.blog.archive_templates tags = template_tags(page=page) for n in archive_templates: try: if n.publishing_mode != publishing_mode.do_not_publish: fileinfo_mappings = FileInfo.select().where( FileInfo.page == page, FileInfo.template_mapping << n.mappings) if fileinfo_mappings.count() == 0: fileinfo_mappings = build_archives_fileinfos_by_mappings( n, (page, )) if len(fileinfo_mappings) == 0: logger.info( 'No archive fileinfos could be built for page {} with template {}' .format(page.for_log, n.for_log)) else: for fileinfo_mapping in fileinfo_mappings: Queue.push(job_type=job_type.archive, blog=page.blog, site=page.blog.site, priority=7, data_integer=fileinfo_mapping.id) except Exception as e: from core.error import QueueAddError raise QueueAddError( 'Archive template {} for page {} could not be queued: '.format( n, page.for_log, e))
def system_queue(): user = auth.is_logged_in(request) permission = auth.is_sys_admin(user) queue = Queue.select().order_by(Queue.site.asc(), Queue.blog.asc(), Queue.job_type.asc(), Queue.date_touched.desc()) return listing(request, None, queue, 'queue', 'system_queue', user=user)
def queue_page_archive_actions(page): ''' Pushes to the publishing queue all the page archives for a given page object. :param page: The page object whose archives will be pushed to the publishing queue. ''' #=========================================================================== # NOTE: I tried to speed this up by checking the list of fileinfos # related to mappings for the page (if any), and then pushing those # if they exist, but I haven't seen evidence it does anything tangible # for performance. # I need to double-check that old mappings are in fact invalidated # when they are changed. #=========================================================================== archive_templates = page.blog.archive_templates tags = template_tags(page=page) for n in archive_templates: try: if n.publishing_mode != publishing_mode.do_not_publish: fileinfo_mappings = FileInfo.select().where(FileInfo.page == page, FileInfo.template_mapping << n.mappings) if fileinfo_mappings.count() == 0: fileinfo_mappings=build_archives_fileinfos_by_mappings(n,(page,)) if len(fileinfo_mappings)==0: logger.info('No archive fileinfos could be built for page {} with template {}'.format( page.for_log, n.for_log)) else: for fileinfo_mapping in fileinfo_mappings: Queue.push(job_type=job_type.archive, blog=page.blog, site=page.blog.site, priority=7, data_integer=fileinfo_mapping.id) except Exception as e: from core.error import QueueAddError raise QueueAddError('Archive template {} for page {} could not be queued: '.format( n, page.for_log, e))
def queue_archive_template_fast(template_id, action, pass_id=0): from core.models import Template, Queue template = Template.load(template_id) blog = template.blog from core import cms from core.libs.bottle import HTTPResponse r = HTTPResponse() fileinfos = FileInfo.select().where(FileInfo.template_mapping << template.mappings).paginate(pass_id, 50) # TODO: if action is fast and no fileinfos present, redirect to full rebuild? if fileinfos.count() > 0: r.body = "Adding {}".format(pass_id * 50) for f in fileinfos: Queue.push(job_type=cms.queue.job_type.archive, blog=blog, site=blog.site, data_integer=f.id) pass_id += 1 r.add_header('Refresh', "0;{}/template/{}/{}/fast/{}".format( BASE_PATH, template_id, action, pass_id)) else: r.body = "Queue insertion finished." if action == 'publish': redir = 'publish' else: redir = 'queue' r.add_header('Refresh', "0;{}/blog/{}/{}".format( BASE_PATH, blog.id, redir)) return r
def remove_from_queue(queue_id): ''' Removes a specific job ID from the queue. :param queue_id: The ID number of the job queue item to remove. ''' queue_delete = Queue.get(Queue.id == queue_id) return queue_delete.delete_instance()
def queue_ssi_actions(blog): ''' Pushes to the publishing queue all the SSIs for a given blog. :param blog: The blog object whose SSI templates will be pushed to the queue. ''' templates = blog.ssi_templates.select() if templates.count() == 0: return None for n in templates: for f in n.fileinfos: Queue.push(job_type=job_type.include, priority=10, blog=blog, site=blog.site, data_integer=f.id)
def check_publishing_lock(blog, action_description, warn_only=False): ''' Checks for a publishing lock and returns a status message if busy. ''' try: Queue.acquire(blog) except QueueInProgressException as e: msg = "{} is not available right now. Proceed with caution. Reason: {}".format( action_description, e) if warn_only is True: return Status( type='warning', message=msg, no_sure=True, ) else: raise QueueInProgressException(msg) else: return None
def queue_archive_template_fast(template_id, action, pass_id=0): from core.models import Template, Queue template = Template.load(template_id) blog = template.blog from core import cms from core.libs.bottle import HTTPResponse r = HTTPResponse() fileinfos = FileInfo.select().where( FileInfo.template_mapping << template.mappings).paginate(pass_id, 50) # TODO: if action is fast and no fileinfos present, redirect to full rebuild? if fileinfos.count() > 0: r.body = "Adding {}".format(pass_id * 50) for f in fileinfos: Queue.push(job_type=cms.queue.job_type.archive, blog=blog, site=blog.site, data_integer=f.id) pass_id += 1 r.add_header( 'Refresh', "0;{}/template/{}/{}/fast/{}".format(BASE_PATH, template_id, action, pass_id)) else: r.body = "Queue insertion finished." if action == 'publish': redir = 'publish' else: redir = 'queue' r.add_header('Refresh', "0;{}/blog/{}/{}".format(BASE_PATH, blog.id, redir)) return r
def queue_archive_template_all(template_id, action, pass_id=0): from core.models import Template, Queue template = Template.load(template_id) blog = template.blog from core.cms import fileinfo, queue from core.libs.bottle import HTTPResponse r = HTTPResponse() pages = blog.pages.published.paginate(pass_id, 50) if pages.count() > 0: r.body = "Adding {}".format(pass_id * 50) for f in fileinfo.build_archives_fileinfos_by_mappings(template, pages): Queue.push(job_type=queue.job_type.archive, blog=blog, site=blog.site, data_integer=f.id) pass_id += 1 r.add_header('Refresh', "0;{}/template/{}/{}/all/{}".format( BASE_PATH, template_id, action, pass_id)) else: r.body = "Queue insertion finished." if action == 'publish': redir = 'publish' else: redir = 'queue' r.add_header('Refresh', "0;{}/blog/{}/{}".format( BASE_PATH, blog.id, redir)) return r
def queue_ssi_actions(blog): ''' Pushes to the publishing queue all the SSIs for a given blog. :param blog: The blog object whose SSI templates will be pushed to the queue. ''' templates = blog.ssi_templates.select() if templates.count() == 0: return None for n in templates: for f in n.fileinfos: Queue.push( job_type=job_type.include, priority=10, blog=blog, site=blog.site, data_integer=f.id)
def blog_break_queue(blog_id): user = auth.is_logged_in(request) blog = Blog.load(blog_id) permission = auth.is_blog_publisher(user, blog) Queue.stop(blog) tags = template_tags(blog=blog, user=user) return template('queue/queue_run_ui', start=None, action_url='', start_message=''' <p>Queue publishing stopped. Note that queued items are still in the queue, and may still be processed on the next queue run.</p> <p><a href="{}/blog/{}/queue/clear"><button class="btn">Clear the queue</button></a> to remove them entirely.</p> '''.format(BASE_URL, blog_id), title='Publishing queue progress', # search_context=(search_context['blog_queue'], blog), menu=generate_menu('blog_queue', blog), **tags.__dict__)
def blog_break_queue(blog_id): user = auth.is_logged_in(request) blog = Blog.load(blog_id) permission = auth.is_blog_publisher(user, blog) Queue.stop(blog) tags = template_tags(blog=blog, user=user) return template( 'queue/queue_run_ui', start=None, action_url='', start_message=''' <p>Queue publishing stopped. Note that queued items are still in the queue, and may still be processed on the next queue run.</p> <p><a href="{}/blog/{}/queue/clear"><button class="btn">Clear the queue</button></a> to remove them entirely.</p> '''.format(BASE_URL, blog_id), title='Publishing queue progress', # search_context=(search_context['blog_queue'], blog), menu=generate_menu('blog_queue', blog), **tags.__dict__)
def queue_archive_template_all(template_id, action, pass_id=0): from core.models import Template, Queue template = Template.load(template_id) blog = template.blog from core.cms import fileinfo, queue from core.libs.bottle import HTTPResponse r = HTTPResponse() pages = blog.pages.published.paginate(pass_id, 50) if pages.count() > 0: r.body = "Adding {}".format(pass_id * 50) for f in fileinfo.build_archives_fileinfos_by_mappings( template, pages): Queue.push(job_type=queue.job_type.archive, blog=blog, site=blog.site, data_integer=f.id) pass_id += 1 r.add_header( 'Refresh', "0;{}/template/{}/{}/all/{}".format(BASE_PATH, template_id, action, pass_id)) else: r.body = "Queue insertion finished." if action == 'publish': redir = 'publish' else: redir = 'queue' r.add_header('Refresh', "0;{}/blog/{}/{}".format(BASE_PATH, blog.id, redir)) return r
def publishing_lock(blog, return_queue=False): ''' Checks to see if a publishing job for a given blog is currently running. If it is, it raises an exception. If the return_queue flag is set, it returns the queue_control object instead. If no job is locked, then it returns None. ''' try: queue_control = Queue.get(Queue.blog == blog, Queue.is_control == True) except Queue.DoesNotExist: return None if return_queue is True: return queue_control else: raise QueueInProgressException( "Publishing job currently running for blog {}".format( queue_control.blog.for_log))
def manage_queue(request, queue_id=None): # Try to locate the object to use it as an instance and if not, create a new one to use it # in a new form. # common_data = common_ticket_data() if request.user.is_superuser: if queue_id: actual_queue = get_object_or_404(Queue, pk=queue_id) else: actual_queue = Queue() # POST mode if request.method == 'POST': form = QueueForm(request.POST, instance=actual_queue) if form.is_valid(): form.save() return redirect("/settings/queue") else: # Non-POST mode, show only form = QueueForm(instance=actual_queue) return render(request, 'queues/create_edit_queue.html', locals()) else: raise Http404
def template_save(request, user, cms_template, blog=None): ''' Core logic for saving changes to a template. ''' # TODO: move the bulk of this into the actual model # the .getunicode stuff should be moved out, # make that part of the ui # we should just submit cms_template as self, # make whatever mods to it are needed in the ui func, # and perform the validation we did elsewhere, perhaps from core.cms import fileinfo, invalidate_cache from core.utils import is_blank from core.error import TemplateSaveException, PageNotChanged import datetime status = [] _forms = request.forms cms_template.title = _forms.getunicode('template_title') cms_template.body = _forms.getunicode('template_body') if is_blank(cms_template.title): cms_template.title = "New Template (#{})".format(cms_template.id) mode = _forms.getunicode('publishing_mode') if mode in publishing_mode.modes: cms_template.publishing_mode = mode else: raise TemplateSaveException("Invalid publishing mode selected.") cms_template.modified_date = datetime.datetime.utcnow() try: cms_template.save(user) except PageNotChanged as e: status.append("(Template unchanged.)") except Exception as e: raise e new_mappings = [] for n in _forms: if n.startswith('template_mapping_'): mapping_id = int(n[len('template_mapping_'):]) try: template_mapping = TemplateMapping.get( TemplateMapping.id == mapping_id) except TemplateMapping.DoesNotExist: raise TemplateSaveException( 'Template mapping with ID #{} does not exist.'.format( mapping_id)) else: if is_blank(_forms.getunicode(n)): raise TemplateSaveException( 'Template mapping #{} ({}) cannot be blank. Use None to specify no mapping.' .format(mapping_id, template_mapping.path_string)) else: if _forms.getunicode(n) != template_mapping.path_string: template_mapping.path_string = _forms.getunicode(n) new_mappings.append(template_mapping) for n in new_mappings: n.save() status.append("Mapping #{} ({}) rebuilt.".format(n.id, n.path_string)) if new_mappings: fileinfo.build_mapping_xrefs(new_mappings) build_action = "all" else: build_action = "fast" invalidate_cache() # TODO: eventually everything after this will be removed b/c of AJAX save # tags = template_tags(template_id=cms_template.id, user=user) save_action = _forms.getunicode('save') from core.libs.bottle import response from settings import BASE_URL from core.models import Queue x_open = False if int(save_action) in (2, 3): if cms_template.template_type == template_type.page: x_open = True response.add_header( 'X-Open', '{}/template/{}/queue/{}'.format(BASE_URL, cms_template.id, build_action)) if cms_template.template_type == template_type.archive: x_open = True response.add_header( 'X-Open', '{}/template/{}/queue/{}'.format(BASE_URL, cms_template.id, build_action)) if cms_template.template_type in (template_type.include, template_type.index): # I don't think this is needed anymore, we can remove it # TODO: test it # if new_mappings: # cms.build_archives_fileinfos_by_mappings(cms_template) for f in cms_template.fileinfos_published: Queue.push(job_type=f.template_mapping.template.template_type, blog=cms_template.blog, site=cms_template.blog.site, data_integer=f.id) status.append( "{} files regenerated from template and sent to publishing queue.". format(cms_template.fileinfos_published.count())) if blog is not None: blog.theme_modified = True blog.save() from core.log import logger logger.info("Template {} edited by user {}.".format( cms_template.for_log, user.for_log)) response.body = ' '.join(status) if x_open: return response else: return response.body
def queue_page_actions(pages, no_neighbors=False, no_archive=False): ''' Pushes a Page object along with all its related items into the queue for publication. This includes any archive indices associated with the page, and the page's next and previous entries in its respective categories. Note that this will only queue items that are actually set to be published. :param page: The Page object whose actions are to be queued. :param no_neighbors: Set to True to suppress generation of next/previous posts. Useful if you've loaded all the posts for a blog into a queue and don't need to have this performed here. :param no_archive: Set to True to suppress generation of archive pages associated with this page. Also useful for mass-queued actions. ''' if pages is None: return for page in pages: if page is None: continue try: blog, site = page.blog, page.blog.site for f in page.fileinfos: if f.template_mapping.template.publishing_mode != publishing_mode.do_not_publish: Queue.push(job_type=job_type.page, blog=blog, site=site, priority=8, data_integer=f.id) if no_archive is False: queue_page_archive_actions(page) if no_neighbors is False: next_page = page.next_page previous_page = page.previous_page if next_page is not None: for f in next_page.fileinfos: if f.template_mapping.template.publishing_mode != publishing_mode.do_not_publish: Queue.push(job_type=job_type.page, blog=blog, site=site, priority=8, data_integer=f.id) queue_page_archive_actions(next_page) if previous_page is not None: for f in previous_page.fileinfos: if f.template_mapping.template.publishing_mode != publishing_mode.do_not_publish: Queue.push(job_type=job_type.page, blog=blog, site=site, priority=8, data_integer=f.id) queue_page_archive_actions(previous_page) except OperationalError as e: raise e except Exception as e: from core.error import QueueAddError raise QueueAddError('Page {} could not be queued: '.format( page.for_log, e))
scheduled_pages = Page.select().where( Page.status == page_status.scheduled, Page.publication_date <= datetime.datetime.utcnow()).order_by( Page.publication_date.desc()) total_pages = scheduled_pages.count() print('{} pages scheduled'.format(total_pages)) if total_pages > 0: for p in scheduled_pages.select(Page.blog).distinct(): b = p.blog blogs_to_check[b.id] = b queue_count = Queue.select(Queue.blog).distinct() if queue_count.count() > 0: for n in queue_count: b = n.blog print('Blog {} has existing queue items'.format(b.id)) blogs_to_check[b.id] = b if blogs_to_check: print("Starting run.") from core.cms.queue import (queue_page_actions, queue_index_actions, queue_ssi_actions) from core.models import db from core.log import logger from time import sleep
def _remove_from_queue(queue_deletes): ''' Batch deletion of queue jobs. ''' deletes = Queue.delete().where(Queue.id << queue_deletes) return deletes.execute()
def queue_page_actions(pages, no_neighbors=False, no_archive=False): ''' Pushes a Page object along with all its related items into the queue for publication. This includes any archive indices associated with the page, and the page's next and previous entries in its respective categories. Note that this will only queue items that are actually set to be published. :param page: The Page object whose actions are to be queued. :param no_neighbors: Set to True to suppress generation of next/previous posts. Useful if you've loaded all the posts for a blog into a queue and don't need to have this performed here. :param no_archive: Set to True to suppress generation of archive pages associated with this page. Also useful for mass-queued actions. ''' if pages is None: return for page in pages: if page is None: continue try: blog, site = page.blog, page.blog.site for f in page.fileinfos: if f.template_mapping.template.publishing_mode != publishing_mode.do_not_publish: Queue.push(job_type=job_type.page, blog=blog, site=site, priority=8, data_integer=f.id) if no_archive is False: queue_page_archive_actions(page) if no_neighbors is False: next_page = page.next_page previous_page = page.previous_page if next_page is not None: for f in next_page.fileinfos: if f.template_mapping.template.publishing_mode != publishing_mode.do_not_publish: Queue.push(job_type=job_type.page, blog=blog, site=site, priority=8, data_integer=f.id) queue_page_archive_actions(next_page) if previous_page is not None: for f in previous_page.fileinfos: if f.template_mapping.template.publishing_mode != publishing_mode.do_not_publish: Queue.push(job_type=job_type.page, blog=blog, site=site, priority=8, data_integer=f.id) queue_page_archive_actions(previous_page) except OperationalError as e: raise e except Exception as e: from core.error import QueueAddError raise QueueAddError('Page {} could not be queued: '.format( page.for_log, e))
def process_queue_publish(queue_control, blog): ''' Processes the publishing queue for a given blog. Takes in a queue_control entry, and returns an integer of the number of jobs remaining in the queue for that blog. Typically invoked by the process_queue function. :param queue_control: The queue_control entry, from the queue, to use for this publishing queue run. :param blog: The blog object that is in context for this job. ''' # from . import invalidate_cache # invalidate_cache() queue_control.lock() queue_original = Queue.select().order_by(Queue.priority.desc(), Queue.date_touched.desc()).where(Queue.blog == blog, Queue.is_control == False) queue = queue_original.limit(MAX_BATCH_OPS).naive() queue_original_length = queue_original.count() queue_length = queue.count() start_queue = time.clock() if queue_length > 0: logger.info("Queue job #{} @ {} (blog #{}, {} items) started.".format( queue_control.id, date_format(queue_control.date_touched), queue_control.blog.id, queue_original_length)) removed_jobs = [] start = time.clock() for q in queue: job_type.action[q.job_type](q) removed_jobs.append(q.id) if (time.clock() - start) > LOOP_TIMEOUT: break Queue.remove(removed_jobs) # we don't need to have an entirely new job! # we should recycle the existing one, yes? new_queue_control = Queue.control_job(blog) # new_queue_control = Queue.get(Queue.blog == blog, # Queue.is_control == True) queue_original_length -= len(removed_jobs) new_queue_control.data_integer = queue_original_length end_queue = time.clock() total_time = end_queue - start_queue if new_queue_control.data_integer <= 0: new_queue_control.delete_instance() logger.info("Queue job #{} @ {} (blog #{}) finished ({:.4f} secs).".format( new_queue_control.id, date_format(new_queue_control.date_touched), new_queue_control.blog.id, total_time)) else: # new_queue_control.is_running = False # new_queue_control.save() new_queue_control.unlock() logger.info("Queue job #{} @ {} (blog #{}) processed {} items ({:.4f} secs, {} remaining).".format( new_queue_control.id, date_format(new_queue_control.date_touched), new_queue_control.blog.id, len(removed_jobs), total_time, queue_original_length, )) return new_queue_control.data_integer
def process_queue_publish(queue_control, blog): ''' Processes the publishing queue for a given blog. Takes in a queue_control entry, and returns an integer of the number of jobs remaining in the queue for that blog. Typically invoked by the process_queue function. :param queue_control: The queue_control entry, from the queue, to use for this publishing queue run. :param blog: The blog object that is in context for this job. ''' # from . import invalidate_cache # invalidate_cache() queue_control.lock() queue_original = Queue.select().order_by(Queue.priority.desc(), Queue.date_touched.desc()).where( Queue.blog == blog, Queue.is_control == False) queue = queue_original.limit(MAX_BATCH_OPS).naive() queue_original_length = queue_original.count() queue_length = queue.count() start_queue = time.clock() if queue_length > 0: logger.info("Queue job #{} @ {} (blog #{}, {} items) started.".format( queue_control.id, date_format(queue_control.date_touched), queue_control.blog.id, queue_original_length)) removed_jobs = [] start = time.clock() for q in queue: job_type.action[q.job_type](q) removed_jobs.append(q.id) if (time.clock() - start) > LOOP_TIMEOUT: break Queue.remove(removed_jobs) # we don't need to have an entirely new job! # we should recycle the existing one, yes? new_queue_control = Queue.control_job(blog) # new_queue_control = Queue.get(Queue.blog == blog, # Queue.is_control == True) queue_original_length -= len(removed_jobs) new_queue_control.data_integer = queue_original_length end_queue = time.clock() total_time = end_queue - start_queue if new_queue_control.data_integer <= 0: new_queue_control.delete_instance() logger.info( "Queue job #{} @ {} (blog #{}) finished ({:.4f} secs).".format( new_queue_control.id, date_format(new_queue_control.date_touched), new_queue_control.blog.id, total_time)) else: # new_queue_control.is_running = False # new_queue_control.save() new_queue_control.unlock() logger.info( "Queue job #{} @ {} (blog #{}) processed {} items ({:.4f} secs, {} remaining)." .format( new_queue_control.id, date_format(new_queue_control.date_touched), new_queue_control.blog.id, len(removed_jobs), total_time, queue_original_length, )) return new_queue_control.data_integer
def push_to_queue(**ka): ''' Inserts a single job item into the work queue. :param job_type: A string representing the type of job to be inserted. 'Page','Index', eventually 'Fileinfo' :param data_integer: Any integer data passed along with the job. For a job control item, this is the number of items remaining for that particular job. :param blog: The blog object associated with the job. :param site: The site object associated with the job. :param priority: An integer, from 0-9, representing the processing priority associated with the job. Higher-priority jobs are processed first. Most individual pages are given a high priority; indexes are lower. ''' try: queue_job = Queue.get(Queue.job_type == ka['job_type'], Queue.data_integer == ka['data_integer'], Queue.blog == ka['blog'], Queue.site == ka['site']) except Queue.DoesNotExist: queue_job = Queue() else: return queue_job.job_type = ka['job_type'] queue_job.data_integer = int(ka.get('data_integer', None)) queue_job.blog = ka.get('blog', Blog()).id queue_job.site = ka.get('site', Site()).id queue_job.priority = ka.get('priority', 9) queue_job.is_control = ka.get('is_control', False) if queue_job.is_control: queue_job.data_string = (queue_job.job_type + ": Blog {}".format(queue_job.blog.for_log)) else: queue_job.data_string = ( queue_job.job_type + ": " + FileInfo.get(FileInfo.id == queue_job.data_integer).file_path) queue_job.date_touched = datetime.datetime.now() queue_job.save()
scheduled_pages = Page.select().where( Page.status == page_status.scheduled, Page.publication_date <= datetime.datetime.utcnow()).order_by( Page.publication_date.desc()) total_pages = scheduled_pages.count() print ('{} pages scheduled'.format(total_pages)) if total_pages > 0: for p in scheduled_pages.select(Page.blog).distinct(): b = p.blog blogs_to_check[b.id] = b queue_count = Queue.select(Queue.blog).distinct() if queue_count.count() > 0: for n in queue_count: b = n.blog print ('Blog {} has existing queue items'.format(b.id)) blogs_to_check[b.id] = b if blogs_to_check: print ("Starting run.") from core.cms.queue import (queue_page_actions, queue_index_actions, queue_ssi_actions) from core.models import db from core.log import logger from time import sleep