def test_publish_no_s3_bucket_configured(root_doc):
"""Test the publish task when no S3 bucket is configured."""
log_mock = mock.Mock()
doc_pks = [root_doc.pk]
publish(doc_pks, log=log_mock)
log_mock.info.assert_called_once_with(
'Skipping publish of {!r}: no S3 bucket configured'.format(doc_pks))
def test_publish_no_s3_bucket_configured(root_doc):
"""Test the publish task when no S3 bucket is configured."""
log_mock = mock.Mock()
doc_pks = [root_doc.pk]
publish(doc_pks, log=log_mock)
log_mock.info.assert_called_once_with(
'Skipping publish of {!r}: no S3 bucket configured'.format(doc_pks))
def handle(self, *args, **options):
Logger = namedtuple('Logger', 'info, error')
log = Logger(info=self.stdout.write, error=self.stderr.write)
if options['all'] or options['locale']:
filters = {}
if options['locale'] and not options['all']:
locale = options['locale']
log.info(
'Publishing all documents in locale {}'.format(locale))
filters.update(locale=locale)
else:
locale = None
log.info('Publishing all documents')
chunk_size = max(options['chunk_size'], 1)
docs = Document.objects.filter(**filters)
doc_pks = docs.values_list('id', flat=True)
num_docs = len(doc_pks)
num_tasks = int(ceil(num_docs / float(chunk_size)))
log.info('...found {} documents.'.format(num_docs))
# Let's publish the documents in a group of chunks, where the
# tasks in the group can be run in parallel.
tasks = []
for i, chunk in enumerate(chunked(doc_pks, chunk_size)):
message = 'Published chunk #{} of {}'.format(i + 1, num_tasks)
tasks.append(publish.si(chunk, completion_message=message))
if num_tasks == 1:
msg = ('Launching a single task handling '
'all {} documents.'.format(num_docs))
else:
msg = ('Launching {} paralellizable tasks, each handling '
'at most {} documents.'.format(num_tasks, chunk_size))
log.info(msg)
group(*tasks).apply_async()
else:
paths = options['paths']
if not paths:
raise CommandError(
'Need at least one document path to publish')
doc_pks = []
get_doc_pk = Document.objects.values_list('id', flat=True).get
for path in paths:
if path.startswith('/'):
path = path[1:]
locale, sep, slug = path.partition('/')
head, sep, tail = slug.partition('/')
if head == 'docs':
slug = tail
try:
doc_pk = get_doc_pk(locale=locale, slug=slug)
except Document.DoesNotExist:
msg = 'Document with locale={} and slug={} does not exist'
log.error(msg.format(locale, slug))
else:
doc_pks.append(doc_pk)
publish(doc_pks, log=log)
def handle(self, *args, **options):
Logger = namedtuple('Logger', 'info, error')
log = Logger(info=self.stdout.write, error=self.stderr.write)
if options['all'] or options['locale']:
filters = {}
if options['locale'] and not options['all']:
locale = options['locale']
log.info('Publishing all documents in locale {}'.format(locale))
filters.update(locale=locale)
else:
locale = None
log.info('Publishing all documents')
chunk_size = max(options['chunk_size'], 1)
docs = Document.objects.filter(**filters)
doc_pks = docs.values_list('id', flat=True)
num_docs = len(doc_pks)
num_tasks = int(ceil(num_docs / float(chunk_size)))
log.info('...found {} documents.'.format(num_docs))
# Let's publish the documents in a group of chunks, where the
# tasks in the group can be run in parallel.
tasks = []
for i, chunk in enumerate(chunked(doc_pks, chunk_size)):
message = 'Published chunk #{} of {}'.format(i + 1, num_tasks)
tasks.append(publish.si(chunk, completion_message=message))
if num_tasks == 1:
msg = ('Launching a single task handling '
'all {} documents.'.format(num_docs))
else:
msg = ('Launching {} paralellizable tasks, each handling '
'at most {} documents.'.format(num_tasks, chunk_size))
log.info(msg)
group(*tasks).apply_async()
else:
paths = options['paths']
if not paths:
raise CommandError('Need at least one document path to publish')
doc_pks = []
get_doc_pk = Document.objects.values_list('id', flat=True).get
for path in paths:
if path.startswith('/'):
path = path[1:]
locale, sep, slug = path.partition('/')
head, sep, tail = slug.partition('/')
if head == 'docs':
slug = tail
try:
doc_pk = get_doc_pk(locale=locale, slug=slug)
except Document.DoesNotExist:
msg = 'Document with locale={} and slug={} does not exist'
log.error(msg.format(locale, slug))
else:
doc_pks.append(doc_pk)
publish(doc_pks, log=log)
def test_publish_multiple(get_s3_bucket_mock, root_doc, redirect_doc,
redirect_to_home, trans_doc):
"""
Test the publish task for multiple documents of various kinds, including
standard documents and redirects.
"""
trans_doc.delete()
log_mock = mock.Mock()
get_s3_bucket_mock.return_value = s3_bucket_mock = get_mocked_s3_bucket()
publish(
[trans_doc.pk, root_doc.pk, redirect_doc.pk, redirect_to_home.pk],
log=log_mock,
completion_message="Done!",
)
s3_bucket_mock.put_object.assert_has_calls([
mock.call(
ACL="public-read",
Key=get_s3_key(root_doc),
Body=json.dumps(document_api_data(root_doc)),
ContentType="application/json",
ContentLanguage=root_doc.locale,
),
mock.call(
ACL="public-read",
Key=get_s3_key(redirect_doc),
WebsiteRedirectLocation=get_s3_key(
root_doc,
prefix_with_forward_slash=True,
suffix_file_extension=False,
),
ContentType="application/json",
ContentLanguage=redirect_doc.locale,
Body=json.dumps(
document_api_data(
redirect_url=get_content_based_redirect(redirect_doc)[0])),
),
mock.call(
ACL="public-read",
Key=get_s3_key(redirect_to_home),
Body=json.dumps(document_api_data(redirect_url="/en-US/")),
ContentType="application/json",
ContentLanguage=redirect_to_home.locale,
),
])
log_mock.error.assert_called_once_with(
"Document with pk={} does not exist".format(trans_doc.pk))
log_mock.info.assert_has_calls([
mock.call("Published S3 Object #1"),
mock.call("Published S3 Object #2"),
mock.call("Published S3 Object #3"),
mock.call("Done!"),
])
def test_publish_standard(get_s3_bucket_mock, root_doc):
"""Test the publish task for a standard (non-redirect) document."""
log_mock = mock.Mock()
get_s3_bucket_mock.return_value = s3_bucket_mock = get_mocked_s3_bucket()
publish.get_logger = mock.Mock(return_value=log_mock)
publish([root_doc.pk])
s3_bucket_mock.put_object.assert_called_once_with(
ACL='public-read',
Key=get_s3_key(root_doc),
Body=json.dumps(document_api_data(root_doc, ensure_contributors=True)),
ContentType='application/json',
ContentLanguage=root_doc.locale
)
log_mock.info.assert_called_once_with('Published S3 Object #1')
def test_publish_redirect_to_home(get_s3_bucket_mock, redirect_to_home):
"""
Test the publish task for a document that redirects to a URL outside the
S3 bucket, in this case the home page.
"""
log_mock = mock.Mock()
get_s3_bucket_mock.return_value = s3_bucket_mock = get_mocked_s3_bucket()
publish([redirect_to_home.pk], log=log_mock)
s3_bucket_mock.put_object.assert_called_once_with(
ACL='public-read',
Key=get_s3_key(redirect_to_home),
Body=json.dumps(document_api_data(redirect_url='/en-US/')),
ContentType='application/json',
ContentLanguage=redirect_to_home.locale)
log_mock.info.assert_called_once_with('Published S3 Object #1')
def test_publish_redirect_to_home(get_s3_bucket_mock, redirect_to_home):
"""
Test the publish task for a document that redirects to a URL outside the
S3 bucket, in this case the home page.
"""
log_mock = mock.Mock()
get_s3_bucket_mock.return_value = s3_bucket_mock = get_mocked_s3_bucket()
publish([redirect_to_home.pk], log=log_mock)
s3_bucket_mock.put_object.assert_called_once_with(
ACL='public-read',
Key=get_s3_key(redirect_to_home),
Body=json.dumps(document_api_data(redirect_url='/en-US/')),
ContentType='application/json',
ContentLanguage=redirect_to_home.locale
)
log_mock.info.assert_called_once_with('Published S3 Object #1')
def test_publish_redirect(get_s3_bucket_mock, root_doc, redirect_doc):
"""
Test the publish task for a document that redirects to another document
within the S3 bucket.
"""
log_mock = mock.Mock()
get_s3_bucket_mock.return_value = s3_bucket_mock = get_mocked_s3_bucket()
publish([redirect_doc.pk], log=log_mock)
s3_bucket_mock.put_object.assert_called_once_with(
ACL='public-read',
Key=get_s3_key(redirect_doc),
WebsiteRedirectLocation=get_s3_key(root_doc, for_redirect=True),
ContentType='application/json',
ContentLanguage=redirect_doc.locale
)
log_mock.info.assert_called_once_with('Published S3 Object #1')
def test_publish_redirect_to_other(get_s3_bucket_mock, redirect_to_macros):
"""
Test the publish task for a document that redirects to a URL outside the
S3 bucket, in this case someting other than the home page.
"""
log_mock = mock.Mock()
get_s3_bucket_mock.return_value = s3_bucket_mock = get_mocked_s3_bucket()
publish([redirect_to_macros.pk], log=log_mock)
s3_bucket_mock.put_object.assert_called_once_with(
ACL='public-read',
Key=get_s3_key(redirect_to_macros),
Body=json.dumps(
document_api_data(redirect_url=absolutify(
'/en-US/dashboards/macros', for_wiki_site=True))),
ContentType='application/json',
ContentLanguage=redirect_to_macros.locale)
log_mock.info.assert_called_once_with('Published S3 Object #1')
def test_publish_redirect_to_other(get_s3_bucket_mock, redirect_to_macros):
"""
Test the publish task for a document that redirects to a URL outside the
S3 bucket, in this case someting other than the home page.
"""
log_mock = mock.Mock()
get_s3_bucket_mock.return_value = s3_bucket_mock = get_mocked_s3_bucket()
publish([redirect_to_macros.pk], log=log_mock)
s3_bucket_mock.put_object.assert_called_once_with(
ACL='public-read',
Key=get_s3_key(redirect_to_macros),
Body=json.dumps(document_api_data(
redirect_url=absolutify('/en-US/dashboards/macros',
for_wiki_site=True))),
ContentType='application/json',
ContentLanguage=redirect_to_macros.locale
)
log_mock.info.assert_called_once_with('Published S3 Object #1')
def test_publish_standard(get_s3_bucket_mock, root_doc, invalidate_cdn_cache):
"""Test the publish task for a standard (non-redirect) document."""
log_mock = mock.Mock()
get_s3_bucket_mock.return_value = s3_bucket_mock = get_mocked_s3_bucket()
publish.get_logger = mock.Mock(return_value=log_mock)
with mock.patch('kuma.api.tasks.request_cdn_cache_invalidation') as mocked:
publish([root_doc.pk], invalidate_cdn_cache=invalidate_cdn_cache)
if invalidate_cdn_cache:
mocked.delay.assert_called_once_with([(root_doc.locale,
root_doc.slug)])
else:
mocked.delay.assert_not_called()
s3_bucket_mock.put_object.assert_called_once_with(
ACL='public-read',
Key=get_s3_key(root_doc),
Body=json.dumps(document_api_data(root_doc)),
ContentType='application/json',
ContentLanguage=root_doc.locale)
log_mock.info.assert_called_once_with('Published S3 Object #1')
def test_publish_multiple(get_s3_bucket_mock, root_doc, redirect_doc,
redirect_to_home, trans_doc):
"""
Test the publish task for multiple documents of various kinds, including
standard documents and redirects.
"""
trans_doc.delete()
log_mock = mock.Mock()
get_s3_bucket_mock.return_value = s3_bucket_mock = get_mocked_s3_bucket()
publish([trans_doc.pk, root_doc.pk, redirect_doc.pk, redirect_to_home.pk],
log=log_mock, completion_message='Done!')
s3_bucket_mock.put_object.assert_has_calls([
mock.call(
ACL='public-read',
Key=get_s3_key(root_doc),
Body=json.dumps(
document_api_data(root_doc, ensure_contributors=True)),
ContentType='application/json',
ContentLanguage=root_doc.locale
),
mock.call(
ACL='public-read',
Key=get_s3_key(redirect_doc),
WebsiteRedirectLocation=get_s3_key(root_doc, for_redirect=True),
ContentType='application/json',
ContentLanguage=redirect_doc.locale
),
mock.call(
ACL='public-read',
Key=get_s3_key(redirect_to_home),
Body=json.dumps(document_api_data(redirect_url='/en-US/')),
ContentType='application/json',
ContentLanguage=redirect_to_home.locale
),
])
log_mock.error.assert_called_once_with(
'Document with pk={} does not exist'.format(trans_doc.pk))
log_mock.info.assert_has_calls([
mock.call('Published S3 Object #1'),
mock.call('Published S3 Object #2'),
mock.call('Published S3 Object #3'),
mock.call('Done!'),
])
def test_publish_redirect(get_s3_bucket_mock, root_doc, redirect_doc):
"""
Test the publish task for a document that redirects to another document
within the S3 bucket.
"""
log_mock = mock.Mock()
get_s3_bucket_mock.return_value = s3_bucket_mock = get_mocked_s3_bucket()
publish([redirect_doc.pk], log=log_mock)
s3_bucket_mock.put_object.assert_called_once_with(
ACL='public-read',
Key=get_s3_key(redirect_doc),
WebsiteRedirectLocation=get_s3_key(
root_doc, prefix_with_forward_slash=True, suffix_file_extension=False),
ContentType='application/json',
ContentLanguage=redirect_doc.locale,
Body=json.dumps(document_api_data(None, redirect_url=get_content_based_redirect(redirect_doc)[0]))
)
log_mock.info.assert_called_once_with('Published S3 Object #1')
def handle(self, *args, **options):
Logger = namedtuple("Logger", "info, error")
log = Logger(info=self.stdout.write, error=self.stderr.write)
if options["all"] or options["locale"]:
if options["locale"] and options["all"]:
raise CommandError(
"Specifying --locale with --all is the same as --all"
)
filters = {}
if options["locale"]:
locale = options["locale"]
log.info("Publishing all documents in locale {}".format(locale))
filters.update(locale=locale)
else:
log.info("Publishing all documents")
chunk_size = max(options["chunk_size"], 1)
docs = Document.objects.filter(**filters)
doc_pks = docs.values_list("id", flat=True)
num_docs = len(doc_pks)
num_tasks = int(ceil(num_docs / float(chunk_size)))
log.info("...found {} documents.".format(num_docs))
# Let's publish the documents in a group of chunks, where the
# tasks in the group can be run in parallel.
tasks = []
for i, chunk in enumerate(chunked(doc_pks, chunk_size)):
message = "Published chunk #{} of {}".format(i + 1, num_tasks)
tasks.append(
publish.si(
chunk, completion_message=message, invalidate_cdn_cache=False
)
)
if num_tasks == 1:
msg = "Launching a single task handling " "all {} documents.".format(
num_docs
)
else:
msg = (
"Launching {} paralellizable tasks, each handling "
"at most {} documents.".format(num_tasks, chunk_size)
)
log.info(msg)
group(*tasks).apply_async()
else:
paths = options["paths"]
if not paths:
raise CommandError("Need at least one document path to publish")
doc_pks = []
get_doc_pk = Document.objects.values_list("id", flat=True).get
for path in paths:
if path.startswith("/"):
path = path[1:]
locale, sep, slug = path.partition("/")
head, sep, tail = slug.partition("/")
if head == "docs":
slug = tail
try:
doc_pk = get_doc_pk(locale=locale, slug=slug)
except Document.DoesNotExist:
msg = "Document with locale={} and slug={} does not exist"
log.error(msg.format(locale, slug))
else:
doc_pks.append(doc_pk)
publish(
doc_pks,
log=log,
invalidate_cdn_cache=(not options["skip_cdn_invalidation"]),
)
