def test_gather_redirect_to_errored_zone_page_is_error(): """If a document is a redirect to an errored zone, doc is also errored.""" parent_path = '/en-US/docs/Root' path = parent_path + '/Zone' zone_path = '/en-US/Zone' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document', 'get_document_rendered', 'get_zone_root', 'get_document_metadata', 'get_document_history' ]) storage.get_document_rendered.return_value = {'redirect_to': zone_path} storage.get_zone_root.return_value = {'errors': 'bad zone'} source.gather(None, storage) assert source.state == source.STATE_ERROR
def test_gather_redirect_to_zone_subpage_complete(): """A zoned subpage has more data passed to storage.save_document()""" parent_path = '/en-US/docs/Root/Zone' path = parent_path + '/Child' zone_root_path = '/en-US/Zone' zone_path = zone_root_path + '/Child' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document', 'get_document_rendered', 'get_zone_root', 'get_document_metadata', 'get_document_history', 'save_document' ]) storage.get_document.return_value = 'Root doc' storage.get_document_rendered.return_value = {'redirect_to': zone_path} storage.get_zone_root.return_value = { 'doc_path': parent_path, 'zone_path': '/en-US/Zone' } metadata = doc_metadata.copy() storage.get_document_metadata.return_value = metadata storage.get_document_history.return_value = [('revisions', path + '$revision/2018', {})] resources = source.gather(None, storage) assert resources == [('document_current', path, {'revisions': 1})] assert source.state == source.STATE_DONE expected = doc_data.copy() expected['slug'] = 'Root/Zone/Child' expected['parent_topic'] = 'Root doc' expected['zone_redirect_path'] = zone_root_path assert storage.save_document.call_count == 1 assert storage.save_document.call_args[0][0] == expected # Better diff storage.save_document.assert_called_once_with(expected)
def test_gather_localized_doc_invalid_english(): """An invalid English document path is an error.""" path = '/fr/docs/Racine' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document_rendered', 'get_document_metadata', 'get_document_history' ]) storage.get_document_rendered.return_value = {} metadata = doc_metadata.copy() metadata['locale'] = 'fr' metadata['slug'] = 'Racine' metadata['url'] = path metadata['translations'] = [ { 'locale': 'en-US', 'url': '/en-US/ZoneRoot' }, ] storage.get_document_metadata.return_value = metadata storage.get_document_history.return_value = [('revisions', path + '$revision/2020', {})] resources = source.gather(None, storage) assert resources == [] assert source.state == source.STATE_ERROR
def test_gather_redirect_to_zone_subpage(): """If a document is a redirect to zone subpage, request the zone root.""" parent_path = '/en-US/docs/Root/Zone' path = parent_path + '/Child' zone_root_path = '/en-US/Zone' zone_path = zone_root_path + '/Child' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document', 'get_document_rendered', 'get_zone_root', 'get_document_metadata', 'get_document_history', 'save_document' ]) storage.get_document_rendered.return_value = {'redirect_to': zone_path} storage.get_zone_root.return_value = { 'doc_path': parent_path, 'zone_path': zone_root_path } resources = source.gather(None, storage) assert resources == [ ('document', parent_path, {}), # Parerent of current page ('document', parent_path, {}), # zone_path from zone root ('document_meta', path, { 'force': True }), ('document_history', path, { 'revisions': 1 }), ] assert source.state == source.STATE_PREREQ
def test_gather_localized_doc_with_metadata(): """A localized document will request the English document.""" path = '/fr/docs/Racine' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document', 'get_document_rendered', 'get_document_metadata', 'get_document_history', 'save_document' ]) storage.get_document_rendered.return_value = {} metadata = doc_metadata.copy() metadata['locale'] = 'fr' metadata['slug'] = 'Racine' metadata['url'] = path metadata['translations'] = [ { 'locale': 'es', 'url': '/es/docs/Raíz' }, { 'locale': 'en-US', 'url': '/en-US/docs/Root' }, ] storage.get_document_metadata.return_value = metadata storage.get_document_history.return_value = [('revisions', path + '$revision/2020', {})] resources = source.gather(None, storage) assert resources == [('document', '/en-US/docs/Root', {})] assert source.state == source.STATE_PREREQ
def test_gather_localized_doc_sets_parent(): """A localized document will set the English document as parent.""" path = '/fr/docs/Racine' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document', 'get_document_rendered', 'get_document_metadata', 'get_document_history', 'save_document' ]) storage.get_document_rendered.return_value = {} metadata = doc_metadata.copy() metadata['locale'] = 'fr' metadata['slug'] = 'Racine' metadata['url'] = path metadata['translations'] = [ { 'locale': 'en-US', 'url': '/en-US/docs/Root' }, ] storage.get_document.return_value = 'English document' storage.get_document_metadata.return_value = metadata storage.get_document_history.return_value = [('revisions', path + '$revision/2020', {})] resources = source.gather(None, storage) assert resources == [('document_current', path, {'revisions': 1})] assert source.state == source.STATE_DONE expected = doc_data.copy() expected['locale'] = 'fr' expected['slug'] = 'Racine' expected['parent'] = 'English document' assert storage.save_document.call_count == 1 assert storage.save_document.call_args[0][0] == expected # Better diff storage.save_document.assert_called_once_with(expected)
def test_gather_zoned_doc_init(): """A zone URL requests the zone doc.""" path = '/en-US/Zone' source = DocumentSource(path, force=True) storage = mock_storage(spec=['get_zone_root']) resources = source.gather(None, storage) assert resources == [('zone_root', path, {})] assert source.state == source.STATE_PREREQ
def test_gather_root_no_prereqs(): doc_path = '/en-US/docs/RootDoc' source = DocumentSource(doc_path) storage = mock_storage(spec=['get_document', 'get_document_rendered']) resources = source.gather(None, storage) assert resources == [('document_rendered', doc_path, {})] assert source.state == source.STATE_PREREQ assert source.freshness == source.FRESH_UNKNOWN
def test_gather_document_in_storage(): """If the Document already exists, short-circuit downloads.""" source = DocumentSource('/en-US/docs/Root/Child') storage = mock_storage(spec=['get_document']) storage.get_document.return_value = "existing document" resources = source.gather(None, storage) assert resources == [] assert source.state == source.STATE_DONE assert source.freshness == source.FRESH_NO
def test_gather_zoned_doc_error(): """If the zoned document fails (isn't a zone), then the doc errors too.""" path = '/en-US/Zone' source = DocumentSource(path, force=True) storage = mock_storage(spec=['get_zone_root']) storage.get_zone_root.return_value = {'errors': ['failed']} resources = source.gather(None, storage) assert resources == [] assert source.state == source.STATE_ERROR
def test_gather_forced(): """Resources are fetched if force=True.""" doc_path = '/en-US/docs/RootDoc' source = DocumentSource(doc_path, force=True) storage = mock_storage(spec=['get_document', 'get_document_rendered']) storage.get_document.return_value = "existing document" resources = source.gather(None, storage) assert resources == [('document_rendered', doc_path, {})] assert source.state == source.STATE_PREREQ assert source.freshness == source.FRESH_UNKNOWN
def test_gather_child_doc(): """A child document requires the parent document.""" parent_path = '/en-US/docs/Root' child_path = parent_path + '/Child' source = DocumentSource(child_path) storage = mock_storage(spec=['get_document', 'get_document_rendered']) resources = source.gather(None, storage) assert resources == [('document', parent_path, {}), ('document_rendered', child_path, {})] assert source.state == source.STATE_PREREQ
def test_gather_normalized_path_moved_page_needed(): """If a document is a redirect, request the target page.""" source = DocumentSource('/en-US/docs/Origin', force=True) storage = mock_storage(spec=['get_document', 'get_document_rendered']) storage.get_document_rendered.return_value = { 'redirect_to': '/en-US/docs/NewLocation' } resources = source.gather(None, storage) assert resources == [('document', '/en-US/docs/NewLocation', {})] storage.get_document.assert_called_once_with('en-US', 'NewLocation') assert source.state == source.STATE_PREREQ
def test_gather_child_doc_parent_in_storage(): """If the parent document is available, it is not requested.""" parent_path = '/en-US/docs/Root' child_path = parent_path + '/Child' source = DocumentSource(child_path, force=True) storage = mock_storage(spec=['get_document', 'get_document_rendered']) storage.get_document.return_value = 'parent document' resources = source.gather(None, storage) assert resources == [('document_rendered', child_path, {})] storage.get_document.assert_called_once_with('en-US', 'Root') assert source.state == source.STATE_PREREQ
def test_gather_standard_doc_empty_history_is_error(): path = '/en-US/docs/RootDoc' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document_rendered', 'get_document_metadata', 'get_document_history' ]) storage.get_document_rendered.return_value = {} # Standard doc storage.get_document_metadata.return_value = {} # Empty for now storage.get_document_history.return_value = [] # No history resources = source.gather(None, storage) assert resources == [] assert source.state == source.STATE_ERROR
def test_gather_localized_doc_without_metadata(): """A localized document will wait for metadata.""" path = '/fr/docs/Racine' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document_rendered', 'get_document_metadata', 'get_document_history', 'save_document' ]) storage.get_document_rendered.return_value = {} storage.get_document_history.return_value = [('revisions', path + '$revision/2020', {})] resources = source.gather(None, storage) assert resources == [('document_meta', path, {'force': True})] assert source.state == source.STATE_PREREQ
def test_gather_standard_doc_all_prereqs(): path = '/en-US/docs/Test' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document_rendered', 'get_document_metadata', 'get_document_history', 'save_document' ]) storage.get_document_rendered.return_value = {} # Standard doc storage.get_document_metadata.return_value = doc_metadata storage.get_document_history.return_value = [('revisions', path + '$revision/2016', {})] resources = source.gather(None, storage) assert resources == [('document_current', path, {'revisions': 1})] assert source.state == source.STATE_DONE storage.save_document.assert_called_once_with(doc_data)
def test_gather_standard_doc_bad_metadata(): """If the metadata has an error, so does the document.""" path = '/en-US/docs/Test' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document_rendered', 'get_document_metadata', 'get_document_history' ]) storage.get_document_rendered.return_value = {} # Standard doc metadata = doc_metadata.copy() metadata['error'] = True storage.get_document_metadata.return_value = metadata storage.get_document_history.return_value = [('revisions', path + '$revision/2016', {})] resources = source.gather(None, storage) assert resources == [] assert source.state == source.STATE_ERROR
def test_gather_standard_doc(): """If the rendered document is standard, get next resources.""" path = '/en-US/docs/RootDoc' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document_rendered', 'get_document_metadata', 'get_document_history' ]) storage.get_document_rendered.return_value = {} resources = source.gather(None, storage) assert resources == [('document_meta', path, { 'force': True }), ('document_history', path, { 'revisions': 1 })] assert source.state == source.STATE_PREREQ storage.get_document_metadata.assert_called_once_with('en-US', 'RootDoc') storage.get_document_history.assert_called_once_with('en-US', 'RootDoc')
def test_gather_document_children_loaded(): """If the $children resource is loaded, use it.""" doc_path = '/en-US/docs/RootDoc' source = DocumentSource(doc_path, depth='all', force=True) storage = mock_storage(spec=[ 'get_document_rendered', 'get_document_metadata', 'get_document_history', 'get_document_children', 'save_document' ]) storage.get_document_rendered.return_value = {} # Standard doc storage.get_document_metadata.return_value = doc_metadata storage.get_document_history.return_value = [ ('revisions', doc_path + '$revision/2016', {}) ] storage.get_document_children.return_value = [] resources = source.gather(None, storage) assert resources == [('document_current', doc_path, {'revisions': 1})] assert source.state == source.STATE_DONE assert source.freshness == source.FRESH_YES
def test_gather_standard_doc_metdata_loses(): """If metadata doesn't match URL, use locale and slug from URL.""" path = '/en-US/docs/Test' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document_rendered', 'get_document_metadata', 'get_document_history', 'save_document' ]) storage.get_document_rendered.return_value = {} # Standard doc metadata = doc_metadata.copy() metadata['locale'] = 'EN-US' metadata['slug'] = 'TEST' storage.get_document_metadata.return_value = metadata storage.get_document_history.return_value = [('revisions', path + '$revision/2016', {})] resources = source.gather(None, storage) assert resources == [('document_current', path, {'revisions': 1})] assert source.state == source.STATE_DONE storage.save_document.assert_called_once_with(doc_data)
def test_gather_zoned_doc_is_normalized(): """The zoned doc is used to normalize the URL.""" path = '/en-US/Zone' source = DocumentSource(path, force=True) assert not source.normalized_path assert not source.locale assert not source.slug storage = mock_storage( spec=['get_zone_root', 'get_document', 'get_document_rendered']) storage.get_zone_root.return_value = { 'zone_path': path, 'doc_path': '/en-US/docs/Root/Zone' } resources = source.gather(None, storage) assert resources == [('document', '/en-US/docs/Root', {}), ('document_rendered', '/en-US/docs/Root/Zone', {})] assert source.state == source.STATE_PREREQ assert source.normalized_path == '/en-US/docs/Root/Zone' assert source.locale == 'en-US' assert source.slug == 'Root/Zone'
def test_gather_document_children(): """A depth > 0 will require the $children resource.""" doc_path = '/en-US/docs/RootDoc' source = DocumentSource(doc_path, depth=1, force=True) storage = mock_storage(spec=[ 'get_document_rendered', 'get_document_metadata', 'get_document_history', 'get_document_children', 'save_document' ]) storage.get_document_rendered.return_value = {} # Standard doc storage.get_document_metadata.return_value = doc_metadata storage.get_document_history.return_value = [ ('revisions', doc_path + '$revision/2016', {}) ] resources = source.gather(None, storage) assert resources == [('document_children', doc_path, { 'depth': 1, 'force': True })] assert source.state == source.STATE_PREREQ assert source.freshness == source.FRESH_UNKNOWN
def test_gather_normalized_path_moved_page_followed(): """If a document is a redirect to a normal page, create a redirect.""" source = DocumentSource('/en-US/docs/Origin', force=True) storage = mock_storage( spec=['get_document', 'get_document_rendered', 'save_document']) storage.get_document_rendered.return_value = { 'redirect_to': '/en-US/docs/NewLocation' } storage.get_document.return_value = "Redirect Document" resources = source.gather(None, storage) assert resources == [('document_current', '/en-US/docs/Origin', { 'revisions': 1 })] assert source.state == source.STATE_DONE expected_data = { 'locale': 'en-US', 'slug': 'Origin', 'redirect_to': '/en-US/docs/NewLocation' } storage.save_document.assert_called_once_with(expected_data)
def test_gather_redirect_to_zone_page_first_pass(): """If a document is a redirect to a zone, request the zone root.""" parent_path = '/en-US/docs/Root' path = parent_path + '/Zone' zone_path = '/en-US/Zone' source = DocumentSource(path, force=True) storage = mock_storage(spec=[ 'get_document', 'get_document_rendered', 'get_zone_root', 'get_document_metadata', 'get_document_history' ]) storage.get_document_rendered.return_value = {'redirect_to': zone_path} resources = source.gather(None, storage) assert resources == [('document', parent_path, {}), ('zone_root', zone_path, {}), ('document_meta', path, { 'force': True }), ('document_history', path, { 'revisions': 1 })] assert source.state == source.STATE_PREREQ