Exemple #1
0
class ExampleZipperTest(unittest.TestCase):
    def setUp(self):
        object_store_creator = ObjectStoreCreator.ForTest()
        self._file_system = CachingFileSystem(
            LocalFileSystem(os.path.join(sys.path[0], 'test_data')),
            object_store_creator)
        self._example_zipper = ExampleZipper(
            CompiledFileSystem.Factory(self._file_system,
                                       object_store_creator), 'example_zipper')

    def testCreateZip(self):
        # Cache manifest.json as unicode and make sure ExampleZipper doesn't error.
        self._file_system.ReadSingle('example_zipper/basic/manifest.json')
        self.assertTrue(len(self._example_zipper.Create('basic')) > 0)
Exemple #2
0
class ServerInstance(object):
    # Lazily create so we don't create github file systems unnecessarily in
    # tests.
    branch_utility = None
    github_file_system = None

    @staticmethod
    @memoize
    def GetOrCreateOffline(channel):
        '''Gets/creates a local ServerInstance, meaning that only resources local to
    the server - memcache, object store, etc, are queried. This amounts to not
    setting up the subversion nor github file systems.
    '''
        branch_utility = ServerInstance._GetOrCreateBranchUtility()
        branch = branch_utility.GetBranchNumberForChannelName(channel)
        object_store_creator_factory = ObjectStoreCreator.Factory(
            GetAppVersion(), branch)
        # No svn nor github file systems. Rely on the crons to fill the caches, and
        # for the caches to exist.
        return ServerInstance(
            channel,
            object_store_creator_factory,
            CachingFileSystem(OfflineFileSystem(SubversionFileSystem),
                              object_store_creator_factory),
            # TODO(kalman): convert GithubFileSystem to be wrappable in a
            # CachingFileSystem so that it can be replaced with an
            # OfflineFileSystem. Currently GFS doesn't set the child versions of
            # stat requests so it doesn't.
            ServerInstance._GetOrCreateGithubFileSystem())

    @staticmethod
    def CreateOnline(channel):
        '''Creates/creates an online server instance, meaning that both local and
    subversion/github resources are queried.
    '''
        branch_utility = ServerInstance._GetOrCreateBranchUtility()
        branch = branch_utility.GetBranchNumberForChannelName(channel)

        if branch == 'trunk':
            svn_url = '/'.join((url_constants.SVN_TRUNK_URL, 'src',
                                svn_constants.EXTENSIONS_PATH))
        else:
            svn_url = '/'.join((url_constants.SVN_BRANCH_URL, branch, 'src',
                                svn_constants.EXTENSIONS_PATH))

        viewvc_url = svn_url.replace(url_constants.SVN_URL,
                                     url_constants.VIEWVC_URL)

        object_store_creator_factory = ObjectStoreCreator.Factory(
            GetAppVersion(), branch, start_empty=True)

        svn_file_system = CachingFileSystem(
            SubversionFileSystem(AppEngineUrlFetcher(svn_url),
                                 AppEngineUrlFetcher(viewvc_url)),
            object_store_creator_factory)

        return ServerInstance(channel, object_store_creator_factory,
                              svn_file_system,
                              ServerInstance._GetOrCreateGithubFileSystem())

    @staticmethod
    def CreateForTest(file_system):
        return ServerInstance('test', ObjectStoreCreator.TestFactory(),
                              file_system, None)

    @staticmethod
    def _GetOrCreateBranchUtility():
        if ServerInstance.branch_utility is None:
            ServerInstance.branch_utility = BranchUtility(
                url_constants.OMAHA_PROXY_URL, AppEngineUrlFetcher())
        return ServerInstance.branch_utility

    @staticmethod
    def _GetOrCreateGithubFileSystem():
        # Initialising github is pointless if samples are disabled, since it's only
        # used for apps samples.
        if ServerInstance.github_file_system is None:
            if _IsSamplesDisabled():
                ServerInstance.github_file_system = EmptyDirFileSystem()
            else:
                ServerInstance.github_file_system = GithubFileSystem(
                    AppEngineUrlFetcher(url_constants.GITHUB_URL),
                    AppEngineBlobstore())
        return ServerInstance.github_file_system

    def __init__(self, channel, object_store_creator_factory, svn_file_system,
                 github_file_system):
        self.svn_file_system = svn_file_system

        self.github_file_system = github_file_system

        self.compiled_fs_factory = CompiledFileSystem.Factory(
            svn_file_system, object_store_creator_factory)

        self.api_list_data_source_factory = APIListDataSource.Factory(
            self.compiled_fs_factory, svn_constants.API_PATH,
            svn_constants.PUBLIC_TEMPLATE_PATH)

        self.api_data_source_factory = APIDataSource.Factory(
            self.compiled_fs_factory, svn_constants.API_PATH)

        self.ref_resolver_factory = ReferenceResolver.Factory(
            self.api_data_source_factory, self.api_list_data_source_factory,
            object_store_creator_factory)

        self.api_data_source_factory.SetReferenceResolverFactory(
            self.ref_resolver_factory)

        # Note: samples are super slow in the dev server because it doesn't support
        # async fetch, so disable them. If you actually want to test samples, then
        # good luck, and modify _IsSamplesDisabled at the top.
        if _IsSamplesDisabled():
            svn_fs_for_samples = EmptyDirFileSystem()
        else:
            svn_fs_for_samples = self.svn_file_system
        self.samples_data_source_factory = SamplesDataSource.Factory(
            channel, svn_fs_for_samples, self.github_file_system,
            self.ref_resolver_factory, object_store_creator_factory,
            svn_constants.EXAMPLES_PATH)

        self.api_data_source_factory.SetSamplesDataSourceFactory(
            self.samples_data_source_factory)

        self.intro_data_source_factory = IntroDataSource.Factory(
            self.compiled_fs_factory, self.ref_resolver_factory,
            [svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])

        self.sidenav_data_source_factory = SidenavDataSource.Factory(
            self.compiled_fs_factory, svn_constants.JSON_PATH)

        self.template_data_source_factory = TemplateDataSource.Factory(
            channel, self.api_data_source_factory,
            self.api_list_data_source_factory, self.intro_data_source_factory,
            self.samples_data_source_factory, self.sidenav_data_source_factory,
            self.compiled_fs_factory, self.ref_resolver_factory,
            svn_constants.PUBLIC_TEMPLATE_PATH,
            svn_constants.PRIVATE_TEMPLATE_PATH)

        self.example_zipper = ExampleZipper(self.compiled_fs_factory,
                                            svn_constants.DOCS_PATH)

        self.path_canonicalizer = PathCanonicalizer(channel,
                                                    self.compiled_fs_factory)

        self.content_cache = self.compiled_fs_factory.GetOrCreateIdentity()

    def _FetchStaticResource(self, path, response):
        """Fetch a resource in the 'static' directory.
    """
        mimetype = mimetypes.guess_type(path)[0] or 'text/plain'
        result = self.content_cache.GetFromFile(
            svn_constants.DOCS_PATH + '/' + path,
            binary=_IsBinaryMimetype(mimetype))
        response.headers['content-type'] = mimetype
        return result

    def Get(self, path, request, response):
        templates = self.template_data_source_factory.Create(request, path)

        content = None
        try:
            if fnmatch(path, 'extensions/examples/*.zip'):
                content = self.example_zipper.Create(
                    path[len('extensions/'):-len('.zip')])
                response.headers['content-type'] = 'application/zip'
            elif path.startswith('extensions/examples/'):
                mimetype = mimetypes.guess_type(path)[0] or 'text/plain'
                content = self.content_cache.GetFromFile(
                    '%s/%s' %
                    (svn_constants.DOCS_PATH, path[len('extensions/'):]),
                    binary=_IsBinaryMimetype(mimetype))
                response.headers['content-type'] = 'text/plain'
            elif path.startswith('static/'):
                content = self._FetchStaticResource(path, response)
            elif path.endswith('.html'):
                content = templates.Render(path)
        except FileNotFoundError as e:
            logging.warning(traceback.format_exc())

        response.headers['x-frame-options'] = 'sameorigin'
        if content is None:
            response.set_status(404)
            response.out.write(templates.Render('404'))
        else:
            if not content:
                logging.error('%s had empty content' % path)
            response.headers['cache-control'] = 'max-age=300'
            response.out.write(content)
Exemple #3
0
class ServerInstance(object):
    # Lazily create so we don't create github file systems unnecessarily in
    # tests.
    branch_utility = None
    github_file_system = None

    @staticmethod
    @memoize
    def GetOrCreateOffline(channel):
        '''Gets/creates a local ServerInstance, meaning that only resources local to
    the server - memcache, object store, etc, are queried. This amounts to not
    setting up the subversion nor github file systems.
    '''
        branch_utility = ServerInstance._GetOrCreateBranchUtility()
        branch = branch_utility.GetBranchNumberForChannelName(channel)
        object_store_creator_factory = ObjectStoreCreator.Factory(branch)
        # No svn nor github file systems. Rely on the crons to fill the caches, and
        # for the caches to exist.
        return ServerInstance(
            channel,
            object_store_creator_factory,
            CachingFileSystem(OfflineFileSystem(SubversionFileSystem),
                              object_store_creator_factory),
            # TODO(kalman): convert GithubFileSystem to be wrappable in a
            # CachingFileSystem so that it can be replaced with an
            # OfflineFileSystem. Currently GFS doesn't set the child versions of
            # stat requests so it doesn't.
            ServerInstance._GetOrCreateGithubFileSystem())

    @staticmethod
    @memoize
    def GetOrCreateOnline(channel):
        '''Creates/creates an online server instance, meaning that both local and
    subversion/github resources are queried.
    '''
        branch_utility = ServerInstance._GetOrCreateBranchUtility()
        branch = branch_utility.GetBranchNumberForChannelName(channel)

        if branch == 'trunk':
            svn_url = '/'.join((url_constants.SVN_TRUNK_URL, 'src',
                                svn_constants.EXTENSIONS_PATH))
        else:
            svn_url = '/'.join((url_constants.SVN_BRANCH_URL, branch, 'src',
                                svn_constants.EXTENSIONS_PATH))

        viewvc_url = svn_url.replace(url_constants.SVN_URL,
                                     url_constants.VIEWVC_URL)

        object_store_creator_factory = ObjectStoreCreator.Factory(branch)

        svn_file_system = CachingFileSystem(
            SubversionFileSystem(AppEngineUrlFetcher(svn_url),
                                 AppEngineUrlFetcher(viewvc_url)),
            object_store_creator_factory)

        return ServerInstance(channel, object_store_creator_factory,
                              svn_file_system,
                              ServerInstance._GetOrCreateGithubFileSystem())

    @staticmethod
    def CreateForTest(file_system):
        return ServerInstance('test', ObjectStoreCreator.Factory('test'),
                              file_system, None)

    @staticmethod
    def _GetOrCreateBranchUtility():
        if ServerInstance.branch_utility is None:
            ServerInstance.branch_utility = BranchUtility(
                url_constants.OMAHA_PROXY_URL, AppEngineUrlFetcher())
        return ServerInstance.branch_utility

    @staticmethod
    def _GetOrCreateGithubFileSystem():
        if ServerInstance.github_file_system is None:
            ServerInstance.github_file_system = GithubFileSystem(
                AppEngineUrlFetcher(url_constants.GITHUB_URL),
                AppEngineBlobstore())
        return ServerInstance.github_file_system

    def __init__(self, channel, object_store_creator_factory, svn_file_system,
                 github_file_system):
        self.svn_file_system = svn_file_system

        self.github_file_system = github_file_system

        self.compiled_fs_factory = CompiledFileSystem.Factory(
            svn_file_system, object_store_creator_factory)

        self.api_list_data_source_factory = APIListDataSource.Factory(
            self.compiled_fs_factory, svn_constants.API_PATH,
            svn_constants.PUBLIC_TEMPLATE_PATH)

        self.api_data_source_factory = APIDataSource.Factory(
            self.compiled_fs_factory, svn_constants.API_PATH)

        self.ref_resolver_factory = ReferenceResolver.Factory(
            self.api_data_source_factory, self.api_list_data_source_factory,
            object_store_creator_factory)

        self.api_data_source_factory.SetReferenceResolverFactory(
            self.ref_resolver_factory)

        self.samples_data_source_factory = SamplesDataSource.Factory(
            channel, self.svn_file_system, ServerInstance.github_file_system,
            self.ref_resolver_factory, object_store_creator_factory,
            svn_constants.EXAMPLES_PATH)

        self.api_data_source_factory.SetSamplesDataSourceFactory(
            self.samples_data_source_factory)

        self.intro_data_source_factory = IntroDataSource.Factory(
            self.compiled_fs_factory, self.ref_resolver_factory,
            [svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])

        self.sidenav_data_source_factory = SidenavDataSource.Factory(
            self.compiled_fs_factory, svn_constants.JSON_PATH)

        self.template_data_source_factory = TemplateDataSource.Factory(
            channel, self.api_data_source_factory,
            self.api_list_data_source_factory, self.intro_data_source_factory,
            self.samples_data_source_factory, self.sidenav_data_source_factory,
            self.compiled_fs_factory, self.ref_resolver_factory,
            svn_constants.PUBLIC_TEMPLATE_PATH,
            svn_constants.PRIVATE_TEMPLATE_PATH)

        self.example_zipper = ExampleZipper(self.compiled_fs_factory,
                                            svn_constants.DOCS_PATH)

        self.path_canonicalizer = PathCanonicalizer(channel,
                                                    self.compiled_fs_factory)

        self.content_cache = self.compiled_fs_factory.GetOrCreateIdentity()

    def _FetchStaticResource(self, path, response):
        """Fetch a resource in the 'static' directory.
    """
        mimetype = mimetypes.guess_type(path)[0] or 'text/plain'
        result = self.content_cache.GetFromFile(
            svn_constants.DOCS_PATH + '/' + path,
            binary=_IsBinaryMimetype(mimetype))
        response.headers['content-type'] = mimetype
        return result

    def Get(self, path, request, response):
        if os.environ.get('CRXDOCZH_SLAVE_TYPE') == 'samples':
            logging.error('NOTREACHED: slave-samples: server_instance: Get')
        elif os.environ.get('CRXDOCZH_SLAVE_TYPE') == 'docs':
            if not self._GetDocsAPI(path, request, response):
                self._OriginalGet(path, request, response)
        else:
            self._OriginalGet(path, request, response)

    def _GetSamplesJSON(self, request, key, update=False):
        templates = self.template_data_source_factory.Create(
            request, key + '/samples.html')
        if key == 'apps' or key == 'extensions':
            cache_url = None
            if key == 'apps':
                cache_url = '/trunk/' + key
            else:
                cache_url = '/' + templates._branch_info['current'] + '/' + key
            logging.info('Serving API request %s' % cache_url)

            data = ResponseCache.Get(cache_url)
            age = ResponseCache.GetAge(cache_url)
            need_update = (age > datetime.timedelta(1))
            logging.info('Cache age: %s' % str(age))
            if data is not None and not (update == True
                                         and need_update == True):
                return data
            else:
                content = []
                try:
                    logging.info('Trying to generate samples.json for %s' %
                                 cache_url)
                    content = templates._samples_data_source.GetAsJSON(key)
                except:
                    logging.getLogger('slave-samples-api').exception(
                        'Error generating samples!')
                    pass
                if len(content) > 0:
                    logging.info('samples.json saved for %s' % cache_url)
                    ResponseCache.Set(cache_url, content)
                return content

    def _GetDocsAPI(self, path, request, response):
        if path.startswith(url_constants.SLAVE_DOCS_API_BASE_URL):
            new_path = path[len(url_constants.SLAVE_DOCS_API_BASE_URL):]
            if new_path.startswith('static/'):
                response.headers['content-type'] = 'text/plain'
                response.out.write(
                    self._FetchStaticResource(new_path, response))
                return True
            if not new_path.endswith('.html'):
                return False
            content = self._GetDocsHTML(request, new_path)
            if len(content) > 0:
                response.headers['content-type'] = 'text/plain'
                response.out.write(content)
            else:
                response.set_status(503)
            return True
        else:
            return False

    def _GetDocsHTML(self, request, path, update=False):
        templates = self.template_data_source_factory.Create(request, path)
        cache_url = '/' + templates._branch_info['current'] + '/' + path

        logging.info('Serving API request %s' % cache_url)
        data = ResponseCache.Get(cache_url)
        age = ResponseCache.GetAge(cache_url)
        need_update = (age > datetime.timedelta(1))
        logging.info('Cache age: %s' % str(age))
        if data is not None and not (update == True and need_update == True):
            return data
        else:
            content = ''
            try:
                logging.info('Trying to render %s' % cache_url)
                content = templates.Render(path)
            except:
                logging.getLogger('slave-docs-api').exception(
                    'Error rendering HTML!')
                pass
            if len(content) > 0:
                logging.info('HTML saved for %s' % cache_url)
                ResponseCache.Set(cache_url, content.encode('utf-8'))
            return content

    def _OriginalGet(self, path, request, response):
        templates = self.template_data_source_factory.Create(request, path)

        content = None
        try:
            if fnmatch(path, 'extensions/examples/*.zip'):
                content = self.example_zipper.Create(
                    path[len('extensions/'):-len('.zip')])
                response.headers['content-type'] = 'application/zip'
            elif path.startswith('extensions/examples/'):
                mimetype = mimetypes.guess_type(path)[0] or 'text/plain'
                content = self.content_cache.GetFromFile(
                    '%s/%s' %
                    (svn_constants.DOCS_PATH, path[len('extensions/'):]),
                    binary=_IsBinaryMimetype(mimetype))
                response.headers['content-type'] = 'text/plain'
            elif path.startswith('static/'):
                content = self._FetchStaticResource(path, response)
            elif path.endswith('.html'):
                content = templates.Render(path)
        except FileNotFoundError as e:
            logging.warning(e)

        response.headers['x-frame-options'] = 'sameorigin'
        if content is None:
            response.set_status(404)
            response.out.write(templates.Render('404'))
        else:
            if not content:
                logging.error('%s had empty content' % path)
            response.headers['cache-control'] = 'max-age=300'
            response.out.write(content)
            cache_url = '/' + templates._branch_info['current'] + '/' + path
            if isinstance(content, unicode):
                ResponseCache.Set(cache_url, content.encode('utf-8'))
            else:
                ResponseCache.Set(cache_url, content)