def test_import_remote(self):
repo_name = parse_repo_name(self.workspace.working_dir)
repo_location = 'http://localhost:8080/repos/%s.json' % repo_name
responses.add_callback(
responses.POST,
'http://localhost:8080/repos.json',
callback=lambda _: (301, {'Location': repo_location}, ''))
self.tool.run(
config=(self.yaml_config, self.config),
verbose=True,
clobber=False,
repo_dir=self.working_dir,
repo_url=self.workspace.working_dir,
ini_config=self.ini_config,
ini_section='app:main',
update_config=True,
repo_name=None,
repo_host='http://localhost:8080')
cp = ConfigParser()
cp.read(self.ini_config)
self.assertEqual(
cp.get('app:main', 'unicore.content_repo_urls').strip(),
repo_location)
with open(self.yaml_config, 'r') as fp:
data = yaml.safe_load(fp)
self.assertEqual(data['repositories'], {
repo_name: self.workspace.working_dir
})
def __init__(self, request):
self.request = request
self.language = request.locale_name
self.settings = request.registry.settings
es_host = self.settings.get('es.host', 'http://localhost:9200')
self.es_settings = {
'urls': [es_host]
}
repo_dir = self.settings.get('unicore.repos_dir', 'repos')
repo_names = config_list(self.settings['unicore.content_repos'])
self.all_repo_paths = map(
lambda repo_name: os.path.join(repo_dir, repo_name),
repo_names)
self.all_index_prefixes = map(
lambda repo_path_or_name: parse_repo_name(repo_path_or_name),
repo_names)
search_config = {
'in_': self.all_repo_paths,
'index_prefixes': self.all_index_prefixes
}
self.all_categories = SM(Category, **search_config).es(
**self.es_settings)
self.all_pages = SM(Page, **search_config).es(**self.es_settings)
self.all_localisations = SM(Localisation, **search_config).es(
**self.es_settings)
self.available_languages = config_list(
self.settings.get('available_languages', ''))
self.featured_languages = config_list(
self.settings.get('featured_languages', ''))
self.display_languages = list(
set(self.featured_languages) - set([self.language]))
def test_import_local(self):
self.tool.run(
config=(self.yaml_config, self.config),
verbose=True,
clobber=False,
repo_dir=self.working_dir,
repo_url=self.workspace.working_dir,
ini_config=self.ini_config,
ini_section='app:main',
update_config=True,
repo_name=None,
repo_host=None)
cp = ConfigParser()
cp.read(self.ini_config)
self.assertEqual(
cp.get('app:main', 'unicore.content_repo_urls').strip(),
os.path.basename(self.workspace.working_dir))
with open(self.yaml_config, 'r') as fp:
data = yaml.safe_load(fp)
repo_name = parse_repo_name(self.workspace.working_dir)
self.assertEqual(data['repositories'], {
repo_name: self.workspace.working_dir
})
def __init__(self, request):
self.request = request
self.language = request.locale_name
self.settings = request.registry.settings
es_host = self.settings.get('es.host', 'http://localhost:9200')
self.es_settings = {'urls': [es_host]}
repo_dir = self.settings.get('unicore.repos_dir', 'repos')
repo_names = config_list(self.settings['unicore.content_repos'])
self.all_repo_paths = map(
lambda repo_name: os.path.join(repo_dir, repo_name), repo_names)
self.all_index_prefixes = map(
lambda repo_path_or_name: parse_repo_name(repo_path_or_name),
repo_names)
search_config = {
'in_': self.all_repo_paths,
'index_prefixes': self.all_index_prefixes
}
self.all_categories = SM(Category,
**search_config).es(**self.es_settings)
self.all_pages = SM(Page, **search_config).es(**self.es_settings)
self.all_localisations = SM(Localisation,
**search_config).es(**self.es_settings)
self.available_languages = config_list(
self.settings.get('available_languages', ''))
self.featured_languages = config_list(
self.settings.get('featured_languages', ''))
self.display_languages = list(
set(self.featured_languages) - set([self.language]))
def pull(repo_url, index_prefix=None, es=None):
if is_remote_repo_url(repo_url):
sm = RemoteStorageManager(repo_url)
sm.pull()
else:
index_prefix = index_prefix or parse_repo_name(repo_url)
workspace = EG.workspace(repo_url, index_prefix=index_prefix, es=es)
workspace.pull()
def test_multiple_repos(self):
workspace1 = self.workspace
workspace2 = self.mk_workspace(name='test_multiple_repos-2')
testing.setUp(
settings={
'unicore.repos_dir':
self.working_dir,
'unicore.content_repos':
'\n%s\n%s' % (workspace1.working_dir, workspace2.working_dir),
})
views = CoreViews(self.mk_request())
indexes = map(lambda path: '%s-master' % parse_repo_name(path),
[workspace1.working_dir, workspace2.working_dir])
self.assertEqual(indexes, views.all_pages.get_indexes())
self.assertEqual(indexes, views.all_categories.get_indexes())
def test_multiple_repos(self):
workspace1 = self.workspace
workspace2 = self.mk_workspace(name="test_multiple_repos-2")
testing.setUp(
settings={
"unicore.repos_dir": self.working_dir,
"unicore.content_repos": "\n%s\n%s" % (workspace1.working_dir, workspace2.working_dir),
}
)
views = CoreViews(self.mk_request())
indexes = map(
lambda path: "%s-master" % parse_repo_name(path), [workspace1.working_dir, workspace2.working_dir]
)
self.assertEqual(indexes, views.all_pages.get_indexes())
self.assertEqual(indexes, views.all_categories.get_indexes())
def run(self, config, verbose, clobber, repo_dir, repo_url,
ini_config, ini_section, update_config, repo_name):
config_file, config_data = config
repo_name = repo_name or parse_repo_name(repo_url)
workdir, _ = self.clone_repo(repo_name=repo_name,
repo_url=repo_url,
repo_dir=repo_dir,
clobber=clobber,
verbose=verbose)
self.bootstrap(
workdir,
config_data.get('models', {}).items(),
clobber=clobber,
verbose=verbose)
if not update_config:
return
repositories = config_data.setdefault('repositories', {})
if repo_name not in repositories:
repositories[repo_name] = repo_url
with open(config_file, 'w') as fp:
yaml.safe_dump(config_data,
stream=fp, default_flow_style=False)
self.emit('Added %s to the %s config file.' % (
repo_name, config_file))
config_key = 'unicore.content_repos'
cp = ConfigParser()
cp.read(ini_config)
if not cp.has_section(ini_section):
cp.add_section(ini_section)
existing_repo_names = (cp.get(ini_section, config_key)
if cp.has_option(ini_section, config_key)
else '')
existing_repo_names = config_list(existing_repo_names)
if repo_name not in existing_repo_names:
cp.set(ini_section, config_key,
'\n'.join(chain(existing_repo_names, [repo_name])))
with open(ini_config, 'w') as fp:
cp.write(fp)
self.emit(
'Updated unicore.content_repos in %s.' % (ini_config,))
def run(self, config, verbose, clobber, repo_dir, repo_url,
ini_config, ini_section, update_config, repo_name, repo_host):
repo_name = repo_name or parse_repo_name(repo_url)
if repo_host:
repo_location = self.import_remote(
config, verbose, clobber, repo_host, repo_url, repo_name)
else:
repo_location = self.import_local(
config, verbose, clobber, repo_dir, repo_url, repo_name)
if not update_config:
return
config_file, config_data = config
repositories = config_data.setdefault('repositories', {})
if repo_name not in repositories:
repositories[repo_name] = repo_url
with open(config_file, 'w') as fp:
yaml.safe_dump(config_data,
stream=fp, default_flow_style=False)
self.emit('Added %s to the %s config file.' % (
repo_name, config_file))
config_key = 'unicore.content_repo_urls'
cp = ConfigParser()
cp.read(ini_config)
if not cp.has_section(ini_section):
cp.add_section(ini_section)
existing_repos = (cp.get(ini_section, config_key)
if cp.has_option(ini_section, config_key)
else '')
existing_repos = config_list(existing_repos)
if repo_location not in existing_repos:
cp.set(ini_section, config_key,
'\n'.join(chain(existing_repos, [repo_location])))
with open(ini_config, 'w') as fp:
cp.write(fp)
self.emit(
'Updated unicore.content_repo_urls in %s.' % (ini_config,))
def run(self, config, verbose, clobber, repo_dir, repo_url, ini_config,
ini_section, update_config, repo_name, repo_host):
repo_name = repo_name or parse_repo_name(repo_url)
if repo_host:
repo_location = self.import_remote(config, verbose, clobber,
repo_host, repo_url, repo_name)
else:
repo_location = self.import_local(config, verbose, clobber,
repo_dir, repo_url, repo_name)
if not update_config:
return
config_file, config_data = config
repositories = config_data.setdefault('repositories', {})
if repo_name not in repositories:
repositories[repo_name] = repo_url
with open(config_file, 'w') as fp:
yaml.safe_dump(config_data,
stream=fp,
default_flow_style=False)
self.emit('Added %s to the %s config file.' %
(repo_name, config_file))
config_key = 'unicore.content_repo_urls'
cp = ConfigParser()
cp.read(ini_config)
if not cp.has_section(ini_section):
cp.add_section(ini_section)
existing_repos = (cp.get(ini_section, config_key) if cp.has_option(
ini_section, config_key) else '')
existing_repos = config_list(existing_repos)
if repo_location not in existing_repos:
cp.set(ini_section, config_key,
'\n'.join(chain(existing_repos, [repo_location])))
with open(ini_config, 'w') as fp:
cp.write(fp)
self.emit('Updated unicore.content_repo_urls in %s.' %
(ini_config, ))
def assertParsed(self, expected, dsn):
return self.assertEqual(expected, parse_repo_name(dsn))
def assertParsed(self, expected, dsn):
return self.assertEqual(expected, parse_repo_name(dsn))