def test_apt_consume_file(self):
from infi.app_repo.indexers.apt import AptIndexer
import gzip
with self._setup_context() as config:
indexer = AptIndexer(config, 'main-stable')
indexer.initialise()
filepath = self.write_new_package_in_incoming_directory(config, extension='deb')
self.assertTrue(indexer.are_you_interested_in_file(filepath, 'linux-ubuntu-xenial', 'x86'))
self.assertFalse(indexer.are_you_interested_in_file('foo.rpm', 'linux-ubuntu-xenial', 'x86'))
indexer.consume_file(filepath, 'linux-ubuntu-xenial', 'i386')
packages_file = path.join(indexer.base_directory, 'linux-ubuntu', 'dists', 'xenial', 'main', 'binary-i386', 'Packages')
with fopen(packages_file) as fd:
packages_contents = fd.read()
self.assertNotEquals(packages_contents, '')
self.assertIn("Filename: dists/xenial/main/binary-i386/some-package.deb", packages_contents)
with gzip.open(packages_file + '.gz', 'rb') as fd:
self.assertEquals(packages_contents, fd.read())
release_dirpath = path.join(indexer.base_directory, 'linux-ubuntu', 'dists', 'xenial')
self.assertTrue(path.exists(path.join(release_dirpath, 'main', 'binary-i386', 'some-package.deb')))
release_filepath = path.join(release_dirpath, 'Release')
# with fopen(release_filepath) as fd:
# self.assertEquals(fd.read(), 'Codename: xenial\nArchitectures: amd64 i386\nComponents: main\nok')
self.assertTrue(path.exists(release_filepath))
def test_wget_consumes_ova(self):
from infi.app_repo.indexers.wget import PrettyIndexer
with self._setup_context() as config:
indexer = PrettyIndexer(config, 'main-stable')
indexer.initialise()
filepath = self.write_new_package_in_incoming_directory(
config,
package_basename=
'application-repository-0.2.31-linux-ubuntu-lucid-x86_OVF10',
extension='ova')
self.assertTrue(
indexer.are_you_interested_in_file(filepath,
'linux-ubuntu-lucid',
'x86_OVF10'))
indexer.consume_file(filepath, 'linux-ubuntu-lucid', 'x86_OVF10')
self.assertTrue(
path.exists(
path.join(
indexer.base_directory, 'packages',
'application-repository', 'releases', '0.2.31',
'distributions', 'vmware-esx', 'architectures',
'x86_OVF10', 'extensions', 'ova',
'application-repository-0.2.31-linux-ubuntu-lucid-x86_OVF10.ova'
)))
packages = read_json_file(
path.join(indexer.base_directory, 'packages.json'))
self.assertIsInstance(packages, list)
self.assertGreater(len(packages), 0)
releases = read_json_file(
path.join(indexer.base_directory, 'packages',
'application-repository', 'releases.json'))
self.assertIsInstance(packages, list)
self.assertGreater(len(packages), 0)
def test_wget_consume_file(self):
from infi.app_repo.indexers.wget import PrettyIndexer
with self._setup_context() as config:
indexer = PrettyIndexer(config, 'main-stable')
indexer.initialise()
filepath = self.write_new_package_in_incoming_directory(
config,
package_basename='my-app-0.1-linux-ubuntu-xenial-x64',
extension='deb')
self.assertTrue(
indexer.are_you_interested_in_file(filepath,
'linux-ubuntu-xenial',
'x64'))
indexer.consume_file(filepath, 'linux-ubuntu-xenial', 'x64')
self.assertTrue(
path.exists(
path.join(indexer.base_directory, 'packages', 'my-app',
'releases', '0.1', 'distributions',
'linux-ubuntu-xenial', 'architectures', 'x64',
'extensions', 'deb',
'my-app-0.1-linux-ubuntu-xenial-x64.deb')))
packages = read_json_file(
path.join(indexer.base_directory, 'packages.json'))
self.assertIsInstance(packages, list)
self.assertGreater(len(packages), 0)
releases = read_json_file(
path.join(indexer.base_directory, 'packages', 'my-app',
'releases.json'))
self.assertIsInstance(packages, list)
self.assertGreater(len(packages), 0)
def _install_service(service_name, commandline_arguments):
from infi.app_repo import PROJECTROOT
from infi.app_repo.__version__ import __version__
script = path.join(PROJECTROOT, 'bin', 'eapp_repo')
kwargs = {
'chdir': PROJECTROOT,
'exec': '{} {}'.format(script, commandline_arguments).strip(),
}
config = TEMPLATE.format(**kwargs)
with fopen(path.join(SYSTEMD, '%s.service' % service_name), 'w') as fd:
fd.write(config)
def _install_upstart_job(service_name, commandline_arguments):
from infi.app_repo import PROJECTROOT
from infi.app_repo.__version__ import __version__
script = path.join(PROJECTROOT, 'bin', 'eapp_repo')
kwargs = {'version': __version__,
'chdir': PROJECTROOT,
'exec': '{} {}'.format(script, commandline_arguments).strip(),
}
config = TEMPLATE.format(**kwargs)
with fopen(path.join(INIT, '%s.conf' % service_name), 'w') as fd:
fd.write(config)
def index_home_page(index_name):
packages_json = path.join(flask.current_app.app_repo_config.packages_directory, index_name, 'index', 'packages.json')
data = decode(read_file(packages_json))
host_url = flask.request.host_url.rstrip('/')
setup_url = '%s%s' % (host_url,
flask.url_for("client_setup_script", index_name=index_name))
return flask.Response(flask.render_template("home.html", packages=data, setup_url=setup_url, host_url=host_url, index_name=index_name))
def _archives():
rpm = flask.Blueprint("archives", __name__)
AutoIndex(rpm,
browse_root=path.join(
self.app_repo_config.artifacts_directory,
'archives'))
self.register_blueprint(rpm, url_prefix="/archives")
def index_home_page(index_name):
packages_json = path.join(flask.current_app.app_repo_config.packages_directory, index_name, 'index', 'packages.json')
data = decode(read_file(packages_json))
host_url = flask.request.host_url.rstrip('/')
setup_url = '%s%s' % (host_url,
flask.url_for("client_setup_script", index_name=index_name))
return flask.Response(flask.render_template("home.html", packages=data, setup_url=setup_url, host_url=host_url, index_name=index_name))
def gpg_key():
from infi.gevent_utils.os import fopen
with fopen(
path.join(flask.current_app.app_repo_config.packages_directory,
'gpg.key')) as fd:
return flask.Response(fd.read(),
content_type='application/octet-stream')
def _ova_updates():
ova = flask.Blueprint("ova", __name__)
AutoIndex(ova,
browse_root=path.join(
self.app_repo_config.artifacts_directory, 'ova',
'updates'))
self.register_blueprint(ova, url_prefix="/ova")
def setUp(self):
from gevent.event import Event
super(FtpWithRpcTestCase, self).setUp()
self.config = self._get_config_for_test()
ensure_directory_exists(
path.join(self.config.incoming_directory, 'main-stable'))
self.test_succeded = Event()
def restart_after():
require_restart = [service for service in SERVICES if
path.exists(path.join(INIT, '%s.conf' % service))]
for service in require_restart:
log_execute_assert_success(['stop', service], allow_to_fail=True)
yield
for service in require_restart:
log_execute_assert_success(['start', service])
def test_upload(self):
with patch.object(ftpserver.AppRepoFtpHandler, "on_file_received") as on_file_received:
on_file_received.side_effect = self.mark_success
fd = StringIO("hello world")
with self.ftp_server_context(self.config), self.ftp_client_context(self.config, True) as client:
client.storbinary("STOR testfile", fd)
self.assertTrue(self.test_succeded)
self.assertTrue(path.exists(path.join(self.config.incoming_directory, 'testfile')))
def test_wget_consumes_ova(self):
from infi.app_repo.indexers.wget import PrettyIndexer
with self._setup_context() as config:
indexer = PrettyIndexer(config, 'main-stable')
indexer.initialise()
filepath = self.write_new_package_in_incoming_directory(config, package_basename='application-repository-0.2.31-linux-ubuntu-lucid-x86_OVF10', extension='ova')
self.assertTrue(indexer.are_you_interested_in_file(filepath, 'linux-ubuntu-lucid', 'x86_OVF10'))
indexer.consume_file(filepath, 'linux-ubuntu-lucid', 'x86_OVF10')
self.assertTrue(path.exists(path.join(indexer.base_directory, 'packages', 'application-repository', 'releases', '0.2.31', 'distributions',
'vmware-esx', 'architectures', 'x86_OVF10', 'extensions', 'ova',
'application-repository-0.2.31-linux-ubuntu-lucid-x86_OVF10.ova')))
packages = read_json_file(path.join(indexer.base_directory, 'packages.json'))
self.assertIsInstance(packages, list)
self.assertGreater(len(packages), 0)
releases = read_json_file(path.join(indexer.base_directory, 'packages', 'application-repository', 'releases.json'))
self.assertIsInstance(packages, list)
self.assertGreater(len(packages), 0)
def test_wget_consume_file(self):
from infi.app_repo.indexers.wget import PrettyIndexer
with self._setup_context() as config:
indexer = PrettyIndexer(config, 'main-stable')
indexer.initialise()
filepath = self.write_new_package_in_incoming_directory(config, package_basename='my-app-0.1-linux-ubuntu-xenial-x64', extension='deb')
self.assertTrue(indexer.are_you_interested_in_file(filepath, 'linux-ubuntu-xenial', 'x64'))
indexer.consume_file(filepath, 'linux-ubuntu-xenial', 'x64')
self.assertTrue(path.exists(path.join(indexer.base_directory, 'packages', 'my-app', 'releases', '0.1', 'distributions',
'linux-ubuntu-xenial', 'architectures', 'x64', 'extensions', 'deb',
'my-app-0.1-linux-ubuntu-xenial-x64.deb')))
packages = read_json_file(path.join(indexer.base_directory, 'packages.json'))
self.assertIsInstance(packages, list)
self.assertGreater(len(packages), 0)
releases = read_json_file(path.join(indexer.base_directory, 'packages', 'my-app', 'releases.json'))
self.assertIsInstance(packages, list)
self.assertGreater(len(packages), 0)
def test_upload(self):
with patch.object(ftpserver.AppRepoFtpHandler, "on_file_received") as on_file_received:
on_file_received.side_effect = self.mark_success
fd = StringIO("hello world")
with self.ftp_server_context(self.config), self.ftp_client_context(self.config, True) as client:
client.storbinary("STOR testfile", fd)
self.assertTrue(self.test_succeded)
self.assertTrue(path.exists(path.join(self.config.incoming_directory, 'testfile')))
def restart_after():
require_restart = [
service for service in SERVICES
if path.exists(path.join(INIT, '%s.conf' % service))
]
for service in require_restart:
log_execute_assert_success(['stop', service], allow_to_fail=True)
yield
for service in require_restart:
log_execute_assert_success(['start', service])
def test_download(self):
with fopen(path.join(self.config.incoming_directory, 'testfile'), 'w') as fd:
fd.write('hello world')
with patch.object(ftpserver.AppRepoFtpHandler, "on_file_sent") as on_file_sent:
on_file_sent.side_effect = self.mark_success
with self.ftp_server_context(self.config), self.ftp_client_context(self.config) as client:
client.retrbinary('RETR incoming/testfile', lambda *args, **kwargs: None)
self.assertTrue(self.test_succeded)
def test_download(self):
with fopen(path.join(self.config.incoming_directory, 'testfile'), 'w') as fd:
fd.write('hello world')
with patch.object(ftpserver.AppRepoFtpHandler, "on_file_sent") as on_file_sent:
on_file_sent.side_effect = self.mark_success
with self.ftp_server_context(self.config), self.ftp_client_context(self.config) as client:
client.retrbinary('RETR incoming/testfile', lambda *args, **kwargs: None)
self.assertTrue(self.test_succeded)
def test_apt_consume_file(self):
from infi.app_repo.indexers.apt import AptIndexer
import gzip
with self._setup_context() as config:
indexer = AptIndexer(config, 'main-stable')
indexer.initialise()
filepath = self.write_new_package_in_incoming_directory(
config, extension='deb')
self.assertTrue(
indexer.are_you_interested_in_file(filepath,
'linux-ubuntu-xenial',
'x86'))
self.assertFalse(
indexer.are_you_interested_in_file('foo.rpm',
'linux-ubuntu-xenial',
'x86'))
indexer.consume_file(filepath, 'linux-ubuntu-xenial', 'i386')
packages_file = path.join(indexer.base_directory, 'linux-ubuntu',
'dists', 'xenial', 'main', 'binary-i386',
'Packages')
with fopen(packages_file) as fd:
packages_contents = fd.read()
self.assertNotEqual(packages_contents, '')
self.assertIn(
"Filename: dists/xenial/main/binary-i386/some-package.deb",
packages_contents)
with gzip.open(packages_file + '.gz', 'rb') as fd:
self.assertEqual(packages_contents, fd.read().decode())
release_dirpath = path.join(indexer.base_directory, 'linux-ubuntu',
'dists', 'xenial')
self.assertTrue(
path.exists(
path.join(release_dirpath, 'main', 'binary-i386',
'some-package.deb')))
release_filepath = path.join(release_dirpath, 'Release')
# with fopen(release_filepath) as fd:
# self.assertEquals(fd.read(), 'Codename: xenial\nArchitectures: amd64 i386\nComponents: main\nok')
self.assertTrue(path.exists(release_filepath))
def restart_after():
require_restart = [
service for service in SERVICES
if path.exists(path.join(SYSTEMD, '%s.service' % service))
]
for service in require_restart:
log_execute_assert_success(
['systemctl', 'stop', '%s.service' % service], allow_to_fail=True)
yield
for service in require_restart:
log_execute_assert_success(
['systemctl', 'start',
'%s.service' % service])
def write_new_package_in_incoming_directory(
self,
config,
index='main-stable',
package_basename='some-package',
extension=None):
filepath = path.join(
config.incoming_directory, index,
('%s.%s' %
(package_basename, extension)) if extension else package_basename)
with fopen(filepath, 'w') as fd:
pass
logger.debug("write_new_package_in_incoming_directory %s" % filepath)
return filepath
def test_python_consume_file(self):
from infi.app_repo.indexers.python import PythonIndexer
with self._setup_context() as config:
indexer = PythonIndexer(config, 'main-stable')
indexer.initialise()
filepath = self.write_new_package_in_incoming_directory(
config,
package_basename='python-v2.7.8.13-linux-oracle-7-x64',
extension='tar.gz')
self.assertTrue(
indexer.are_you_interested_in_file(filepath, 'linux-oracle-7',
'x64'))
indexer.consume_file(filepath, 'linux-oracle-7', 'x64')
print(filepath)
self.assertTrue(
path.exists(
path.join(indexer.base_directory,
'python-v2.7.8.13-linux-oracle-7-x64.tar.gz')))
def assert_package_exists(self, config, package_name):
assert path.exists(
path.join(config.packages_directory, 'main-stable', 'index',
'packages', package_name))
def write_new_package_in_incoming_directory(self, config, index='main-stable', package_basename='some-package', extension=None):
filepath = path.join(config.incoming_directory, index, ('%s.%s' % (package_basename, extension)) if extension else package_basename)
with fopen(filepath, 'w') as fd:
pass
logger.debug("write_new_package_in_incoming_directory %s" % filepath)
return filepath
def setUp(self):
from gevent.event import Event
super(FtpWithRpcTestCase, self).setUp()
self.config = Configuration.from_disk(None)
ensure_directory_exists(path.join(self.config.incoming_directory, 'main-stable'))
self.test_succeded = Event()
from infi.app_repo.utils import path, log_execute_assert_success, fopen
from infi.pyutils.contexts import contextmanager
INIT = path.join(path.sep, 'etc', 'init')
SERVICES = {"app-repo-ftp": 'ftp-server --signal-upstart --process-incoming-on-startup',
"app-repo-web": 'web-server --signal-upstart',
"app-repo-rpc": 'rpc-server --signal-upstart'}
TEMPLATE = """
author "INFINIDAT, Ltd."
description "INFINIDAT Host Powertools for VMware"
version {version}
chdir {chdir}
exec {exec}
expect stop
respawn
respawn limit 30 2
start on (local-filesystems and net-device-up IFACE!=lo)
stop on runlevel [016]
"""
@contextmanager
def restart_after():
require_restart = [service for service in SERVICES if
path.exists(path.join(INIT, '%s.conf' % service))]
for service in require_restart:
log_execute_assert_success(['stop', service], allow_to_fail=True)
yield
for service in require_restart:
log_execute_assert_success(['start', service])
def gpg_key():
from infi.gevent_utils.os import fopen
with fopen(path.join(flask.current_app.app_repo_config.packages_directory, 'gpg.key')) as fd:
return flask.Response(fd.read(), content_type='application/octet-stream')
def _deb():
deb = flask.Blueprint("deb", __name__)
AutoIndex(deb,
browse_root=path.join(
self.app_repo_config.artifacts_directory, 'deb'))
self.register_blueprint(deb, url_prefix="/deb")
def assert_package_exists(self, config, package_name):
assert path.exists(path.join(config.packages_directory, 'main-stable', 'index', 'packages', package_name))
def setUp(self):
from gevent.event import Event
super(FtpWithRpcTestCase, self).setUp()
self.config = self._get_config_for_test()
ensure_directory_exists(path.join(self.config.incoming_directory, 'main-stable'))
self.test_succeded = Event()
from infi.app_repo.utils import path, log_execute_assert_success, fopen
from infi.pyutils.contexts import contextmanager
SYSTEMD = path.join(path.sep, 'etc', 'systemd', 'system')
SERVICES = {
"app-repo-ftp": 'ftp-server --process-incoming-on-startup',
"app-repo-web": 'web-server',
"app-repo-rpc": 'rpc-server'
}
TEMPLATE = """
[Unit]
Description=INFINIDAT app-repo server
Type=notify
[Service]
WorkingDirectory={chdir}
ExecStart={exec}
Restart=on-failure
RestartSec=2
[Install]
WantedBy=multi-user.target
"""
@contextmanager
def restart_after():
require_restart = [
service for service in SERVICES
if path.exists(path.join(SYSTEMD, '%s.service' % service))
def _ova_updates():
ova = flask.Blueprint("ova", __name__)
AutoIndex(ova, browse_root=path.join(self.app_repo_config.artifacts_directory, 'ova', 'updates'))
self.register_blueprint(ova, url_prefix="/ova")
def _python():
rpm = flask.Blueprint("python", __name__)
AutoIndex(rpm, browse_root=path.join(self.app_repo_config.artifacts_directory, 'python'))
self.register_blueprint(rpm, url_prefix="/python")
def _python():
rpm = flask.Blueprint("python", __name__)
AutoIndex(rpm,
browse_root=path.join(
self.app_repo_config.artifacts_directory, 'python'))
self.register_blueprint(rpm, url_prefix="/python")
def _deb():
deb = flask.Blueprint("deb", __name__)
AutoIndex(deb, browse_root=path.join(self.app_repo_config.artifacts_directory, 'deb'))
self.register_blueprint(deb, url_prefix="/deb")
from infi.app_repo.utils import path, log_execute_assert_success, fopen
from infi.pyutils.contexts import contextmanager
INIT = path.join(path.sep, 'etc', 'init')
SERVICES = {
"app-repo-ftp":
'ftp-server --signal-upstart --process-incoming-on-startup',
"app-repo-web": 'web-server --signal-upstart',
"app-repo-rpc": 'rpc-server --signal-upstart'
}
TEMPLATE = """
author "INFINIDAT, Ltd."
description "INFINIDAT Host Powertools for VMware"
version {version}
chdir {chdir}
exec {exec}
expect stop
respawn
respawn limit 30 2
start on (local-filesystems and net-device-up IFACE!=lo)
stop on runlevel [016]
"""
@contextmanager
def restart_after():
require_restart = [
service for service in SERVICES
if path.exists(path.join(INIT, '%s.conf' % service))
]
def _archives():
rpm = flask.Blueprint("archives", __name__)
AutoIndex(rpm, browse_root=path.join(self.app_repo_config.artifacts_directory, 'archives'))
self.register_blueprint(rpm, url_prefix="/archives")
