def test_publish(self, mock_repo_ctrl):
# Setup
self.populate()
with mock_config.patch({'server': {'storage_dir': self.parentfs}}):
# Test
dist = NodesHttpDistributor()
repo = Repository(self.REPO_ID)
conduit = RepoPublishConduit(self.REPO_ID,
constants.HTTP_DISTRIBUTOR)
dist.publish_repo(repo, conduit, self.dist_conf())
# Verify
conf = DownloaderConfig()
downloader = LocalFileDownloader(conf)
pub = dist.publisher(repo, self.dist_conf())
url = pathlib.url_join(pub.base_url, pub.manifest_path())
working_dir = self.childfs
manifest = RemoteManifest(url, downloader, working_dir)
manifest.fetch()
manifest.fetch_units()
units = [u for u, r in manifest.get_units()]
self.assertEqual(len(units), self.NUM_UNITS)
for n in range(0, self.NUM_UNITS):
unit = units[n]
created = self.units[n]
for p, v in unit['unit_key'].items():
self.assertEqual(created[p], v)
for p, v in unit['metadata'].items():
if p in ('_ns', '_content_type_id'):
continue
self.assertEqual(created[p], v)
self.assertEqual(created.get('_storage_path'),
unit['storage_path'])
self.assertEqual(unit['type_id'], self.UNIT_TYPE_ID)
def test_publisher(self):
# setup
units = self.populate()
# test
# publish
repo_id = 'test_repo'
base_url = 'file://'
publish_dir = os.path.join(self.tmpdir, 'nodes/repos')
virtual_host = (publish_dir, publish_dir)
with HttpPublisher(base_url, virtual_host, repo_id) as p:
p.publish(units)
p.commit()
# verify
conf = DownloaderConfig()
downloader = LocalFileDownloader(conf)
manifest_path = p.manifest_path()
working_dir = os.path.join(self.tmpdir, 'working_dir')
os.makedirs(working_dir)
url = pathlib.url_join(base_url, manifest_path)
manifest = RemoteManifest(url, downloader, working_dir)
manifest.fetch()
manifest.fetch_units()
self.assertTrue(manifest.has_valid_units())
units = manifest.get_units()
n = 0
for unit, ref in units:
self.assertEqual(manifest.publishing_details[constants.BASE_URL],
pathlib.url_join(base_url, publish_dir, repo_id))
if n == 0: # TARBALL
path = pathlib.join(publish_dir, repo_id,
unit[constants.TARBALL_PATH])
self.assertTrue(os.path.isfile(path))
else:
path = pathlib.join(publish_dir, repo_id,
unit[constants.RELATIVE_PATH])
self.assertTrue(os.path.islink(path))
self.assertEqual(unit[constants.FILE_SIZE],
os.path.getsize(path))
if n == 0: # TARBALL
path = pathlib.join(publish_dir, repo_id,
unit[constants.TARBALL_PATH])
tb = tarfile.open(path)
try:
files = sorted(tb.getnames())
finally:
tb.close()
self.assertEqual(len(files), self.NUM_TARED_FILES)
else:
path = pathlib.join(publish_dir, repo_id,
unit[constants.RELATIVE_PATH])
with open(path, 'rb') as fp:
unit_content = fp.read()
self.assertEqual(unit_content, unit_content)
self.assertEqual(unit['unit_key']['n'], n)
n += 1
def test_publish(self, mock_repo_ctrl):
# Setup
self.populate()
with mock_config.patch({'server': {'storage_dir': self.parentfs}}):
# Test
dist = NodesHttpDistributor()
repo = Repository(self.REPO_ID)
conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR)
dist.publish_repo(repo, conduit, self.dist_conf())
# Verify
conf = DownloaderConfig()
downloader = LocalFileDownloader(conf)
pub = dist.publisher(repo, self.dist_conf())
url = pathlib.url_join(pub.base_url, pub.manifest_path())
working_dir = self.childfs
manifest = RemoteManifest(url, downloader, working_dir)
manifest.fetch()
manifest.fetch_units()
units = [u for u, r in manifest.get_units()]
self.assertEqual(len(units), self.NUM_UNITS)
for n in range(0, self.NUM_UNITS):
unit = units[n]
created = self.units[n]
for p, v in unit['unit_key'].items():
self.assertEqual(created[p], v)
for p, v in unit['metadata'].items():
if p in ('_ns', '_content_type_id'):
continue
self.assertEqual(created[p], v)
self.assertEqual(created.get('_storage_path'), unit['storage_path'])
self.assertEqual(unit['type_id'], self.UNIT_TYPE_ID)
def _unit_inventory(self, request):
"""
Build the unit inventory.
:param request: A synchronization request.
:type request: SyncRequest
:return: The built inventory.
:rtype: UnitInventory
"""
# fetch child units
try:
conduit = NodesConduit()
child_units = conduit.get_units(request.repo_id)
except NodeError:
raise
except Exception:
log.exception(request.repo_id)
raise GetChildUnitsError(request.repo_id)
# fetch parent units
try:
request.progress.begin_manifest_download()
url = request.config.get(constants.MANIFEST_URL_KEYWORD)
manifest = Manifest(request.working_dir)
try:
manifest.read(migrate)
except IOError, e:
if e.errno == errno.ENOENT:
pass
except ValueError:
# json decoding failed
pass
fetched_manifest = RemoteManifest(url, request.downloader, request.working_dir)
fetched_manifest.fetch(migrate)
if manifest != fetched_manifest or \
not manifest.is_valid() or not manifest.has_valid_units():
fetched_manifest.fetch_units()
manifest = fetched_manifest
if not manifest.is_valid():
raise InvalidManifestError()
def test_publisher(self):
# setup
units = self.populate()
# test
# publish
repo_id = 'test_repo'
base_url = 'file://'
publish_dir = os.path.join(self.tmpdir, 'nodes/repos')
repo_publish_dir = os.path.join(publish_dir, repo_id)
virtual_host = (publish_dir, publish_dir)
with HttpPublisher(base_url, virtual_host, repo_id, repo_publish_dir) as p:
p.publish(units)
p.commit()
# verify
conf = DownloaderConfig()
downloader = LocalFileDownloader(conf)
manifest_path = p.manifest_path()
working_dir = os.path.join(self.tmpdir, 'working_dir')
os.makedirs(working_dir)
url = pathlib.url_join(base_url, manifest_path)
manifest = RemoteManifest(url, downloader, working_dir)
manifest.fetch()
manifest.fetch_units()
self.assertTrue(manifest.has_valid_units())
units = manifest.get_units()
n = 0
for unit, ref in units:
self.assertEqual(
manifest.publishing_details[constants.BASE_URL],
pathlib.url_join(base_url, publish_dir, repo_id))
if n == 0: # TARBALL
path = pathlib.join(publish_dir, repo_id, unit[constants.TARBALL_PATH])
self.assertTrue(os.path.isfile(path))
else:
path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH])
self.assertTrue(os.path.islink(path))
self.assertEqual(unit[constants.FILE_SIZE], os.path.getsize(path))
if n == 0: # TARBALL
path = pathlib.join(publish_dir, repo_id, unit[constants.TARBALL_PATH])
tb = tarfile.open(path)
try:
files = sorted(tb.getnames())
finally:
tb.close()
self.assertEqual(len(files), self.NUM_TARED_FILES)
else:
path = pathlib.join(publish_dir, repo_id, unit[constants.RELATIVE_PATH])
with open(path, 'rb') as fp:
unit_content = fp.read()
self.assertEqual(unit_content, unit_content)
self.assertEqual(unit['unit_key']['n'], n)
n += 1
def _unit_inventory(self, request):
"""
Build the unit inventory.
:param request: A synchronization request.
:type request: SyncRequest
:return: The built inventory.
:rtype: UnitInventory
"""
# fetch child units
try:
conduit = NodesConduit()
child_units = conduit.get_units(request.repo_id)
except NodeError:
raise
except Exception:
_log.exception(request.repo_id)
raise GetChildUnitsError(request.repo_id)
# fetch parent units
try:
request.progress.begin_manifest_download()
url = request.config.get(constants.MANIFEST_URL_KEYWORD)
manifest = Manifest(request.working_dir)
try:
manifest.read()
except IOError, e:
if e.errno == errno.ENOENT:
pass
except ValueError:
# json decoding failed
pass
fetched_manifest = RemoteManifest(url, request.downloader,
request.working_dir)
fetched_manifest.fetch()
if manifest != fetched_manifest or \
not manifest.is_valid() or not manifest.has_valid_units():
fetched_manifest.write()
fetched_manifest.fetch_units()
manifest = fetched_manifest
if not manifest.is_valid():
raise InvalidManifestError()