def create(self, request, exporter_pk):
"""
Generates a Task to export the set of repositories assigned to a specific PulpExporter.
"""
# Validate Exporter
exporter = PulpExporter.objects.get(pk=exporter_pk).cast()
ExporterSerializer.validate_path(exporter.path, check_is_dir=True)
# Validate Export
serializer = PulpExportSerializer(data=request.data,
context={"exporter": exporter})
serializer.is_valid(raise_exception=True)
# Invoke the export
export = PulpExport.objects.create(exporter=exporter,
params=request.data)
export.validated_versions = serializer.validated_data.get(
"versions", None)
export.validated_start_versions = serializer.validated_data.get(
"start_versions", None)
export.validated_chunk_size = serializer.validated_data.get(
"chunk_size", None)
task = dispatch(pulp_export, [exporter], kwargs={"the_export": export})
return OperationPostponedResponse(task, request)
def test_read_only_params(self):
data = {
"full": True,
"dry_run": False,
"output_file_info": {
"bar": "blech"
}
}
serializer = PulpExportSerializer(data=data)
self.assertTrue(serializer.is_valid())
with self.assertRaises(AttributeError):
serializer.output_file_info["bar"]
def create(self, request, exporter_pk):
"""
Generates a Task to export the set of repositories assigned to a specific PulpExporter.
"""
# Validate Exporter
exporter = PulpExporter.objects.get(pk=exporter_pk).cast()
ExporterSerializer.validate_path(exporter.path, check_is_dir=True)
# Validate Export
serializer = PulpExportSerializer(data=request.data,
context={"exporter": exporter})
serializer.is_valid(raise_exception=True)
# Invoke the export
task = dispatch(
pulp_export,
exclusive_resources=[exporter],
kwargs={
"exporter_pk": str(exporter.pk),
"params": request.data
},
)
return OperationPostponedResponse(task, request)
def test_validate_no_params(self):
data = {}
serializer = PulpExportSerializer(data=data)
self.assertTrue(serializer.is_valid())
def test_chunk_size(self):
# positive tests
# bytes
data = {"chunk_size": "100B"}
serializer = PulpExportSerializer(data=data)
self.assertTrue(serializer.is_valid())
self.assertEqual(100, serializer.validated_data["chunk_size"])
# kilobytes
data = {"chunk_size": "100KB"}
serializer = PulpExportSerializer(data=data)
self.assertTrue(serializer.is_valid())
self.assertEqual(100 * 1024, serializer.validated_data["chunk_size"])
# megabytes
data = {"chunk_size": "100MB"}
serializer = PulpExportSerializer(data=data)
self.assertTrue(serializer.is_valid())
self.assertEqual(100 * 1024 * 1024,
serializer.validated_data["chunk_size"])
# gigabytes
data = {"chunk_size": "100GB"}
serializer = PulpExportSerializer(data=data)
self.assertTrue(serializer.is_valid())
self.assertEqual(100 * 1024 * 1024 * 1024,
serializer.validated_data["chunk_size"])
# terabytes
data = {"chunk_size": "100TB"}
serializer = PulpExportSerializer(data=data)
self.assertTrue(serializer.is_valid())
self.assertEqual(100 * 1024 * 1024 * 1024 * 1024,
serializer.validated_data["chunk_size"])
# float-units
data = {"chunk_size": "2.4GB"}
serializer = PulpExportSerializer(data=data)
self.assertTrue(serializer.is_valid())
self.assertEqual(int(2.4 * 1024 * 1024 * 1024),
serializer.validated_data["chunk_size"])
# negative tests
# no units
data = {"chunk_size": "100"}
serializer = PulpExportSerializer(data=data)
self.assertFalse(serializer.is_valid())
# not-a-number
data = {"chunk_size": "bazMB"}
serializer = PulpExportSerializer(data=data)
serializer.is_valid()
# non-positive
data = {"chunk_size": "0GB"}
serializer = PulpExportSerializer(data=data)
self.assertFalse(serializer.is_valid())
# non-positive
data = {"chunk_size": "-10KB"}
serializer = PulpExportSerializer(data=data)
self.assertFalse(serializer.is_valid())
def test_bad_params(self):
data = {"baz": "bar"}
serializer = PulpExportSerializer(data=data)
self.assertFalse(serializer.is_valid())
def test_validate_bad_param_values(self):
data = {"full": "bar", "dry_run": 0}
serializer = PulpExportSerializer(data=data)
self.assertFalse(serializer.is_valid())
def pulp_export(exporter_pk, params):
"""
Create a PulpExport to export pulp_exporter.repositories.
1) Spit out all Artifacts, ArtifactResource.json, and RepositoryResource.json
2) Spit out all *resource JSONs in per-repo-version directories
3) Compute and store the sha256 and filename of the resulting tar.gz/chunks
Args:
exporter_pk (str): PulpExporter
params (dict): request data
Raises:
ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting,
OR path exists and is not a directory
"""
pulp_exporter = PulpExporter.objects.get(pk=exporter_pk)
serializer = PulpExportSerializer(data=params,
context={"exporter": pulp_exporter})
serializer.is_valid(raise_exception=True)
the_export = PulpExport.objects.create(exporter=pulp_exporter,
params=params)
the_export.validated_versions = serializer.validated_data.get(
"versions", None)
the_export.validated_start_versions = serializer.validated_data.get(
"start_versions", None)
the_export.validated_chunk_size = serializer.validated_data.get(
"chunk_size", None)
try:
the_export.task = Task.current()
tarfile_fp = the_export.export_tarfile_path()
path = Path(pulp_exporter.path)
if not path.is_dir():
path.mkdir(mode=0o775, parents=True)
rslts = {}
if the_export.validated_chunk_size:
# write it into chunks
with subprocess.Popen(
[
"split",
"-a",
"4",
"-b",
str(the_export.validated_chunk_size),
"-d",
"-",
tarfile_fp + ".",
],
stdin=subprocess.PIPE,
) as split_process:
try:
with tarfile.open(tarfile_fp,
"w|gz",
fileobj=split_process.stdin) as tar:
_do_export(pulp_exporter, tar, the_export)
except Exception:
# no matter what went wrong, we can't trust the files we (may have) created.
# Delete the ones we can find and pass the problem up.
for pathname in glob(tarfile_fp + ".*"):
os.remove(pathname)
raise
# compute the hashes
global_hash = hashlib.sha256()
paths = sorted([str(Path(p)) for p in glob(tarfile_fp + ".*")])
for a_file in paths:
a_hash = _compute_hash(a_file, global_hash)
rslts[a_file] = a_hash
tarfile_hash = global_hash.hexdigest()
else:
# write into the file
try:
with tarfile.open(tarfile_fp, "w:gz") as tar:
_do_export(pulp_exporter, tar, the_export)
except Exception:
# no matter what went wrong, we can't trust the file we created.
# Delete it if it exists and pass the problem up.
if os.path.exists(tarfile_fp):
os.remove(tarfile_fp)
raise
# compute the hash
tarfile_hash = _compute_hash(tarfile_fp)
rslts[tarfile_fp] = tarfile_hash
# store the outputfile/hash info
the_export.output_file_info = rslts
# write outputfile/hash info to a file 'next to' the output file(s)
output_file_info_path = tarfile_fp.replace(".tar.gz", "-toc.json")
with open(output_file_info_path, "w") as outfile:
if the_export.validated_chunk_size:
chunk_size = the_export.validated_chunk_size
else:
chunk_size = 0
chunk_toc = {
"meta": {
"chunk_size": chunk_size,
"file": os.path.basename(tarfile_fp),
"global_hash": tarfile_hash,
},
"files": {},
}
# Build a toc with just filenames (not the path on the exporter-machine)
for a_path in rslts.keys():
chunk_toc["files"][os.path.basename(a_path)] = rslts[a_path]
json.dump(chunk_toc, outfile)
# store toc info
toc_hash = _compute_hash(output_file_info_path)
the_export.output_file_info[output_file_info_path] = toc_hash
the_export.toc_info = {
"file": output_file_info_path,
"sha256": toc_hash
}
finally:
# whatever may have happened, make sure we save the export
the_export.save()
# If an exception was thrown, we'll never get here - which is good, because we don't want a
# 'failed' export to be the last_export we derive the next incremental from
# mark it as 'last'
pulp_exporter.last_export = the_export
# save the exporter
pulp_exporter.save()