def test_validate_against_xml_file_with_multiple_files(self):
content2 = b'test file 2'
md5 = hashlib.md5(content2)
checksum2 = md5.hexdigest()
test_file2 = tempfile.NamedTemporaryFile(dir=self.datadir, delete=False)
test_file2.write(content2)
test_file2.seek(0)
test_file2.close()
xml_str = '''
<root>
<file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}">
<FLocat href="{file}"/>
</file>
<file CHECKSUM="{hash2}" CHECKSUMTYPE="{alg}">
<FLocat href="{file2}"/>
</file>
</root>'''.format(
hash=self.checksum, alg='md5', file=self.test_file.name,
hash2=checksum2, file2=test_file2.name)
xml_str = six.binary_type(xml_str.encode('utf-8'))
self.xml_file.write(xml_str)
self.xml_file.seek(0)
self.xml_file.close()
options = {'expected': self.xml_file.name, 'algorithm': 'md5'}
self.validator = ChecksumValidator(context='xml_file', options=options)
self.validator.validate(self.test_file.name)
self.validator.validate(test_file2.name)
def test_validate_against_string(self):
options = {'expected': self.checksum}
self.validator = ChecksumValidator(context='checksum_str', options=options)
self.validator.validate(self.test_file)
self.validator.options = {'expected': 'incorrect'}
with self.assertRaises(ValidationError):
self.validator.validate(self.test_file)
def test_validate_against_xml_file_valid(self):
xml_str = '<root><file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}"><FLocat href="{file}"/></file></root>'.format(
hash=self.checksum, alg='md5', file=self.test_file.name)
xml_str = six.binary_type(xml_str.encode('utf-8'))
self.xml_file.write(xml_str)
self.xml_file.seek(0)
self.xml_file.close()
options = {'expected': self.xml_file.name, 'algorithm': 'md5'}
self.validator = ChecksumValidator(context='xml_file', options=options)
self.validator.validate(self.test_file.name)
def test_validate_against_checksum_file(self):
checksum_file = '%s.md5' % self.test_file
options = {'expected': checksum_file}
with open(checksum_file, 'w') as f:
f.write(self.checksum)
self.validator = ChecksumValidator(context='checksum_file', options=options)
self.validator.validate(self.test_file)
with open(checksum_file, 'a') as f:
f.write('appended')
with self.assertRaises(ValidationError):
self.validator.validate(self.test_file)
class ChecksumValidatorTests(TestCase, fake_filesystem_unittest.TestCase):
def setUp(self):
self.setUpPyfakefs()
self.test_file = 'foo.txt'
self.content = 'test file'
with open(self.test_file, 'w') as f:
f.write(self.content)
md5 = hashlib.md5(self.content.encode('utf-8'))
self.checksum = md5.hexdigest()
def test_validate_against_string(self):
options = {'expected': self.checksum}
self.validator = ChecksumValidator(context='checksum_str', options=options)
self.validator.validate(self.test_file)
self.validator.options = {'expected': 'incorrect'}
with self.assertRaises(ValidationError):
self.validator.validate(self.test_file)
def test_validate_against_checksum_file(self):
checksum_file = '%s.md5' % self.test_file
options = {'expected': checksum_file}
with open(checksum_file, 'w') as f:
f.write(self.checksum)
self.validator = ChecksumValidator(context='checksum_file', options=options)
self.validator.validate(self.test_file)
with open(checksum_file, 'a') as f:
f.write('appended')
with self.assertRaises(ValidationError):
self.validator.validate(self.test_file)
def test_validate_against_xml_file_invalid(self):
xml_str = '<root><file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}"><FLocat href="{file}"/></file></root>'.format(
hash=self.checksum + 'appended',
alg='md5',
file=self.test_file.name)
xml_str = bytes(xml_str.encode('utf-8'))
self.xml_file.write(xml_str)
self.xml_file.seek(0)
self.xml_file.close()
options = {'expected': self.xml_file.name, 'algorithm': 'md5'}
self.validator = ChecksumValidator(context='xml_file', options=options)
with self.assertRaises(ValidationError):
self.validator.validate(self.test_file.name)
def add_file_complete(self, request, pk=None):
entry = self.get_object()
if entry.ip.responsible != self.request.user and not request.user.has_perm(
'ip.can_receive_remote_files'):
raise exceptions.PermissionDenied
path = entry.ip.policy.cache_storage.value
md5 = request.data['md5']
filepath = request.data['path']
filepath = os.path.join(path, filepath)
options = {'expected': md5, 'algorithm': 'md5'}
validator = ChecksumValidator(context='checksum_str', options=options)
validator.validate(filepath)
return Response('Upload of %s complete' % filepath)
def verify(self):
if self.content_location_type == TAPE:
verifydir = Path.objects.get(entity='verify').value
tmppath = os.path.join(verifydir,
self.storage_medium.storage_target.target)
if not os.path.exists(tmppath):
try:
os.mkdir(tmppath)
except OSError as e:
if e.errno != errno.EEXIST:
raise
ProcessTask.objects.create(
name='ESSArch_Core.tasks.SetTapeFileNumber',
params={
'medium': self.storage_medium_id,
'num': int(self.content_location_value)
},
information_package=self.ip,
).run().get()
ProcessTask.objects.create(
name='ESSArch_Core.tasks.ReadTape',
params={
'medium': self.storage_medium_id,
'path': tmppath,
'block_size': self.storage_medium.block_size * 512,
},
information_package=self.ip,
).run().get()
filename = os.path.join(tmppath,
self.ip.object_identifier_value + '.tar'),
algorithm = self.ip.get_message_digest_algorithm_display()
options = {
'expected': self.ip.message_digest,
'algorithm': algorithm
}
validator = ChecksumValidator(context='checksum_str',
options=options)
validator.validate(filename)
def run(self,
ip=None,
xmlfile=None,
validate_fileformat=True,
validate_integrity=True,
rootdir=None):
if any([validate_fileformat, validate_integrity]):
if rootdir is None:
rootdir = InformationPackage.objects.values_list(
'object_path', flat=True).get(pk=ip)
format_validator = FormatValidator()
for f in find_files(xmlfile, rootdir):
filename = os.path.join(rootdir, f.path)
if validate_fileformat and f.format is not None:
format_validator.validate(filename, (f.format, None, None))
if validate_integrity and f.checksum is not None and f.checksum_type is not None:
options = {
'expected': f.checksum,
'algorithm': f.checksum_type
}
validator = ChecksumValidator(context='checksum_str',
options=options)
try:
validator.validate(filename)
except Exception as e:
recipient = User.objects.get(pk=self.responsible).email
if recipient and self.ip:
ip = InformationPackage.objects.get(pk=self.ip)
subject = 'Rejected "%s"' % ip.object_identifier_value
body = '"%s" was rejected:\n%s' % (
ip.object_identifier_value, str(e))
send_mail(subject,
body,
None, [recipient],
fail_silently=False)
raise
def verify(self):
if self.content_location_type == TAPE:
verifydir = Path.objects.get(entity='verify').value
tmppath = os.path.join(verifydir,
self.storage_medium.storage_target.target)
if not os.path.exists(tmppath):
try:
os.mkdir(tmppath)
except OSError as e:
if e.errno != errno.EEXIST:
raise
drive = self.storage_medium.tape_drive
if drive is None:
raise ValueError("Tape not mounted")
set_tape_file_number(drive.device,
int(self.content_location_value))
read_tape(drive.device,
path=tmppath,
block_size=self.storage_medium.block_size * 512)
drive.last_change = timezone.now()
drive.save(update_fields=['last_change'])
filename = os.path.join(tmppath,
self.ip.object_identifier_value + '.tar'),
algorithm = self.ip.get_message_digest_algorithm_display()
options = {
'expected': self.ip.message_digest,
'algorithm': algorithm
}
validator = ChecksumValidator(context='checksum_str',
options=options)
validator.validate(filename)
class ChecksumValidatorXMLTests(TestCase):
"""
pyfakefs doesn't support lxml, need separate test case without pyfakefs:
https://github.com/jmcgeheeiv/pyfakefs#compatibility
"""
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.datadir)
self.content = b'test file'
md5 = hashlib.md5(self.content)
self.checksum = md5.hexdigest()
self.test_file = tempfile.NamedTemporaryFile(dir=self.datadir, delete=False)
self.test_file.write(self.content)
self.test_file.seek(0)
self.test_file.close()
self.xml_file = tempfile.NamedTemporaryFile(dir=self.datadir, delete=False)
def test_validate_against_xml_file_valid(self):
xml_str = '<root><file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}"><FLocat href="{file}"/></file></root>'.format(
hash=self.checksum, alg='md5', file=self.test_file.name)
xml_str = six.binary_type(xml_str.encode('utf-8'))
self.xml_file.write(xml_str)
self.xml_file.seek(0)
self.xml_file.close()
options = {'expected': self.xml_file.name, 'algorithm': 'md5'}
self.validator = ChecksumValidator(context='xml_file', options=options)
self.validator.validate(self.test_file.name)
def test_validate_against_xml_file_invalid(self):
xml_str = '<root><file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}"><FLocat href="{file}"/></file></root>'.format(
hash=self.checksum + 'appended', alg='md5', file=self.test_file.name)
xml_str = six.binary_type(xml_str.encode('utf-8'))
self.xml_file.write(xml_str)
self.xml_file.seek(0)
self.xml_file.close()
options = {'expected': self.xml_file.name, 'algorithm': 'md5'}
self.validator = ChecksumValidator(context='xml_file', options=options)
with self.assertRaises(ValidationError):
self.validator.validate(self.test_file.name)
def test_validate_against_xml_file_with_multiple_files(self):
content2 = b'test file 2'
md5 = hashlib.md5(content2)
checksum2 = md5.hexdigest()
test_file2 = tempfile.NamedTemporaryFile(dir=self.datadir, delete=False)
test_file2.write(content2)
test_file2.seek(0)
test_file2.close()
xml_str = '''
<root>
<file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}">
<FLocat href="{file}"/>
</file>
<file CHECKSUM="{hash2}" CHECKSUMTYPE="{alg}">
<FLocat href="{file2}"/>
</file>
</root>'''.format(
hash=self.checksum, alg='md5', file=self.test_file.name,
hash2=checksum2, file2=test_file2.name)
xml_str = six.binary_type(xml_str.encode('utf-8'))
self.xml_file.write(xml_str)
self.xml_file.seek(0)
self.xml_file.close()
options = {'expected': self.xml_file.name, 'algorithm': 'md5'}
self.validator = ChecksumValidator(context='xml_file', options=options)
self.validator.validate(self.test_file.name)
self.validator.validate(test_file2.name)
