def test_validate_against_xml_file_with_multiple_files(self): content2 = b'test file 2' md5 = hashlib.md5(content2) checksum2 = md5.hexdigest() test_file2 = tempfile.NamedTemporaryFile(dir=self.datadir, delete=False) test_file2.write(content2) test_file2.seek(0) test_file2.close() xml_str = ''' <root> <file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}"> <FLocat href="{file}"/> </file> <file CHECKSUM="{hash2}" CHECKSUMTYPE="{alg}"> <FLocat href="{file2}"/> </file> </root>'''.format( hash=self.checksum, alg='md5', file=self.test_file.name, hash2=checksum2, file2=test_file2.name) xml_str = six.binary_type(xml_str.encode('utf-8')) self.xml_file.write(xml_str) self.xml_file.seek(0) self.xml_file.close() options = {'expected': self.xml_file.name, 'algorithm': 'md5'} self.validator = ChecksumValidator(context='xml_file', options=options) self.validator.validate(self.test_file.name) self.validator.validate(test_file2.name)
def test_validate_against_string(self): options = {'expected': self.checksum} self.validator = ChecksumValidator(context='checksum_str', options=options) self.validator.validate(self.test_file) self.validator.options = {'expected': 'incorrect'} with self.assertRaises(ValidationError): self.validator.validate(self.test_file)
def test_validate_against_xml_file_valid(self): xml_str = '<root><file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}"><FLocat href="{file}"/></file></root>'.format( hash=self.checksum, alg='md5', file=self.test_file.name) xml_str = six.binary_type(xml_str.encode('utf-8')) self.xml_file.write(xml_str) self.xml_file.seek(0) self.xml_file.close() options = {'expected': self.xml_file.name, 'algorithm': 'md5'} self.validator = ChecksumValidator(context='xml_file', options=options) self.validator.validate(self.test_file.name)
def test_validate_against_checksum_file(self): checksum_file = '%s.md5' % self.test_file options = {'expected': checksum_file} with open(checksum_file, 'w') as f: f.write(self.checksum) self.validator = ChecksumValidator(context='checksum_file', options=options) self.validator.validate(self.test_file) with open(checksum_file, 'a') as f: f.write('appended') with self.assertRaises(ValidationError): self.validator.validate(self.test_file)
class ChecksumValidatorTests(TestCase, fake_filesystem_unittest.TestCase): def setUp(self): self.setUpPyfakefs() self.test_file = 'foo.txt' self.content = 'test file' with open(self.test_file, 'w') as f: f.write(self.content) md5 = hashlib.md5(self.content.encode('utf-8')) self.checksum = md5.hexdigest() def test_validate_against_string(self): options = {'expected': self.checksum} self.validator = ChecksumValidator(context='checksum_str', options=options) self.validator.validate(self.test_file) self.validator.options = {'expected': 'incorrect'} with self.assertRaises(ValidationError): self.validator.validate(self.test_file) def test_validate_against_checksum_file(self): checksum_file = '%s.md5' % self.test_file options = {'expected': checksum_file} with open(checksum_file, 'w') as f: f.write(self.checksum) self.validator = ChecksumValidator(context='checksum_file', options=options) self.validator.validate(self.test_file) with open(checksum_file, 'a') as f: f.write('appended') with self.assertRaises(ValidationError): self.validator.validate(self.test_file)
def test_validate_against_xml_file_invalid(self): xml_str = '<root><file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}"><FLocat href="{file}"/></file></root>'.format( hash=self.checksum + 'appended', alg='md5', file=self.test_file.name) xml_str = bytes(xml_str.encode('utf-8')) self.xml_file.write(xml_str) self.xml_file.seek(0) self.xml_file.close() options = {'expected': self.xml_file.name, 'algorithm': 'md5'} self.validator = ChecksumValidator(context='xml_file', options=options) with self.assertRaises(ValidationError): self.validator.validate(self.test_file.name)
def add_file_complete(self, request, pk=None): entry = self.get_object() if entry.ip.responsible != self.request.user and not request.user.has_perm( 'ip.can_receive_remote_files'): raise exceptions.PermissionDenied path = entry.ip.policy.cache_storage.value md5 = request.data['md5'] filepath = request.data['path'] filepath = os.path.join(path, filepath) options = {'expected': md5, 'algorithm': 'md5'} validator = ChecksumValidator(context='checksum_str', options=options) validator.validate(filepath) return Response('Upload of %s complete' % filepath)
def verify(self): if self.content_location_type == TAPE: verifydir = Path.objects.get(entity='verify').value tmppath = os.path.join(verifydir, self.storage_medium.storage_target.target) if not os.path.exists(tmppath): try: os.mkdir(tmppath) except OSError as e: if e.errno != errno.EEXIST: raise ProcessTask.objects.create( name='ESSArch_Core.tasks.SetTapeFileNumber', params={ 'medium': self.storage_medium_id, 'num': int(self.content_location_value) }, information_package=self.ip, ).run().get() ProcessTask.objects.create( name='ESSArch_Core.tasks.ReadTape', params={ 'medium': self.storage_medium_id, 'path': tmppath, 'block_size': self.storage_medium.block_size * 512, }, information_package=self.ip, ).run().get() filename = os.path.join(tmppath, self.ip.object_identifier_value + '.tar'), algorithm = self.ip.get_message_digest_algorithm_display() options = { 'expected': self.ip.message_digest, 'algorithm': algorithm } validator = ChecksumValidator(context='checksum_str', options=options) validator.validate(filename)
def run(self, ip=None, xmlfile=None, validate_fileformat=True, validate_integrity=True, rootdir=None): if any([validate_fileformat, validate_integrity]): if rootdir is None: rootdir = InformationPackage.objects.values_list( 'object_path', flat=True).get(pk=ip) format_validator = FormatValidator() for f in find_files(xmlfile, rootdir): filename = os.path.join(rootdir, f.path) if validate_fileformat and f.format is not None: format_validator.validate(filename, (f.format, None, None)) if validate_integrity and f.checksum is not None and f.checksum_type is not None: options = { 'expected': f.checksum, 'algorithm': f.checksum_type } validator = ChecksumValidator(context='checksum_str', options=options) try: validator.validate(filename) except Exception as e: recipient = User.objects.get(pk=self.responsible).email if recipient and self.ip: ip = InformationPackage.objects.get(pk=self.ip) subject = 'Rejected "%s"' % ip.object_identifier_value body = '"%s" was rejected:\n%s' % ( ip.object_identifier_value, str(e)) send_mail(subject, body, None, [recipient], fail_silently=False) raise
def verify(self): if self.content_location_type == TAPE: verifydir = Path.objects.get(entity='verify').value tmppath = os.path.join(verifydir, self.storage_medium.storage_target.target) if not os.path.exists(tmppath): try: os.mkdir(tmppath) except OSError as e: if e.errno != errno.EEXIST: raise drive = self.storage_medium.tape_drive if drive is None: raise ValueError("Tape not mounted") set_tape_file_number(drive.device, int(self.content_location_value)) read_tape(drive.device, path=tmppath, block_size=self.storage_medium.block_size * 512) drive.last_change = timezone.now() drive.save(update_fields=['last_change']) filename = os.path.join(tmppath, self.ip.object_identifier_value + '.tar'), algorithm = self.ip.get_message_digest_algorithm_display() options = { 'expected': self.ip.message_digest, 'algorithm': algorithm } validator = ChecksumValidator(context='checksum_str', options=options) validator.validate(filename)
class ChecksumValidatorXMLTests(TestCase): """ pyfakefs doesn't support lxml, need separate test case without pyfakefs: https://github.com/jmcgeheeiv/pyfakefs#compatibility """ def setUp(self): self.datadir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.datadir) self.content = b'test file' md5 = hashlib.md5(self.content) self.checksum = md5.hexdigest() self.test_file = tempfile.NamedTemporaryFile(dir=self.datadir, delete=False) self.test_file.write(self.content) self.test_file.seek(0) self.test_file.close() self.xml_file = tempfile.NamedTemporaryFile(dir=self.datadir, delete=False) def test_validate_against_xml_file_valid(self): xml_str = '<root><file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}"><FLocat href="{file}"/></file></root>'.format( hash=self.checksum, alg='md5', file=self.test_file.name) xml_str = six.binary_type(xml_str.encode('utf-8')) self.xml_file.write(xml_str) self.xml_file.seek(0) self.xml_file.close() options = {'expected': self.xml_file.name, 'algorithm': 'md5'} self.validator = ChecksumValidator(context='xml_file', options=options) self.validator.validate(self.test_file.name) def test_validate_against_xml_file_invalid(self): xml_str = '<root><file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}"><FLocat href="{file}"/></file></root>'.format( hash=self.checksum + 'appended', alg='md5', file=self.test_file.name) xml_str = six.binary_type(xml_str.encode('utf-8')) self.xml_file.write(xml_str) self.xml_file.seek(0) self.xml_file.close() options = {'expected': self.xml_file.name, 'algorithm': 'md5'} self.validator = ChecksumValidator(context='xml_file', options=options) with self.assertRaises(ValidationError): self.validator.validate(self.test_file.name) def test_validate_against_xml_file_with_multiple_files(self): content2 = b'test file 2' md5 = hashlib.md5(content2) checksum2 = md5.hexdigest() test_file2 = tempfile.NamedTemporaryFile(dir=self.datadir, delete=False) test_file2.write(content2) test_file2.seek(0) test_file2.close() xml_str = ''' <root> <file CHECKSUM="{hash}" CHECKSUMTYPE="{alg}"> <FLocat href="{file}"/> </file> <file CHECKSUM="{hash2}" CHECKSUMTYPE="{alg}"> <FLocat href="{file2}"/> </file> </root>'''.format( hash=self.checksum, alg='md5', file=self.test_file.name, hash2=checksum2, file2=test_file2.name) xml_str = six.binary_type(xml_str.encode('utf-8')) self.xml_file.write(xml_str) self.xml_file.seek(0) self.xml_file.close() options = {'expected': self.xml_file.name, 'algorithm': 'md5'} self.validator = ChecksumValidator(context='xml_file', options=options) self.validator.validate(self.test_file.name) self.validator.validate(test_file2.name)