def test(self):
""" Test view for resource depicts output of various integrity checking scripts """
# print("TESTING {}".format(self.short_id)) # leave this for debugging
try:
self.resource = get_resource_by_shortkey(self.short_id, or_404=False)
except BaseResource.DoesNotExist:
print("{} does not exist in Django".format(self.short_id))
return
# skip federated resources if not configured to handle these
if self.resource.is_federated and not settings.REMOTE_USE_IRODS:
msg = "check_resource: skipping check of federated resource {} in unfederated mode"\
.format(self.resource.short_id)
print(msg)
istorage = self.resource.get_irods_storage()
if not istorage.exists(self.resource.root_path):
self.label()
print(" root path {} does not exist in iRODS".format(self.resource.root_path))
print(" ... resource {} has type {} and title {}"
.format(self.resource.short_id,
self.resource.resource_type,
self.resource.title))
return
for a in ('bag_modified', 'isPublic', 'resourceType', 'quotaUserName'):
value = self.check_avu(a)
if a == 'resourceType' and value is not None and value != self.resource.resource_type:
self.label()
print((" AVU resourceType is {}, should be {}".format(value,
self.resource.resource_type)))
if a == 'isPublic' and value is not None and value != self.resource.raccess.public:
self.label()
print((" AVU isPublic is {}, but public is {}".format(value,
self.resource.raccess.public)))
irods_issues, irods_errors = check_irods_files(self.resource,
log_errors=False,
echo_errors=False,
return_errors=True)
if irods_errors:
self.label()
print(" iRODS errors:")
for e in irods_issues:
print(" {}".format(e))
if self.resource.resource_type == 'CompositeResource':
logical_issues = []
for res_file in self.resource.files.all():
file_type = get_logical_file_type(res=self.resource, user=None,
file_id=res_file.pk, fail_feedback=False)
if not res_file.has_logical_file and file_type is not None:
msg = "check_resource: file {} does not have required logical file {}"\
.format(res_file.storage_path,
file_type.__name__)
logical_issues.append(msg)
elif res_file.has_logical_file and file_type is None:
msg = "check_resource: logical file for {} has type {}, not needed"\
.format(res_file.storage_path,
type(res_file.logical_file).__name__)
logical_issues.append(msg)
elif res_file.has_logical_file and file_type is not None and \
not isinstance(res_file.logical_file, file_type):
msg = "check_resource: logical file for {} has type {}, should be {}"\
.format(res_file.storage_path,
type(res_file.logical_file).__name__,
file_type.__name__)
logical_issues.append(msg)
if logical_issues:
self.label()
print(" Logical file errors:")
for e in logical_issues:
print(" {}".format(e))
def test(self):
""" Test view for resource depicts output of various integrity checking scripts """
# print("TESTING {}".format(self.short_id)) # leave this for debugging
try:
self.resource = get_resource_by_shortkey(self.short_id, or_404=False)
except BaseResource.DoesNotExist:
print("{} does not exist in Django".format(self.short_id))
return
# skip federated resources if not configured to handle these
if self.resource.is_federated and not settings.REMOTE_USE_IRODS:
msg = "check_resource: skipping check of federated resource {} in unfederated mode"\
.format(self.resource.short_id)
print(msg)
istorage = self.resource.get_irods_storage()
if not istorage.exists(self.resource.root_path):
self.label()
print(" root path {} does not exist in iRODS".format(self.resource.root_path))
print(" ... resource {} has type {} and title {}"
.format(self.resource.short_id,
self.resource.resource_type,
self.resource.title.encode('ascii', 'replace')))
return
for a in ('bag_modified', 'isPublic', 'resourceType', 'quotaUserName'):
value = self.check_avu(a)
if a == 'resourceType' and value is not None and value != self.resource.resource_type:
self.label()
print(" AVU resourceType is {}, should be {}".format(value.encode('ascii',
'replace'),
self.resource.resource_type))
if a == 'isPublic' and value is not None and value != self.resource.raccess.public:
self.label()
print(" AVU isPublic is {}, but public is {}".format(value.encode('ascii',
'replace'),
self.resource.raccess.public))
irods_issues, irods_errors = check_irods_files(self.resource,
log_errors=False,
echo_errors=False,
return_errors=True)
if irods_errors:
self.label()
print(" iRODS errors:")
for e in irods_issues:
print(" {}".format(e))
if self.resource.resource_type == 'CompositeResource':
logical_issues = []
for res_file in self.resource.files.all():
file_type = get_logical_file_type(res=self.resource, user=None,
file_id=res_file.pk, fail_feedback=False)
if not res_file.has_logical_file and file_type is not None:
msg = "check_resource: file {} does not have required logical file {}"\
.format(res_file.storage_path.encode('ascii', 'replace'),
file_type.__name__)
logical_issues.append(msg)
elif res_file.has_logical_file and file_type is None:
msg = "check_resource: logical file for {} has type {}, not needed"\
.format(res_file.storage_path.encode('ascii', 'replace'),
type(res_file.logical_file).__name__)
logical_issues.append(msg)
elif res_file.has_logical_file and file_type is not None and \
not isinstance(res_file.logical_file, file_type):
msg = "check_resource: logical file for {} has type {}, should be {}"\
.format(res_file.storage_path.encode('ascii', 'replace'),
type(res_file.logical_file).__name__,
file_type.__name__)
logical_issues.append(msg)
if logical_issues:
self.label()
print(" Logical file errors:")
for e in logical_issues:
print(" {}".format(e))
def __ingest_irods_directory(resource,
dir,
logger,
stop_on_error=False,
log_errors=True,
echo_errors=False,
return_errors=False):
"""
list a directory and ingest files there for conformance with django ResourceFiles
:param stop_on_error: whether to raise a ValidationError exception on first error
:param log_errors: whether to log errors to Django log
:param echo_errors: whether to print errors on stdout
:param return_errors: whether to collect errors in an array and return them.
"""
errors = []
ecount = 0
istorage = resource.get_irods_storage()
try:
listing = istorage.listdir(dir)
for fname in listing[1]: # files
# do not use os.path.join because fname might contain unicode characters
fullpath = dir + '/' + fname
found = False
for res_file in resource.files.all():
if res_file.storage_path == fullpath:
found = True
if not found and not resource.is_aggregation_xml_file(fullpath):
ecount += 1
msg = "ingest_irods_files: file {} in iRODs does not exist in Django (INGESTING)"\
.format(fullpath)
if echo_errors:
print(msg)
if log_errors:
logger.error(msg)
if return_errors:
errors.append(msg)
if stop_on_error:
raise ValidationError(msg)
# TODO: does not ingest logical file structure for composite resources
link_irods_file_to_django(resource, fullpath)
# Create required logical files as necessary
if resource.resource_type == "CompositeResource":
file_type = get_logical_file_type(res=resource, user=None,
file_id=res_file.pk, fail_feedback=False)
if not res_file.has_logical_file and file_type is not None:
msg = "ingest_irods_files: setting required logical file for {}"\
.format(fullpath)
if echo_errors:
print(msg)
if log_errors:
logger.error(msg)
if return_errors:
errors.append(msg)
if stop_on_error:
raise ValidationError(msg)
set_logical_file_type(res=resource, user=None, file_id=res_file.pk,
fail_feedback=False)
elif res_file.has_logical_file and file_type is not None and \
not isinstance(res_file.logical_file, file_type):
msg = "ingest_irods_files: logical file for {} has type {}, should be {}"\
.format(res_file.storage_path,
type(res_file.logical_file).__name__,
file_type.__name__)
if echo_errors:
print(msg)
if log_errors:
logger.error(msg)
if return_errors:
errors.append(msg)
if stop_on_error:
raise ValidationError(msg)
elif res_file.has_logical_file and file_type is None:
msg = "ingest_irods_files: logical file for {} has type {}, not needed"\
.format(res_file.storage_path, type(res_file.logical_file).__name__,
file_type.__name__)
if echo_errors:
print(msg)
if log_errors:
logger.error(msg)
if return_errors:
errors.append(msg)
if stop_on_error:
raise ValidationError(msg)
for dname in listing[0]: # directories
# do not use os.path.join because fname might contain unicode characters
error2, ecount2 = __ingest_irods_directory(resource,
dir + '/' + dname,
logger,
stop_on_error=stop_on_error,
echo_errors=echo_errors,
log_errors=log_errors,
return_errors=return_errors)
errors.extend(error2)
ecount += ecount2
except SessionException as se:
print("iRODs error: {}".format(se.stderr))
logger.error("iRODs error: {}".format(se.stderr))
return errors, ecount # empty unless return_errors=True
def __ingest_irods_directory(resource,
dir,
logger,
stop_on_error=False,
log_errors=True,
echo_errors=False,
return_errors=False):
"""
list a directory and ingest files there for conformance with django ResourceFiles
:param stop_on_error: whether to raise a ValidationError exception on first error
:param log_errors: whether to log errors to Django log
:param echo_errors: whether to print errors on stdout
:param return_errors: whether to collect errors in an array and return them.
"""
errors = []
ecount = 0
istorage = resource.get_irods_storage()
try:
listing = istorage.listdir(dir)
for fname in listing[1]: # files
# do not use os.path.join because fname might contain unicode characters
fullpath = dir + '/' + fname
found = False
for res_file in resource.files.all():
if res_file.storage_path == fullpath:
found = True
if not found and not resource.is_aggregation_xml_file(fullpath):
ecount += 1
msg = "ingest_irods_files: file {} in iRODs does not exist in Django (INGESTING)"\
.format(fullpath.encode('ascii', 'replace'))
if echo_errors:
print(msg)
if log_errors:
logger.error(msg)
if return_errors:
errors.append(msg)
if stop_on_error:
raise ValidationError(msg)
# TODO: does not ingest logical file structure for composite resources
link_irods_file_to_django(resource, fullpath)
# Create required logical files as necessary
if resource.resource_type == "CompositeResource":
file_type = get_logical_file_type(res=resource, user=None,
file_id=res_file.pk, fail_feedback=False)
if not res_file.has_logical_file and file_type is not None:
msg = "ingest_irods_files: setting required logical file for {}"\
.format(fullpath)
if echo_errors:
print(msg)
if log_errors:
logger.error(msg)
if return_errors:
errors.append(msg)
if stop_on_error:
raise ValidationError(msg)
set_logical_file_type(res=resource, user=None, file_id=res_file.pk,
fail_feedback=False)
elif res_file.has_logical_file and file_type is not None and \
not isinstance(res_file.logical_file, file_type):
msg = "ingest_irods_files: logical file for {} has type {}, should be {}"\
.format(res_file.storage_path.encode('ascii', 'replace'),
type(res_file.logical_file).__name__,
file_type.__name__)
if echo_errors:
print(msg)
if log_errors:
logger.error(msg)
if return_errors:
errors.append(msg)
if stop_on_error:
raise ValidationError(msg)
elif res_file.has_logical_file and file_type is None:
msg = "ingest_irods_files: logical file for {} has type {}, not needed"\
.format(res_file.storage_path, type(res_file.logical_file).__name__,
file_type.__name__)
if echo_errors:
print(msg)
if log_errors:
logger.error(msg)
if return_errors:
errors.append(msg)
if stop_on_error:
raise ValidationError(msg)
for dname in listing[0]: # directories
# do not use os.path.join because fname might contain unicode characters
error2, ecount2 = __ingest_irods_directory(resource,
dir + '/' + dname,
logger,
stop_on_error=stop_on_error,
echo_errors=echo_errors,
log_errors=log_errors,
return_errors=return_errors)
errors.extend(error2)
ecount += ecount2
except SessionException as se:
print("iRODs error: {}".format(se.stderr))
logger.error("iRODs error: {}".format(se.stderr))
return errors, ecount # empty unless return_errors=True
