def handle(self, *args, **options):
if options['unreferenced']:
print("LOOKING FOR IRODS RESOURCES NOT IN DJANGO")
check_for_dangling_irods(echo_errors=not options['log'],
log_errors=options['log'],
return_errors=False)
elif len(options['resource_ids']
) > 0: # an array of resource short_id to check.
for rid in options['resource_ids']:
try:
resource = BaseResource.objects.get(short_id=rid)
except BaseResource.DoesNotExist:
msg = "Resource with id {} not found in Django Resources".format(
rid)
print(msg)
print("LOOKING FOR FILE ERRORS FOR RESOURCE {}".format(rid))
if options['clean_irods']:
print(' (deleting unreferenced iRODs files)')
if options['clean_django']:
print(' (deleting Django file objects without files)')
if options['sync_ispublic']:
print(' (correcting isPublic in iRODs)')
check_irods_files(resource,
stop_on_error=False,
echo_errors=not options['log'],
log_errors=options['log'],
return_errors=False,
clean_irods=options['clean_irods'],
clean_django=options['clean_django'],
sync_ispublic=options['sync_ispublic'])
else: # check all resources
print("LOOKING FOR FILE ERRORS FOR ALL RESOURCES")
if options['clean_irods']:
print(' (deleting unreferenced iRODs files)')
if options['clean_django']:
print(' (deleting Django file objects without files)')
if options['sync_ispublic']:
print(' (correcting isPublic in iRODs)')
for r in BaseResource.objects.all():
check_irods_files(
r,
stop_on_error=False,
echo_errors=not options['log'], # Don't both log and echo
log_errors=options['log'],
return_errors=False,
clean_irods=options['clean_irods'],
clean_django=options['clean_django'],
sync_ispublic=options['sync_ispublic'])
def debug_resource(short_id):
""" Debug view for resource depicts output of various integrity checking scripts """
try:
res = BaseResource.objects.get(short_id=short_id)
except BaseResource.DoesNotExist:
print("{} does not exist".format(short_id))
resource = res.get_content_model()
assert resource, (res, res.content_model)
irods_issues, irods_errors = check_irods_files(resource, log_errors=False, return_errors=True)
print("resource: {}".format(short_id))
print("resource type: {}".format(resource.resource_type))
print("resource creator: {} {}".format(resource.creator.first_name, resource.creator.last_name))
print("resource irods bag modified: {}".format(str(resource.getAVU('bag_modified'))))
print("resource irods isPublic: {}".format(str(resource.getAVU('isPublic'))))
print("resource irods resourceType: {}".format(str(resource.getAVU('resourceType'))))
print("resource irods quotaUserName: {}".format(str(resource.getAVU('quotaUserName'))))
if irods_errors:
print("iRODS errors:")
for e in irods_issues:
print(" {}".format(e))
else:
print("No iRODS errors")
if resource.resource_type == 'CompositeResource':
print("Resource file logical files:")
for res_file in resource.files.all():
if res_file.has_logical_file:
print((" {} logical file {} is [{}]".format(res_file.short_path,
str(type(res_file.logical_file)),
str(res_file.logical_file.id))))
def debug_resource(short_id):
""" Debug view for resource depicts output of various integrity checking scripts """
try:
res = BaseResource.objects.get(short_id=short_id)
except BaseResource.DoesNotExist:
print("{} does not exist".format(short_id))
resource = res.get_content_model()
assert resource, (res, res.content_model)
irods_issues, irods_errors = check_irods_files(resource, log_errors=False, return_errors=True)
print("resource: {}".format(short_id))
print("resource type: {}".format(resource.resource_type))
print("resource creator: {} {}".format(resource.creator.first_name, resource.creator.last_name))
print("resource irods bag modified: {}".format(str(resource.getAVU('bag_modified'))))
print("resource irods isPublic: {}".format(str(resource.getAVU('isPublic'))))
print("resource irods resourceType: {}".format(str(resource.getAVU('resourceType'))))
print("resource irods quotaUserName: {}".format(str(resource.getAVU('quotaUserName'))))
if irods_errors:
print("iRODS errors:")
for e in irods_issues:
print(" {}".format(e))
else:
print("No iRODS errors")
if resource.resource_type == 'CompositeResource':
print("Resource file logical files:")
for res_file in resource.files.all():
if res_file.has_logical_file:
print(" {} logical file {} is [{}]".format(res_file.short_path,
str(type(res_file.logical_file)),
str(res_file.logical_file.id)))
def resource_debug(resource_id):
"""Update web services hosted by GeoServer and HydroServer.
"""
from hs_core.hydroshare.utils import get_resource_by_shortkey
resource = get_resource_by_shortkey(resource_id)
from hs_core.management.utils import check_irods_files
return check_irods_files(resource, log_errors=False, return_errors=True)
def handle(self, *args, **options):
if options['unreferenced']:
print("LOOKING FOR IRODS RESOURCES NOT IN DJANGO")
check_for_dangling_irods(echo_errors=not options['log'],
log_errors=options['log'],
return_errors=False)
elif len(options['resource_ids']) > 0: # an array of resource short_id to check.
for rid in options['resource_ids']:
try:
resource = BaseResource.objects.get(short_id=rid)
except BaseResource.DoesNotExist:
msg = "Resource with id {} not found in Django Resources".format(rid)
print(msg)
print("LOOKING FOR FILE ERRORS FOR RESOURCE {}".format(rid))
if options['clean_irods']:
print(' (deleting unreferenced iRODs files)')
if options['clean_django']:
print(' (deleting Django file objects without files)')
if options['sync_ispublic']:
print(' (correcting isPublic in iRODs)')
check_irods_files(resource, stop_on_error=False,
echo_errors=not options['log'],
log_errors=options['log'],
return_errors=False,
clean_irods=options['clean_irods'],
clean_django=options['clean_django'],
sync_ispublic=options['sync_ispublic'])
else: # check all resources
print("LOOKING FOR FILE ERRORS FOR ALL RESOURCES")
if options['clean_irods']:
print(' (deleting unreferenced iRODs files)')
if options['clean_django']:
print(' (deleting Django file objects without files)')
if options['sync_ispublic']:
print(' (correcting isPublic in iRODs)')
for r in BaseResource.objects.all():
check_irods_files(r, stop_on_error=False,
echo_errors=not options['log'], # Don't both log and echo
log_errors=options['log'],
return_errors=False,
clean_irods=options['clean_irods'],
clean_django=options['clean_django'],
sync_ispublic=options['sync_ispublic'])
def debug_resource(request, shortkey):
""" Debug view for resource depicts output of various integrity checking scripts """
resource, _, _ = authorize(
request, shortkey, needed_permission=ACTION_TO_AUTHORIZE.VIEW_RESOURCE)
irods_issues, irods_errors = check_irods_files(resource,
log_errors=False,
return_errors=True)
template = loader.get_template('debug/debug_resource.html')
context = {
'shortkey':
shortkey,
'creator':
resource.creator,
'resource':
resource,
'raccess':
resource.raccess,
'owners':
resource.raccess.owners,
'editors':
resource.raccess.get_users_with_explicit_access(PrivilegeCodes.CHANGE),
'viewers':
resource.raccess.get_users_with_explicit_access(PrivilegeCodes.VIEW),
'public_AVU':
resource.getAVU('isPublic'),
'type_AVU':
resource.getAVU('resourceType'),
'modified_AVU':
resource.getAVU('bag_modified'),
'quota_AVU':
resource.getAVU('quotaUserName'),
'irods_issues':
irods_issues,
'irods_errors':
irods_errors,
}
return HttpResponse(template.render(context, request))
def test_unfederated_root_path_checks(self):
""" an unfederated file in the root folder has the proper state after state changes """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
check_irods_files(self.res, stop_on_error=True)
# add one file to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1)
# should succeed without errors
check_irods_files(self.res, stop_on_error=True)
# cleaning should not change anything
check_irods_files(self.res, stop_on_error=True, log_errors=False, return_errors=True,
clean_irods=True, clean_django=True, sync_ispublic=True)
# resource should has only one file at this point
self.assertEqual(self.res.files.all().count(), 1,
msg="resource file count didn't match")
# get the handle of the file created above
resfile = self.res.files.all()[0]
# determine where that file should live
shortpath = os.path.join(self.res.short_id, "data", "contents", "file1.txt")
self.assertEqual(resfile.file_folder, None)
self.assertEqual(resfile.storage_path, shortpath)
self.assertTrue(resfile.path_is_acceptable(shortpath))
# now try to intentionally corrupt it
resfile.set_short_path("fuzz.txt")
# should raise exception
with self.assertRaises(ValidationError):
check_irods_files(self.res, stop_on_error=True)
# now don't raise exception and read error
errors, ecount = check_irods_files(self.res, return_errors=True, log_errors=False)
self.assertTrue(errors[0].endswith(
'data/contents/fuzz.txt does not exist in iRODS'))
self.assertTrue(errors[1].endswith(
'data/contents/file1.txt in iRODs does not exist in Django'))
self.assertTrue(errors[2].endswith(
"type is GenericResource, title is 'My Test Resource'"))
# now try to clean it up
errors, ecount = check_irods_files(self.res, return_errors=True, log_errors=False,
clean_irods=True, clean_django=True)
self.assertTrue(errors[0].endswith(
'data/contents/fuzz.txt does not exist in iRODS (DELETED FROM DJANGO)'))
self.assertTrue(errors[1].endswith(
'data/contents/file1.txt in iRODs does not exist in Django (DELETED FROM IRODS)'))
self.assertTrue(errors[2].endswith(
"type is GenericResource, title is 'My Test Resource'"))
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
# now check should succeed
errors, ecount = check_irods_files(self.res, stop_on_error=True, log_errors=False)
self.assertEqual(ecount, 0)
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def test_unfederated_root_path_checks(self):
""" an unfederated file in the root folder has the proper state after state changes """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(),
0,
msg="resource file count didn't match")
check_irods_files(self.res, stop_on_error=True)
# add one file to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1)
# should succeed without errors
check_irods_files(self.res, stop_on_error=True)
# cleaning should not change anything
check_irods_files(self.res,
stop_on_error=True,
log_errors=False,
return_errors=True,
clean_irods=True,
clean_django=True,
sync_ispublic=True)
# resource should has only one file at this point
self.assertEqual(self.res.files.all().count(),
1,
msg="resource file count didn't match")
# get the handle of the file created above
resfile = self.res.files.all()[0]
# determine where that file should live
shortpath = os.path.join(self.res.short_id, "data", "contents",
"file1.txt")
self.assertEqual(resfile.file_folder, None)
self.assertEqual(resfile.storage_path, shortpath)
self.assertTrue(resfile.path_is_acceptable(shortpath))
# now try to intentionally corrupt it
resfile.set_short_path("fuzz.txt")
# should raise exception
with self.assertRaises(ValidationError):
check_irods_files(self.res, stop_on_error=True)
# now don't raise exception and read error
errors, ecount = check_irods_files(self.res,
return_errors=True,
log_errors=False)
self.assertTrue(errors[0].endswith(
'data/contents/fuzz.txt does not exist in iRODS'))
self.assertTrue(errors[1].endswith(
'data/contents/file1.txt in iRODs does not exist in Django'))
self.assertTrue(errors[2].endswith(
"type is GenericResource, title is 'My Test Resource'"))
# now try to clean it up
errors, ecount = check_irods_files(self.res,
return_errors=True,
log_errors=False,
clean_irods=True,
clean_django=True)
self.assertTrue(errors[0].endswith(
'data/contents/fuzz.txt does not exist in iRODS (DELETED FROM DJANGO)'
))
self.assertTrue(errors[1].endswith(
'data/contents/file1.txt in iRODs does not exist in Django (DELETED FROM IRODS)'
))
self.assertTrue(errors[2].endswith(
"type is GenericResource, title is 'My Test Resource'"))
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(),
0,
msg="resource file count didn't match")
# now check should succeed
errors, ecount = check_irods_files(self.res,
stop_on_error=True,
log_errors=False)
self.assertEqual(ecount, 0)
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)