def replace_resource_file_on_irods(new_file, original_resource_file, user): """ Replaces the specified resource file with file (new_file) by copying to iRODS (local or federated zone) :param new_file: file path for the file to be copied to iRODS :param original_resource_file: an instance of ResourceFile that is to be replaced :param user: user who is replacing the resource file. :return: """ ori_res = original_resource_file.resource if original_resource_file.resource_file: istorage = IrodsStorage() destination_file = original_resource_file.resource_file.name else: istorage = IrodsStorage('federated') if original_resource_file.fed_resource_file: destination_file = original_resource_file.fed_resource_file.name else: destination_file = os.path.join( ori_res.resource_federation_path, ori_res.short_id, original_resource_file.fed_resource_file_name_or_path) istorage.saveFile(new_file, destination_file, True) # need to do this so that the bag will be regenerated prior to download of the bag resource_modified(ori_res, by_user=user, overwrite_bag=False)
def is_federated(homepath): """ Check if the selected file via the iRODS browser is from a federated zone or not Args: homepath: the logical iRODS file name with full logical path, e.g., selected from iRODS browser Returns: True is the selected file indicated by homepath is from a federated zone, False if otherwise """ homepath = homepath.strip() homepath_list = homepath.split('/') # homepath is an iRODS logical path in the format of # /irods_zone/home/irods_account_username/collection_relative_path, so homepath_list[1] # is the irods_zone which we can use to form the fed_proxy_path to check whether # fed_proxy_path exists to hold hydroshare resources in a federated zone if homepath_list[1]: fed_proxy_path = os.path.join(homepath_list[1], 'home', settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) fed_proxy_path = '/' + fed_proxy_path else: # the test path input is invalid, return False meaning it is not federated return False if settings.REMOTE_USE_IRODS: irods_storage = IrodsStorage('federated') else: irods_storage = IrodsStorage() # if the iRODS proxy user in hydroshare zone can list homepath and the federation zone proxy # user path, it is federated; otherwise, it is not federated return irods_storage.exists(homepath) and irods_storage.exists( fed_proxy_path)
def assert_federated_irods_available(self): """assert federated iRODS is available before proceeding with federation-related tests.""" self.assertTrue(settings.REMOTE_USE_IRODS and settings.HS_USER_ZONE_HOST == 'users.local.org' and settings.IRODS_HOST == 'data.local.org', "irods docker containers are not set up properly for federation testing") self.irods_fed_storage = IrodsStorage('federated') self.irods_storage = IrodsStorage()
def create_empty_contents_directory(resource): res_id = resource.short_id if resource.resource_federation_path: istorage = IrodsStorage('federated') res_contents_dir = '{}/{}/data/contents'.format( resource.resource_federation_path, res_id) else: istorage = IrodsStorage() res_contents_dir = '{}/data/contents'.format(res_id) if not istorage.exists(res_contents_dir): istorage.session.run("imkdir", None, '-p', res_contents_dir)
def get_fed_zone_files(irods_fnames): """ Get the files from iRODS federated zone to Django server for metadata extraction on-demand for specific resource types Args: irods_fnames: the logical iRODS file names with full logical path separated by comma Returns: a list of the named temp files which have been copied over to local Django server or raise exceptions if input parameter is wrong or iRODS operations fail """ ret_file_list = [] if isinstance(irods_fnames, basestring): ifnames = string.split(irods_fnames, ',') elif isinstance(irods_fnames, list): ifnames = irods_fnames else: raise ValueError( "Input parameter to get_fed_zone_files() must be String or List") irods_storage = IrodsStorage('federated') for ifname in ifnames: fname = os.path.basename(ifname.rstrip(os.sep)) tmpdir = os.path.join(settings.TEMP_FILE_DIR, uuid4().hex) tmpfile = os.path.join(tmpdir, fname) try: os.makedirs(tmpdir) except OSError as ex: if ex.errno == errno.EEXIST: shutil.rmtree(tmpdir) os.makedirs(tmpdir) else: raise Exception(ex.message) irods_storage.getFile(ifname, tmpfile) ret_file_list.append(tmpfile) return ret_file_list
def check_for_dangling_irods(echo_errors=True, log_errors=False, return_errors=False): """ This checks for resource trees in iRODS with no correspondence to Django at all :param log_errors: whether to log errors to Django log :param echo_errors: whether to print errors on stdout :param return_errors: whether to collect errors in an array and return them. """ istorage = IrodsStorage() # local only toplevel = istorage.listdir('.') # list the resources themselves logger = logging.getLogger(__name__) errors = [] for id in toplevel[0]: # directories try: get_resource_by_shortkey(id, or_404=False) except BaseResource.DoesNotExist: msg = "resource {} does not exist in Django".format(id) if echo_errors: print(msg) if log_errors: logger.error(msg) if return_errors: errors.append(msg) return errors
def upload_from_irods(username, password, host, port, zone, irods_fnames, res_files): """ use iget to transfer selected data object from irods zone to local as a NamedTemporaryFile :param username: iRODS login account username used to download irods data object for uploading :param password: iRODS login account password used to download irods data object for uploading :param host: iRODS login host used to download irods data object for uploading :param port: iRODS login port used to download irods data object for uploading :param zone: iRODS login zone used to download irods data object for uploading :param irods_fnames: the data object file name to download to local for uploading :param res_files: list of files for uploading to create resources :raises SessionException(proc.returncode, stdout, stderr) defined in django_irods/icommands.py to capture iRODS exceptions raised from iRODS icommand subprocess run triggered from any method calls from IrodsStorage() if an error or exception ever occurs :return: None, but the downloaded file from the iRODS will be appended to res_files list for uploading """ irods_storage = IrodsStorage() irods_storage.set_user_session(username=username, password=password, host=host, port=port, zone=zone) ifnames = string.split(irods_fnames, ',') for ifname in ifnames: size = irods_storage.size(ifname) tmpFile = irods_storage.download(ifname) fname = os.path.basename(ifname.rstrip(os.sep)) fileobj = File(file=tmpFile, name=fname) fileobj.size = size res_files.append(fileobj) # delete the user session after iRODS file operations are done irods_storage.delete_user_session()
def nightly_zips_cleanup(): # delete 2 days ago date_folder = (date.today() - timedelta(2)).strftime('%Y-%m-%d') zips_daily_date = "zips/{daily_date}".format(daily_date=date_folder) istorage = IrodsStorage() if istorage.exists(zips_daily_date): istorage.delete(zips_daily_date)
def handle(self, *args, **options): istorage = IrodsStorage() # reset quota for data zone root_path = '/{}/home/{}'.format(settings.IRODS_ZONE, settings.IRODS_USERNAME) istorage.setAVU(root_path, 'resetQuotaDir', 1) # reset quota for user zone user_root_path = '/{}/home/{}'.format(settings.HS_USER_IRODS_ZONE, settings.HS_IRODS_PROXY_USER_IN_USER_ZONE) istorage.setAVU(user_root_path, 'resetQuotaDir', 1)
def test_create_temp_zip(self): input_path = "/data/contents/foo" try: self.assertTrue(create_temp_zip(self.res.short_id, input_path, self.output_path)) self.assertTrue(IrodsStorage().exists(self.output_path)) except Exception as ex: self.fail("create_temp_zip() raised exception.{}".format(ex.message))
def tearDown(self): super(TestFolderDownloadZip, self).tearDown() if self.res: self.res.delete() GenericResource.objects.all().delete() istorage = IrodsStorage() if istorage.exists(self.output_path): istorage.delete(self.output_path)
def delete_fed_zone_file(file_name_with_full_path): ''' Args: file_name_with_full_path: the absolute full logical path in a federated iRODS zone Returns: None, but exceptions will be raised if there is an issue with iRODS delete operation ''' istorage = IrodsStorage('federated') istorage.delete(file_name_with_full_path)
def get_file_from_irods(res_file): """ Copy the file (res_file) from iRODS (local or federated zone) over to django (temp directory) which is necessary for manipulating the file (e.g. metadata extraction). Note: The caller is responsible for cleaning the temp directory :param res_file: an instance of ResourceFile :return: location of the copied file """ res = res_file.resource if res_file.fed_resource_file or res_file.fed_resource_file_name_or_path: istorage = IrodsStorage('federated') else: istorage = IrodsStorage() if res_file.resource_file: res_file_path = res_file.resource_file.name file_name = os.path.basename(res_file.resource_file.name) elif res_file.fed_resource_file: res_file_path = res_file.fed_resource_file.name file_name = os.path.basename(res_file.fed_resource_file.name) else: res_file_path = os.path.join(res.resource_federation_path, res.short_id, res_file.fed_resource_file_name_or_path) file_name = os.path.basename(res_file.fed_resource_file_name_or_path) tmpdir = os.path.join(settings.TEMP_FILE_DIR, uuid4().hex) tmpfile = os.path.join(tmpdir, file_name) try: os.makedirs(tmpdir) except OSError as ex: if ex.errno == errno.EEXIST: shutil.rmtree(tmpdir) os.makedirs(tmpdir) else: raise Exception(ex.message) istorage.getFile(res_file_path, tmpfile) copied_file = tmpfile return copied_file
def save_files_to_user_zone(self, file_name_to_target_name_dict): """Save a list of files to iRODS user zone using the same IrodsStorage() object. :param file_name_to_target_name_dict: a dictionary in the form of {ori_file, target_file} where ori_file is the file to be save to, and the target_file is the full path file name in iRODS user zone to save ori_file to :return: """ self.irods_storage = IrodsStorage('federated') for file_name, target_name in file_name_to_target_name_dict.iteritems(): self.irods_storage.saveFile(file_name, target_name)
def test_create_temp_zip(self): input_path = "{}/data/contents/foo".format(self.res.short_id) output_path = "zips/rand/foo.zip" self.assertTrue( create_temp_zip(self.res.short_id, input_path, output_path, False)) self.assertTrue(IrodsStorage().exists(output_path)) # test aggregation input_path = "{}/data/contents/multi_sites_formatted_version1.0.refts.json"\ .format(self.res.short_id) output_path = "zips/rand/multi_sites_formatted_version1.0.refts.json.zip" self.assertTrue( create_temp_zip(self.res.short_id, input_path, output_path, True, sf_zip=True)) self.assertTrue(IrodsStorage().exists(output_path))
def get_fed_zone_file_size(fname): """ Get size of a data object from iRODS user zone Args: fname: the logical iRODS file name with full logical path Returns: the size of the file """ irods_storage = IrodsStorage('federated') return irods_storage.size(fname)
def get_quota_usage_from_irods(username): """ Query iRODS AVU to get quota usage for a user reported in iRODS quota microservices :param username: the user name to get quota usage for. :return: the combined quota usage from iRODS data zone and user zone; raise ValidationError if quota usage cannot be retrieved from iRODS """ attname = username + '-usage' istorage = IrodsStorage() # get quota size for user in iRODS data zone by retrieving AVU set on irods bagit path # collection try: uqDataZoneSize = istorage.getAVU(settings.IRODS_BAGIT_PATH, attname) if uqDataZoneSize is None: # user may not have resources in data zone, so corresponding quota size AVU may not # exist for this user uqDataZoneSize = -1 else: uqDataZoneSize = float(uqDataZoneSize) except SessionException: # user may not have resources in data zone, so corresponding quota size AVU may not exist # for this user uqDataZoneSize = -1 # get quota size for the user in iRODS user zone try: uz_bagit_path = os.path.join('/', settings.HS_USER_IRODS_ZONE, 'home', settings.HS_IRODS_PROXY_USER_IN_USER_ZONE, settings.IRODS_BAGIT_PATH) uqUserZoneSize = istorage.getAVU(uz_bagit_path, attname) if uqUserZoneSize is None: # user may not have resources in user zone, so corresponding quota size AVU may not # exist for this user uqUserZoneSize = -1 else: uqUserZoneSize = float(uqUserZoneSize) except SessionException: # user may not have resources in user zone, so corresponding quota size AVU may not exist # for this user uqUserZoneSize = -1 if uqDataZoneSize < 0 and uqUserZoneSize < 0: err_msg = 'no quota size AVU in data zone and user zone for user {}'.format( username) logger.error(err_msg) raise ValidationError(err_msg) elif uqUserZoneSize < 0: used_val = uqDataZoneSize elif uqDataZoneSize < 0: used_val = uqUserZoneSize else: used_val = uqDataZoneSize + uqUserZoneSize return used_val
def tearDown(self): super(TestFolderDownloadZip, self).tearDown() if self.res: self.res.delete() if self.test_file: os.remove(self.test_file.name) if self.refts_file: os.remove(self.refts_file.name) GenericResource.objects.all().delete() istorage = IrodsStorage() if istorage.exists("zips"): istorage.delete("zips")
def create_temp_zip(resource_id, input_path, output_path): from hs_core.hydroshare.utils import get_resource_by_shortkey res = get_resource_by_shortkey(resource_id) full_input_path = '{root_path}/{path}'.format(root_path=res.root_path, path=input_path) try: IrodsStorage().zipup(full_input_path, output_path) except SessionException as ex: logger.error(ex.stderr) return False return True
def nightly_zips_cleanup(): # delete 2 days ago date_folder = (date.today() - timedelta(2)).strftime('%Y-%m-%d') zips_daily_date = "zips/{daily_date}".format(daily_date=date_folder) if __debug__: logger.debug("cleaning up {}".format(zips_daily_date)) istorage = IrodsStorage() if istorage.exists(zips_daily_date): istorage.delete(zips_daily_date) federated_prefixes = BaseResource.objects.all().values_list('resource_federation_path')\ .distinct() for p in federated_prefixes: prefix = p[0] # strip tuple if prefix != "": zips_daily_date = "{prefix}/zips/{daily_date}"\ .format(prefix=prefix, daily_date=date_folder) if __debug__: logger.debug("cleaning up {}".format(zips_daily_date)) istorage = IrodsStorage("federated") if istorage.exists(zips_daily_date): istorage.delete(zips_daily_date)
def resource_modified(resource, by_user=None, overwrite_bag=True): resource.last_changed_by = by_user resource.updated = now().isoformat() resource.save() if resource.metadata.dates.all().filter(type='modified'): res_modified_date = resource.metadata.dates.all().filter( type='modified')[0] resource.metadata.update_element('date', res_modified_date.id) if overwrite_bag: create_bag_files(resource) istorage = IrodsStorage() # set bag_modified-true AVU pair for the modified resource in iRODS to indicate # the resource is modified for on-demand bagging. istorage.setAVU(resource.short_id, "bag_modified", "true")
def verify_user_quota_usage_avu_in_user_zone(self, attname, qsize): ''' Have to use LINUX_ADMIN_USER_FOR_HS_USER_ZONE with rodsadmin role to get user type AVU in user zone and verify its quota usage is set correctly :param attname: quota usage attribute name set on iRODS proxy user in user zone :param qsize: quota size (type string) to be verified to equal to the value set for attname. ''' istorage = IrodsStorage() istorage.set_user_session(username=settings.LINUX_ADMIN_USER_FOR_HS_USER_ZONE, password=settings.LINUX_ADMIN_USER_PWD_FOR_HS_USER_ZONE, host=settings.HS_USER_ZONE_HOST, port=settings.IRODS_PORT, zone=settings.HS_USER_IRODS_ZONE, sess_id='user_proxy_session') uz_bagit_path = os.path.join('/', settings.HS_USER_IRODS_ZONE, 'home', settings.HS_IRODS_PROXY_USER_IN_USER_ZONE, settings.IRODS_BAGIT_PATH) get_qsize = istorage.getAVU(uz_bagit_path, attname) self.assertEqual(qsize, get_qsize)
def setUp(self): super(TestFolderDownloadZip, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') self.user = create_account('*****@*****.**', username='******', first_name='Shaun', last_name='Livingston', superuser=False, groups=[]) self.res = create_resource(resource_type='CompositeResource', owner=self.user, title='Test Resource', metadata=[]) ResourceFile.create_folder(self.res, 'foo') # create files self.n1 = "test1.txt" test_file = open(self.n1, 'w') test_file.write("Test text file in test1.txt") test_file.close() self.test_file = open(self.n1, "rb") add_resource_files(self.res.short_id, self.test_file, folder='foo') # copy refts file into new file to be added to the resource as an aggregation reft_data_file = open( 'hs_core/tests/data/multi_sites_formatted_version1.0.refts.json', 'rb') refts_file = open('multi_sites_formatted_version1.0.refts.json', 'wb') refts_file.writelines(reft_data_file.readlines()) refts_file.close() self.refts_file = open('multi_sites_formatted_version1.0.refts.json', 'rb') add_resource_files(self.res.short_id, self.refts_file) self.res.create_aggregation_xml_documents() self.istorage = IrodsStorage()
def migrate_tif_file(apps, schema_editor): # create a vrt file from tif file for each of the Raster Resources log = logging.getLogger() istorage = IrodsStorage() for res in RasterResource.objects.all(): try: if len(res.files.all()) == 1: res_file = res.files.all().first() vrt_file_path = create_vrt_file(res_file.resource_file) if os.path.isfile(vrt_file_path): files = (UploadedFile( file=open(vrt_file_path, 'r'), name=os.path.basename(vrt_file_path))) hydroshare.add_resource_files(res.short_id, files) bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id) if istorage.exists(bag_name): # delete the resource bag as the old bag is not valid istorage.delete(bag_name) print("Deleted bag for resource ID:" + str(res.short_id)) resource_modified(res, res.creator) log.info( 'Tif file conversion to VRT successful for resource:ID:{} ' 'Title:{}'.format(res.short_id, res.metadata.title.value)) else: log.error( 'Tif file conversion to VRT unsuccessful for resource:ID:{} ' 'Title:{}'.format(res.short_id, res.metadata.title.value)) if os.path.exists(vrt_file_path): shutil.rmtree(os.path.dirname(vrt_file_path)) except: pass
def delete_bag(resource): """ delete the resource bag Parameters: :param resource: the resource to delete the bag for. :return: none """ res_id = resource.short_id istorage = IrodsStorage() # delete resource directory first to remove all generated bag-related files for the resource istorage.delete(res_id) # the resource bag may not exist due to on-demand bagging bagname = 'bags/{res_id}.zip'.format(res_id=res_id) if istorage.exists(bagname): # delete the resource bag istorage.delete(bagname) # delete the bags table for bag in resource.bags.all(): bag.delete()
def get_size_and_avu_for_irods_ref_files(username, password, host, port, zone, irods_fnames): """ use iget to transfer selected data object from irods zone to local as a NamedTemporaryFile :param username: iRODS login account username used to download irods data object for uploading :param password: iRODS login account password used to download irods data object for uploading :param host: iRODS login host used to download irods data object for uploading :param port: iRODS login port used to download irods data object for uploading :param zone: iRODS login zone used to download irods data object for uploading :param irods_fnames: the data object file name to download to local for uploading :raises SessionException(proc.returncode, stdout, stderr) defined in django_irods/icommands.py to capture iRODS exceptions raised from iRODS icommand subprocess run triggered from any method calls from IrodsStorage() if an error or exception ever occurs :return: list of file sizes corresponding to irods_fnames, and extra metadata dicts with each dict item corresponding to iRODS AVUs on the file or data object with file name preappended to attributes """ irods_storage = IrodsStorage() irods_storage.set_user_session(username=username, password=password, host=host, port=port, zone=zone) ifnames = string.split(irods_fnames, ',') ifsizes = [] ifextra_mds = {} for ifname in ifnames: size = irods_storage.size(ifname) ifsizes.append(size) extra_md_dict = irods_storage.getAVU(ifname, type='-d') for key, val in extra_md_dict.iteritems(): ukey = ifname + '_' + key ifextra_mds[ukey] = val # delete the user session after iRODS file operations are done irods_storage.delete_user_session() return ifsizes, ifextra_mds
def create_bag_by_irods(resource_id, istorage=None): """ create a resource bag on iRODS side by running the bagit rule followed by ibun zipping operation Parameters: :param resource_id: the resource uuid that is used to look for the resource to create the bag for. istorage: IrodsStorage object that is used to call irods bagit rule operation and zipping up operation :return: none """ if not istorage: istorage = IrodsStorage() # only proceed when the resource is not deleted potentially by another request when being downloaded if istorage.exists(resource_id): # call iRODS bagit rule here irods_dest_prefix = "/" + settings.IRODS_ZONE + "/home/" + settings.IRODS_USERNAME irods_bagit_input_path = os.path.join(irods_dest_prefix, resource_id) bagit_input_path = "*BAGITDATA='{path}'".format( path=irods_bagit_input_path) bagit_input_resource = "*DESTRESC='{def_res}'".format( def_res=settings.IRODS_DEFAULT_RESOURCE) bagit_rule_file = getattr(settings, 'IRODS_BAGIT_RULE', 'hydroshare/irods/ruleGenerateBagIt_HS.r') try: # call iRODS run and ibun command to create and zip the bag, # ignore SessionException for now as a workaround which could be raised # from potential race conditions when multiple ibun commands try to create the same zip file or # the very same resource gets deleted by another request when being downloaded istorage.runBagitRule(bagit_rule_file, bagit_input_path, bagit_input_resource) istorage.zipup(irods_bagit_input_path, 'bags/{res_id}.zip'.format(res_id=resource_id)) except SessionException: pass
def update_quota_usage_task(username): """update quota usage. This function runs as a celery task, invoked asynchronously with 1 minute delay to give enough time for iRODS real time quota update micro-services to update quota usage AVU for the user before this celery task to check this AVU to get the updated quota usage for the user. Note iRODS micro-service quota update only happens on HydroShare iRODS data zone and user zone independently, so the aggregation of usage in both zones need to be accounted for in this function to update Django DB as an aggregated usage for hydroshare internal zone. :param username: the name of the user that needs to update quota usage for. :return: True if quota usage update succeeds; False if there is an exception raised or quota cannot be updated. See log for details. """ hs_internal_zone = "hydroshare" uq = UserQuota.objects.filter(user__username=username, zone=hs_internal_zone).first() if uq is None: # the quota row does not exist in Django logger.error( 'quota row does not exist in Django for hydroshare zone for ' 'user ' + username) return False attname = username + '-usage' istorage = IrodsStorage() # get quota size for user in iRODS data zone by retrieving AVU set on irods bagit path # collection try: uqDataZoneSize = istorage.getAVU(settings.IRODS_BAGIT_PATH, attname) if uqDataZoneSize is None: # user may not have resources in data zone, so corresponding quota size AVU may not # exist for this user uqDataZoneSize = -1 else: uqDataZoneSize = float(uqDataZoneSize) except SessionException: # user may not have resources in data zone, so corresponding quota size AVU may not exist # for this user uqDataZoneSize = -1 # get quota size for the user in iRODS user zone try: uz_bagit_path = os.path.join('/', settings.HS_USER_IRODS_ZONE, 'home', settings.HS_IRODS_PROXY_USER_IN_USER_ZONE, settings.IRODS_BAGIT_PATH) uqUserZoneSize = istorage.getAVU(uz_bagit_path, attname) if uqUserZoneSize is None: # user may not have resources in user zone, so corresponding quota size AVU may not # exist for this user uqUserZoneSize = -1 else: uqUserZoneSize = float(uqUserZoneSize) except SessionException: # user may not have resources in user zone, so corresponding quota size AVU may not exist # for this user uqUserZoneSize = -1 if uqDataZoneSize < 0 and uqUserZoneSize < 0: logger.error( 'no quota size AVU in data zone and user zone for the user ' + username) return False elif uqUserZoneSize < 0: used_val = uqDataZoneSize elif uqDataZoneSize < 0: used_val = uqUserZoneSize else: used_val = uqDataZoneSize + uqUserZoneSize uq.update_used_value(used_val) return True
def create_bag_files(resource): """ create and update all files needed by bagit operation that is conducted on iRODS server; no bagit operation is performed, only files that will be included in the bag are created or updated. Parameters: :param resource: A resource whose files will be created or updated to be included in the resource bag. :return: istorage, an IrodsStorage object,that will be used by subsequent operation to create a bag on demand as needed. """ from . import utils as hs_core_utils DATE_FORMAT = "YYYY-MM-DDThh:mm:ssTZD" istorage = IrodsStorage() dest_prefix = getattr(settings, 'BAGIT_TEMP_LOCATION', '/tmp/hydroshare/') bagit_path = os.path.join( dest_prefix, resource.short_id, arrow.get(resource.updated).format("YYYY.MM.DD.HH.mm.ss")) for d in (dest_prefix, bagit_path): try: os.makedirs(d) except OSError as ex: if ex.errno == errno.EEXIST: shutil.rmtree(d) os.makedirs(d) else: raise Exception(ex.message) # an empty visualization directory will not be put into the zipped bag file by ibun command, so creating an empty # visualization directory to be put into the zip file as done by the two statements below does not work. However, # if visualization directory has content to be uploaded, it will work. This is to be implemented as part of the # resource model in the future. The following two statements are placeholders serving as reminder # to_file_name = '{res_id}/data/visualization/'.format(res_id=resource.short_id) # istorage.saveFile('', to_file_name, create_directory=True) # create resourcemetadata.xml and upload it to iRODS from_file_name = '{path}/resourcemetadata.xml'.format(path=bagit_path) with open(from_file_name, 'w') as out: out.write(resource.metadata.get_xml()) to_file_name = '{res_id}/data/resourcemetadata.xml'.format( res_id=resource.short_id) istorage.saveFile(from_file_name, to_file_name, True) # make the resource map current_site_url = hs_core_utils.current_site_url() hs_res_url = '{hs_url}/resource/{res_id}/data'.format( hs_url=current_site_url, res_id=resource.short_id) metadata_url = os.path.join(hs_res_url, 'resourcemetadata.xml') res_map_url = os.path.join(hs_res_url, 'resourcemap.xml') ##make the resource map: # utils.namespaces['hsterms'] = Namespace('{hs_url}/hsterms/'.format(hs_url=current_site_url)) # utils.namespaceSearchOrder.append('hsterms') utils.namespaces['citoterms'] = Namespace('http://purl.org/spar/cito/') utils.namespaceSearchOrder.append('citoterms') ag_url = os.path.join(hs_res_url, 'resourcemap.xml#aggregation') a = Aggregation(ag_url) #Set properties of the aggregation a._dc.title = resource.title a._dcterms.type = resource._meta.object_name a._citoterms.isDocumentedBy = metadata_url a._ore.isDescribedBy = res_map_url #Create a description of the metadata document that describes the whole resource and add it to the aggregation resMetaFile = AggregatedResource(metadata_url) resMetaFile._dc.title = "Dublin Core science metadata document describing the HydroShare resource" resMetaFile._citoterms.documents = ag_url resMetaFile._ore.isAggregatedBy = ag_url resMetaFile._dc.format = "application/rdf+xml" #Create a description of the content file and add it to the aggregation files = ResourceFile.objects.filter(object_id=resource.id) resFiles = [] for n, f in enumerate(files): filename = os.path.basename(f.resource_file.name) resFiles.append( AggregatedResource( os.path.join( '{hs_url}/resource/{res_id}/data/contents/{file_name}'. format(hs_url=current_site_url, res_id=resource.short_id, file_name=filename)))) resFiles[n]._ore.isAggregatedBy = ag_url resFiles[n]._dc.format = hs_core_utils.get_file_mime_type(filename) #Add the resource files to the aggregation a.add_resource(resMetaFile) for f in resFiles: a.add_resource(f) #Register a serializer with the aggregation. The registration creates a new ResourceMap, which needs a URI serializer = RdfLibSerializer('xml') resMap = a.register_serialization(serializer, res_map_url) resMap._dc.identifier = resource.short_id #"resource_identifier" #Fetch the serialization remdoc = a.get_serialization() # change the namespace for the 'creator' element from 'dcterms' to 'dc' xml_string = remdoc.data.replace('dcterms:creator', 'dc:creator') # create resourcemap.xml and upload it to iRODS from_file_name = os.path.join(bagit_path, 'resourcemap.xml') with open(from_file_name, 'w') as out: out.write(xml_string) to_file_name = os.path.join(resource.short_id, 'data', 'resourcemap.xml') istorage.saveFile(from_file_name, to_file_name, False) shutil.rmtree(bagit_path) return istorage
def get_file_storage(): return IrodsStorage() if getattr(settings, 'USE_IRODS', False) else DefaultStorage()