def __init__(self,
parent_inode,
inodes,
api,
num_retries,
collection_record=None,
explicit_collection=None):
super(CollectionDirectory, self).__init__(parent_inode, inodes,
api.config, None)
self.api = api
self.num_retries = num_retries
self.collection_record_file = None
self.collection_record = None
self._poll = True
try:
self._poll_time = (
api._rootDesc.get('blobSignatureTtl', 60 * 60 * 2) // 2)
except:
_logger.debug(
"Error getting blobSignatureTtl from discovery document: %s",
sys.exc_info()[0])
self._poll_time = 60 * 60
if isinstance(collection_record, dict):
self.collection_locator = collection_record['uuid']
self._mtime = convertTime(collection_record.get('modified_at'))
else:
self.collection_locator = collection_record
self._mtime = 0
self._manifest_size = 0
if self.collection_locator:
self._writable = (uuid_pattern.match(self.collection_locator)
is not None)
self._updating_lock = threading.Lock()
def __contains__(self, k):
if k in self._entries:
return True
if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)):
return False
try:
e = self.inodes.add_entry(CollectionDirectory(
self.inode, self.inodes, self.api, self.num_retries, k))
if e.update():
if k not in self._entries:
self._entries[k] = e
else:
self.inodes.del_entry(e)
return True
else:
self.inodes.invalidate_entry(self.inode, k)
self.inodes.del_entry(e)
return False
except Exception as ex:
_logger.debug('arv-mount exception keep %s', ex)
self.inodes.del_entry(e)
return False
def __contains__(self, k):
if k in self._entries:
return True
if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)):
return False
try:
e = None
e = self.inodes.add_entry(CollectionDirectory(
self.inode, self.inodes, self.api, self.num_retries, k))
if e.update():
if k not in self._entries:
self._entries[k] = e
else:
self.inodes.del_entry(e)
return True
else:
self.inodes.invalidate_entry(self, k)
self.inodes.del_entry(e)
return False
except Exception as ex:
_logger.exception("arv-mount lookup '%s':", k)
if e is not None:
self.inodes.del_entry(e)
return False
def __contains__(self, k):
if k in self._entries:
return True
if not portable_data_hash_pattern.match(k) and not uuid_pattern.match(
k):
return False
try:
e = self.inodes.add_entry(
CollectionDirectory(self.inode, self.inodes, self.api,
self.num_retries, k))
if e.update():
if k not in self._entries:
self._entries[k] = e
else:
self.inodes.del_entry(e)
return True
else:
self.inodes.del_entry(e)
return False
except Exception as e:
_logger.debug('arv-mount exception keep %s', e)
self.inodes.del_entry(e)
return False
def createDirectory(self, i):
if collection_uuid_pattern.match(i['uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i)
elif group_uuid_pattern.match(i['uuid']):
return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time)
elif link_uuid_pattern.match(i['uuid']):
if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid'])
else:
return None
elif uuid_pattern.match(i['uuid']):
return ObjectFile(self.parent_inode, i)
else:
return None
def createDirectory(self, i):
if collection_uuid_pattern.match(i['uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i)
elif group_uuid_pattern.match(i['uuid']):
return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time)
elif link_uuid_pattern.match(i['uuid']):
if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid'])
else:
return None
elif uuid_pattern.match(i['uuid']):
return ObjectFile(self.parent_inode, i)
else:
return None
def __init__(self, parent_inode, inodes, api, num_retries, collection_record=None, explicit_collection=None):
super(CollectionDirectory, self).__init__(parent_inode, inodes, None)
self.api = api
self.num_retries = num_retries
self.collection_record_file = None
self.collection_record = None
if isinstance(collection_record, dict):
self.collection_locator = collection_record['uuid']
self._mtime = convertTime(collection_record.get('modified_at'))
else:
self.collection_locator = collection_record
self._mtime = 0
self._manifest_size = 0
if self.collection_locator:
self._writable = (uuid_pattern.match(self.collection_locator) is not None)
self._updating_lock = threading.Lock()
def __contains__(self, k):
if k in self._entries:
return True
if not portable_data_hash_pattern.match(k) and (
self.pdh_only or not uuid_pattern.match(k)):
return False
try:
e = None
if group_uuid_pattern.match(k):
project = self.api.groups().list(
filters=[['group_class', '=', 'project'], ["uuid", "=", k]
]).execute(num_retries=self.num_retries)
if project[u'items_available'] == 0:
return False
e = self.inodes.add_entry(
ProjectDirectory(self.inode, self.inodes, self.api,
self.num_retries, project[u'items'][0]))
else:
import sys
e = self.inodes.add_entry(
CollectionDirectory(self.inode, self.inodes, self.api,
self.num_retries, k))
if e.update():
if k not in self._entries:
self._entries[k] = e
else:
self.inodes.del_entry(e)
return True
else:
self.inodes.invalidate_entry(self, k)
self.inodes.del_entry(e)
return False
except Exception as ex:
_logger.exception("arv-mount lookup '%s':", k)
if e is not None:
self.inodes.del_entry(e)
return False
def __init__(self, parent_inode, inodes, api, num_retries, collection_record=None, explicit_collection=None):
super(CollectionDirectory, self).__init__(parent_inode, inodes, None)
self.api = api
self.num_retries = num_retries
self.collection_record_file = None
self.collection_record = None
self._poll = True
try:
self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2)/2)
except:
_logger.debug("Error getting blobSignatureTtl from discovery document: %s", sys.exc_info()[0])
self._poll_time = 60*60
if isinstance(collection_record, dict):
self.collection_locator = collection_record['uuid']
self._mtime = convertTime(collection_record.get('modified_at'))
else:
self.collection_locator = collection_record
self._mtime = 0
self._manifest_size = 0
if self.collection_locator:
self._writable = (uuid_pattern.match(self.collection_locator) is not None)
self._updating_lock = threading.Lock()
def __init__(self,
parent_inode,
inodes,
api,
num_retries,
collection_record=None,
explicit_collection=None):
super(CollectionDirectory, self).__init__(parent_inode, inodes, None)
self.api = api
self.num_retries = num_retries
self.collection_record_file = None
self.collection_record = None
if isinstance(collection_record, dict):
self.collection_locator = collection_record['uuid']
self._mtime = convertTime(collection_record.get('modified_at'))
else:
self.collection_locator = collection_record
self._mtime = 0
self._manifest_size = 0
if self.collection_locator:
self._writable = (uuid_pattern.match(self.collection_locator)
is not None)
self._updating_lock = threading.Lock()
def __contains__(self, k):
if k in self._entries:
return True
if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)):
return False
try:
e = None
if group_uuid_pattern.match(k):
project = self.api.groups().list(
filters=[['group_class', '=', 'project'], ["uuid", "=", k]]).execute(num_retries=self.num_retries)
if project[u'items_available'] == 0:
return False
e = self.inodes.add_entry(ProjectDirectory(
self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0]))
else:
e = self.inodes.add_entry(CollectionDirectory(
self.inode, self.inodes, self.api, self.num_retries, k))
if e.update():
if k not in self._entries:
self._entries[k] = e
else:
self.inodes.del_entry(e)
return True
else:
self.inodes.invalidate_entry(self, k)
self.inodes.del_entry(e)
return False
except Exception as ex:
_logger.exception("arv-mount lookup '%s':", k)
if e is not None:
self.inodes.del_entry(e)
return False
def update(self, to_record_version=None):
try:
if self.collection_record is not None and portable_data_hash_pattern.match(self.collection_locator):
return True
if self.collection_locator is None:
self.fresh()
return True
try:
with llfuse.lock_released:
self._updating_lock.acquire()
if not self.stale():
return
_logger.debug("Updating collection %s inode %s to record version %s", self.collection_locator, self.inode, to_record_version)
if self.collection is not None:
if self.collection.known_past_version(to_record_version):
_logger.debug("%s already processed %s", self.collection_locator, to_record_version)
else:
self.collection.update()
else:
if uuid_pattern.match(self.collection_locator):
coll_reader = arvados.collection.Collection(
self.collection_locator, self.api, self.api.keep,
num_retries=self.num_retries)
else:
coll_reader = arvados.collection.CollectionReader(
self.collection_locator, self.api, self.api.keep,
num_retries=self.num_retries)
new_collection_record = coll_reader.api_response() or {}
# If the Collection only exists in Keep, there will be no API
# response. Fill in the fields we need.
if 'uuid' not in new_collection_record:
new_collection_record['uuid'] = self.collection_locator
if "portable_data_hash" not in new_collection_record:
new_collection_record["portable_data_hash"] = new_collection_record["uuid"]
if 'manifest_text' not in new_collection_record:
new_collection_record['manifest_text'] = coll_reader.manifest_text()
if self.collection_record is None or self.collection_record["portable_data_hash"] != new_collection_record.get("portable_data_hash"):
self.new_collection(new_collection_record, coll_reader)
self._manifest_size = len(coll_reader.manifest_text())
_logger.debug("%s manifest_size %i", self, self._manifest_size)
# end with llfuse.lock_released, re-acquire lock
self.fresh()
return True
finally:
self._updating_lock.release()
except arvados.errors.NotFoundError as e:
_logger.error("Error fetching collection '%s': %s", self.collection_locator, e)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
if self.collection_record is not None and "manifest_text" in self.collection_record:
_logger.warning("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
except Exception:
_logger.exception("arv-mount %s: error", self.collection_locator)
if self.collection_record is not None and "manifest_text" in self.collection_record:
_logger.error("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
self.invalidate()
return False
def want_event_subscribe(self):
return (uuid_pattern.match(self.collection_locator) is not None)
def update(self, to_record_version=None):
try:
if self.collection_record is not None and portable_data_hash_pattern.match(
self.collection_locator):
return True
if self.collection_locator is None:
self.fresh()
return True
try:
with llfuse.lock_released:
self._updating_lock.acquire()
if not self.stale():
return
_logger.debug(
"Updating collection %s inode %s to record version %s",
self.collection_locator, self.inode, to_record_version)
if self.collection is not None:
if self.collection.known_past_version(
to_record_version):
_logger.debug("%s already processed %s",
self.collection_locator,
to_record_version)
else:
self.collection.update()
else:
if uuid_pattern.match(self.collection_locator):
coll_reader = arvados.collection.Collection(
self.collection_locator,
self.api,
self.api.keep,
num_retries=self.num_retries)
else:
coll_reader = arvados.collection.CollectionReader(
self.collection_locator,
self.api,
self.api.keep,
num_retries=self.num_retries)
new_collection_record = coll_reader.api_response(
) or {}
# If the Collection only exists in Keep, there will be no API
# response. Fill in the fields we need.
if 'uuid' not in new_collection_record:
new_collection_record[
'uuid'] = self.collection_locator
if "portable_data_hash" not in new_collection_record:
new_collection_record[
"portable_data_hash"] = new_collection_record[
"uuid"]
if 'manifest_text' not in new_collection_record:
new_collection_record[
'manifest_text'] = coll_reader.manifest_text()
if self.collection_record is None or self.collection_record[
"portable_data_hash"] != new_collection_record.get(
"portable_data_hash"):
self.new_collection(new_collection_record,
coll_reader)
self._manifest_size = len(coll_reader.manifest_text())
_logger.debug("%s manifest_size %i", self,
self._manifest_size)
# end with llfuse.lock_released, re-acquire lock
self.fresh()
return True
finally:
self._updating_lock.release()
except arvados.errors.NotFoundError as e:
_logger.error("Error fetching collection '%s': %s",
self.collection_locator, e)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator,
detail)
if self.collection_record is not None and "manifest_text" in self.collection_record:
_logger.warning("arv-mount manifest_text is: %s",
self.collection_record["manifest_text"])
except Exception:
_logger.exception("arv-mount %s: error", self.collection_locator)
if self.collection_record is not None and "manifest_text" in self.collection_record:
_logger.error("arv-mount manifest_text is: %s",
self.collection_record["manifest_text"])
self.invalidate()
return False
def want_event_subscribe(self):
return (uuid_pattern.match(self.collection_locator) is not None)