def __contains__(self, k):
if k in self._entries:
return True
if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)):
return False
try:
e = self.inodes.add_entry(CollectionDirectory(
self.inode, self.inodes, self.api, self.num_retries, k))
if e.update():
if k not in self._entries:
self._entries[k] = e
else:
self.inodes.del_entry(e)
return True
else:
self.inodes.invalidate_entry(self.inode, k)
self.inodes.del_entry(e)
return False
except Exception as ex:
_logger.debug('arv-mount exception keep %s', ex)
self.inodes.del_entry(e)
return False
def update(self):
try:
if self.collection_object is not None and portable_data_hash_pattern.match(self.collection_locator):
return True
with llfuse.lock_released:
new_collection_object = self.api.collections().get(
uuid=self.collection_locator
).execute(num_retries=self.num_retries)
if "portable_data_hash" not in new_collection_object:
new_collection_object["portable_data_hash"] = new_collection_object["uuid"]
# end with llfuse.lock_released, re-acquire lock
if self.collection_object is None or self.collection_object["portable_data_hash"] != new_collection_object["portable_data_hash"]:
self.new_collection(new_collection_object)
self.fresh()
return True
except apiclient.errors.HttpError as e:
if e.resp.status == 404:
_logger.warn("arv-mount %s: not found", self.collection_locator)
else:
_logger.error("arv-mount %s: error", self.collection_locator)
_logger.exception(detail)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
if self.collection_object is not None and "manifest_text" in self.collection_object:
_logger.warning("arv-mount manifest_text is: %s", self.collection_object["manifest_text"])
except Exception as detail:
_logger.error("arv-mount %s: error", self.collection_locator)
if self.collection_object is not None and "manifest_text" in self.collection_object:
_logger.error("arv-mount manifest_text is: %s", self.collection_object["manifest_text"])
_logger.exception(detail)
return False
def __contains__(self, k):
if k in self._entries:
return True
if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)):
return False
try:
e = None
e = self.inodes.add_entry(CollectionDirectory(
self.inode, self.inodes, self.api, self.num_retries, k))
if e.update():
if k not in self._entries:
self._entries[k] = e
else:
self.inodes.del_entry(e)
return True
else:
self.inodes.invalidate_entry(self, k)
self.inodes.del_entry(e)
return False
except Exception as ex:
_logger.exception("arv-mount lookup '%s':", k)
if e is not None:
self.inodes.del_entry(e)
return False
def __contains__(self, k):
if k in self._entries:
return True
if not portable_data_hash_pattern.match(k) and not uuid_pattern.match(
k):
return False
try:
e = self.inodes.add_entry(
CollectionDirectory(self.inode, self.inodes, self.api,
self.num_retries, k))
if e.update():
if k not in self._entries:
self._entries[k] = e
else:
self.inodes.del_entry(e)
return True
else:
self.inodes.del_entry(e)
return False
except Exception as e:
_logger.debug('arv-mount exception keep %s', e)
self.inodes.del_entry(e)
return False
def update(self):
try:
if self.collection_object is not None and portable_data_hash_pattern.match(
self.collection_locator):
return True
if self.collection_locator is None:
self.fresh()
return True
with llfuse.lock_released:
coll_reader = arvados.CollectionReader(
self.collection_locator,
self.api,
self.api.keep,
num_retries=self.num_retries)
new_collection_object = coll_reader.api_response() or {}
# If the Collection only exists in Keep, there will be no API
# response. Fill in the fields we need.
if 'uuid' not in new_collection_object:
new_collection_object['uuid'] = self.collection_locator
if "portable_data_hash" not in new_collection_object:
new_collection_object[
"portable_data_hash"] = new_collection_object["uuid"]
if 'manifest_text' not in new_collection_object:
new_collection_object[
'manifest_text'] = coll_reader.manifest_text()
coll_reader.normalize()
# end with llfuse.lock_released, re-acquire lock
if self.collection_object is None or self.collection_object[
"portable_data_hash"] != new_collection_object[
"portable_data_hash"]:
self.new_collection(new_collection_object, coll_reader)
self.fresh()
return True
except arvados.errors.NotFoundError:
_logger.exception("arv-mount %s: error", self.collection_locator)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator,
detail)
if self.collection_object is not None and "manifest_text" in self.collection_object:
_logger.warning("arv-mount manifest_text is: %s",
self.collection_object["manifest_text"])
except Exception:
_logger.exception("arv-mount %s: error", self.collection_locator)
if self.collection_object is not None and "manifest_text" in self.collection_object:
_logger.error("arv-mount manifest_text is: %s",
self.collection_object["manifest_text"])
return False
def createDirectory(self, i):
if collection_uuid_pattern.match(i['uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i)
elif group_uuid_pattern.match(i['uuid']):
return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time)
elif link_uuid_pattern.match(i['uuid']):
if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid'])
else:
return None
elif uuid_pattern.match(i['uuid']):
return ObjectFile(self.parent_inode, i)
else:
return None
def createDirectory(self, i):
if collection_uuid_pattern.match(i['uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i)
elif group_uuid_pattern.match(i['uuid']):
return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time)
elif link_uuid_pattern.match(i['uuid']):
if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid'])
else:
return None
elif uuid_pattern.match(i['uuid']):
return ObjectFile(self.parent_inode, i)
else:
return None
def __contains__(self, k):
if k in self._entries:
return True
if not portable_data_hash_pattern.match(k) and (
self.pdh_only or not uuid_pattern.match(k)):
return False
try:
e = None
if group_uuid_pattern.match(k):
project = self.api.groups().list(
filters=[['group_class', '=', 'project'], ["uuid", "=", k]
]).execute(num_retries=self.num_retries)
if project[u'items_available'] == 0:
return False
e = self.inodes.add_entry(
ProjectDirectory(self.inode, self.inodes, self.api,
self.num_retries, project[u'items'][0]))
else:
import sys
e = self.inodes.add_entry(
CollectionDirectory(self.inode, self.inodes, self.api,
self.num_retries, k))
if e.update():
if k not in self._entries:
self._entries[k] = e
else:
self.inodes.del_entry(e)
return True
else:
self.inodes.invalidate_entry(self, k)
self.inodes.del_entry(e)
return False
except Exception as ex:
_logger.exception("arv-mount lookup '%s':", k)
if e is not None:
self.inodes.del_entry(e)
return False
def __contains__(self, k):
if k in self._entries:
return True
if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)):
return False
try:
e = None
if group_uuid_pattern.match(k):
project = self.api.groups().list(
filters=[['group_class', '=', 'project'], ["uuid", "=", k]]).execute(num_retries=self.num_retries)
if project[u'items_available'] == 0:
return False
e = self.inodes.add_entry(ProjectDirectory(
self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0]))
else:
e = self.inodes.add_entry(CollectionDirectory(
self.inode, self.inodes, self.api, self.num_retries, k))
if e.update():
if k not in self._entries:
self._entries[k] = e
else:
self.inodes.del_entry(e)
return True
else:
self.inodes.invalidate_entry(self, k)
self.inodes.del_entry(e)
return False
except Exception as ex:
_logger.exception("arv-mount lookup '%s':", k)
if e is not None:
self.inodes.del_entry(e)
return False
def update(self, to_record_version=None):
try:
if self.collection_record is not None and portable_data_hash_pattern.match(self.collection_locator):
return True
if self.collection_locator is None:
self.fresh()
return True
try:
with llfuse.lock_released:
self._updating_lock.acquire()
if not self.stale():
return
_logger.debug("Updating collection %s inode %s to record version %s", self.collection_locator, self.inode, to_record_version)
if self.collection is not None:
if self.collection.known_past_version(to_record_version):
_logger.debug("%s already processed %s", self.collection_locator, to_record_version)
else:
self.collection.update()
else:
if uuid_pattern.match(self.collection_locator):
coll_reader = arvados.collection.Collection(
self.collection_locator, self.api, self.api.keep,
num_retries=self.num_retries)
else:
coll_reader = arvados.collection.CollectionReader(
self.collection_locator, self.api, self.api.keep,
num_retries=self.num_retries)
new_collection_record = coll_reader.api_response() or {}
# If the Collection only exists in Keep, there will be no API
# response. Fill in the fields we need.
if 'uuid' not in new_collection_record:
new_collection_record['uuid'] = self.collection_locator
if "portable_data_hash" not in new_collection_record:
new_collection_record["portable_data_hash"] = new_collection_record["uuid"]
if 'manifest_text' not in new_collection_record:
new_collection_record['manifest_text'] = coll_reader.manifest_text()
if self.collection_record is None or self.collection_record["portable_data_hash"] != new_collection_record.get("portable_data_hash"):
self.new_collection(new_collection_record, coll_reader)
self._manifest_size = len(coll_reader.manifest_text())
_logger.debug("%s manifest_size %i", self, self._manifest_size)
# end with llfuse.lock_released, re-acquire lock
self.fresh()
return True
finally:
self._updating_lock.release()
except arvados.errors.NotFoundError as e:
_logger.error("Error fetching collection '%s': %s", self.collection_locator, e)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
if self.collection_record is not None and "manifest_text" in self.collection_record:
_logger.warning("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
except Exception:
_logger.exception("arv-mount %s: error", self.collection_locator)
if self.collection_record is not None and "manifest_text" in self.collection_record:
_logger.error("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
self.invalidate()
return False
def update(self, to_record_version=None):
try:
if self.collection_record is not None and portable_data_hash_pattern.match(
self.collection_locator):
return True
if self.collection_locator is None:
self.fresh()
return True
try:
with llfuse.lock_released:
self._updating_lock.acquire()
if not self.stale():
return
_logger.debug(
"Updating collection %s inode %s to record version %s",
self.collection_locator, self.inode, to_record_version)
if self.collection is not None:
if self.collection.known_past_version(
to_record_version):
_logger.debug("%s already processed %s",
self.collection_locator,
to_record_version)
else:
self.collection.update()
else:
if uuid_pattern.match(self.collection_locator):
coll_reader = arvados.collection.Collection(
self.collection_locator,
self.api,
self.api.keep,
num_retries=self.num_retries)
else:
coll_reader = arvados.collection.CollectionReader(
self.collection_locator,
self.api,
self.api.keep,
num_retries=self.num_retries)
new_collection_record = coll_reader.api_response(
) or {}
# If the Collection only exists in Keep, there will be no API
# response. Fill in the fields we need.
if 'uuid' not in new_collection_record:
new_collection_record[
'uuid'] = self.collection_locator
if "portable_data_hash" not in new_collection_record:
new_collection_record[
"portable_data_hash"] = new_collection_record[
"uuid"]
if 'manifest_text' not in new_collection_record:
new_collection_record[
'manifest_text'] = coll_reader.manifest_text()
if self.collection_record is None or self.collection_record[
"portable_data_hash"] != new_collection_record.get(
"portable_data_hash"):
self.new_collection(new_collection_record,
coll_reader)
self._manifest_size = len(coll_reader.manifest_text())
_logger.debug("%s manifest_size %i", self,
self._manifest_size)
# end with llfuse.lock_released, re-acquire lock
self.fresh()
return True
finally:
self._updating_lock.release()
except arvados.errors.NotFoundError as e:
_logger.error("Error fetching collection '%s': %s",
self.collection_locator, e)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator,
detail)
if self.collection_record is not None and "manifest_text" in self.collection_record:
_logger.warning("arv-mount manifest_text is: %s",
self.collection_record["manifest_text"])
except Exception:
_logger.exception("arv-mount %s: error", self.collection_locator)
if self.collection_record is not None and "manifest_text" in self.collection_record:
_logger.error("arv-mount manifest_text is: %s",
self.collection_record["manifest_text"])
self.invalidate()
return False
