async def save_one(self, src, dest):
""" use the contents manager to write a single file/folder
"""
# pylint: disable=broad-except
stat = src.stat()
is_dir = src.is_dir()
model = dict(
name=src.name,
type="directory" if is_dir else "file",
path=dest,
last_modified=tz.utcfromtimestamp(stat.st_mtime),
created=tz.utcfromtimestamp(stat.st_ctime),
content=None
if is_dir else base64.b64encode(src.read_bytes()).decode("utf-8"),
format=None if is_dir else "base64",
mimetype=None,
size=stat.st_size,
)
allow_hidden = None
if hasattr(self.contents_manager, "allow_hidden"):
allow_hidden = self.contents_manager.allow_hidden
self.contents_manager.allow_hidden = True
try:
await maybe_future(self.contents_manager.save(model, dest))
except Exception as err:
self.log.error(f"Couldn't save {dest}: {err}")
finally:
if allow_hidden is not None:
self.contents_manager.allow_hidden = allow_hidden
def _base_model(self, path):
"""Build the common base of a contents model"""
os_path = self._get_os_path(path)
info = os.lstat(os_path)
try:
last_modified = tz.utcfromtimestamp(info.st_mtime)
except (ValueError, OSError):
# Files can rarely have an invalid timestamp
# https://github.com/jupyter/notebook/issues/2539
# https://github.com/jupyter/notebook/issues/2757
# Use the Unix epoch as a fallback so we don't crash.
self.log.warning('Invalid mtime %s for %s', info.st_mtime, os_path)
last_modified = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC)
try:
created = tz.utcfromtimestamp(info.st_ctime)
except (ValueError, OSError): # See above
self.log.warning('Invalid ctime %s for %s', info.st_ctime, os_path)
created = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC)
# Create the base model.
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
model['mimetype'] = None
try:
model['writable'] = os.access(os_path, os.W_OK)
except OSError:
self.log.error("Failed to check write permissions on %s", os_path)
model['writable'] = False
return model
def _base_model(self, path):
"""Build the common base of a contents model"""
os_path = self._get_os_path(path)
info = os.lstat(os_path)
try:
last_modified = tz.utcfromtimestamp(info.st_mtime)
except (ValueError, OSError):
# Files can rarely have an invalid timestamp
# https://github.com/jupyter/notebook/issues/2539
# https://github.com/jupyter/notebook/issues/2757
# Use the Unix epoch as a fallback so we don't crash.
self.log.warning('Invalid mtime %s for %s', info.st_mtime, os_path)
last_modified = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC)
try:
created = tz.utcfromtimestamp(info.st_ctime)
except (ValueError, OSError): # See above
self.log.warning('Invalid ctime %s for %s', info.st_ctime, os_path)
created = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC)
# Create the base model.
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
model['mimetype'] = None
try:
model['writable'] = os.access(os_path, os.W_OK)
except OSError:
self.log.error("Failed to check write permissions on %s", os_path)
model['writable'] = False
return model
def _base_model(self, path):
"""Build the common base of a hdfscontents model"""
hdfs_path = to_os_path(path, self.root_dir)
info = self.hdfs.get_path_info(hdfs_path)
last_modified = tz.utcfromtimestamp(info.get(u'last_mod'))
# TODO: don't have time created! now storing last accessed instead
created = tz.utcfromtimestamp(info.get(u'last_access'))
# Create the base model.
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
model['mimetype'] = None
# TODO: Now just checking if user have write permission in HDFS. Need to cover all cases and check the user & group?
try:
model['writable'] = (info.get(u'permissions') & 0o0200) > 0
except OSError:
self.log.error("Failed to check write permissions on %s",
hdfs_path)
model['writable'] = False
return model
def checkpoint_model(self, checkpoint_id, os_path):
"""construct the info dict for a given checkpoint"""
stats = os.stat(os_path)
last_modified = tz.utcfromtimestamp(stats.st_mtime)
info = dict(
id=checkpoint_id,
last_modified=last_modified,
)
return info
def checkpoint_model(self, checkpoint_id, os_path):
"""construct the info dict for a given checkpoint"""
stats = os.stat(os_path)
last_modified = tz.utcfromtimestamp(stats.st_mtime)
info = dict(
id=checkpoint_id,
last_modified=last_modified,
)
return info
def checkpoint_model(self, checkpoint_id, hdfs_path):
"""construct the info dict for a given checkpoint"""
stats = self.hdfs.info(hdfs_path)
last_modified = tz.utcfromtimestamp(stats.get(u'last_mod'))
info = dict(
id=checkpoint_id,
last_modified=last_modified,
)
return info
def checkpoint_model(self, checkpoint_id, path):
"""construct the info dict for a given checkpoint"""
fs_path = self._to_fs_path(path)
stats = self.fs.info(fs_path)
last_modified = tz.utcfromtimestamp(stats[u'last_modified'])
info = dict(
id=checkpoint_id,
last_modified=last_modified,
)
return info
def _base_model(self, path):
"""Build the common base of a contents model"""
os_path = self._get_os_path(path)
info = os.lstat(os_path)
last_modified = tz.utcfromtimestamp(info.st_mtime)
created = tz.utcfromtimestamp(info.st_ctime)
# Create the base model.
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
model['mimetype'] = None
try:
model['writable'] = os.access(os_path, os.W_OK)
except OSError:
self.log.error("Failed to check write permissions on %s", os_path)
model['writable'] = False
return model
def _base_model(self, path):
"""Build the common base of a contents model"""
os_path = self._get_os_path(path)
info = os.stat(os_path)
last_modified = tz.utcfromtimestamp(info.st_mtime)
created = tz.utcfromtimestamp(info.st_ctime)
# Create the base model.
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
model['mimetype'] = None
try:
model['writable'] = os.access(os_path, os.W_OK)
except OSError:
self.log.error("Failed to check write permissions on %s", os_path)
model['writable'] = False
return model
def __base_model(self, path, type):
"""Build the common base of a model"""
self._ensure_path_is_valid(path, type, enforce_exists=True)
fs_path = self._to_fs_path(path)
info = self.fs.info(fs_path)
last_modified = tz.utcfromtimestamp(info[u'last_modified'])
created = tz.utcfromtimestamp(info[u'last_accessed'])
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
model['mimetype'] = None
model['type'] = info[u'kind']
try:
model['writable'] = (info[u'permissions'] & 0o0200) > 0
except OSError:
self.log.error(f"Failed to check write permissions on {path}")
model['writable'] = False
return model
def get(self):
# if started was missing, use unix epoch
started = self.settings.get('started', utcfromtimestamp(0))
started = isoformat(started)
kernels = yield maybe_future(self.kernel_manager.list_kernels())
total_connections = sum(k['connections'] for k in kernels)
last_activity = isoformat(self.application.last_activity())
model = {
'started': started,
'last_activity': last_activity,
'kernels': len(kernels),
'connections': total_connections,
}
self.finish(json.dumps(model, sort_keys=True))
def get(self):
# if started was missing, use unix epoch
started = self.settings.get('started', utcfromtimestamp(0))
started = isoformat(started)
kernels = yield gen.maybe_future(self.kernel_manager.list_kernels())
total_connections = sum(k['connections'] for k in kernels)
last_activity = isoformat(self.application.last_activity())
model = {
'started': started,
'last_activity': last_activity,
'kernels': len(kernels),
'connections': total_connections,
}
self.finish(json.dumps(model, sort_keys=True))
def get(self):
# if started was missing, use unix epoch
started = self.settings.get('started', utcfromtimestamp(0))
# if we've never seen API activity, use started date
api_last_activity = self.settings.get('api_last_activity', started)
started = isoformat(started)
api_last_activity = isoformat(api_last_activity)
kernels = yield gen.maybe_future(self.kernel_manager.list_kernels())
total_connections = sum(k['connections'] for k in kernels)
last_activity = max(chain([api_last_activity], [k['last_activity'] for k in kernels]))
model = {
'started': started,
'last_activity': last_activity,
'kernels': len(kernels),
'connections': total_connections,
}
self.finish(json.dumps(model, sort_keys=True))
