def _read_file(self, os_path, format):
"""Read a non-notebook file.
os_path: The path to be read.
format:
If 'text', the contents will be decoded as UTF-8.
If 'base64', the raw bytes contents will be encoded as base64.
If not specified, try to decode as UTF-8, and fall back to base64
"""
if os_path.endswith('.ipynb'):
meta = self.omega.jobs.metadata(os_path)
else:
meta = self.omega.datasets.metadata(os_path)
if meta is None or meta.gridfile is None:
raise HTTPError(400, "Cannot read non-file %s" % os_path)
if meta.gridfile:
bcontent = meta.gridfile.read()
meta.gridfile.close()
if format is None or format == 'text':
# Try to interpret as unicode if format is unknown or if unicode
# was explicitly requested.
try:
return bcontent.decode('utf8'), 'text'
except UnicodeError:
if format == 'text':
raise HTTPError(
400,
"%s is not UTF-8 encoded" % os_path,
reason='bad format',
)
return encodebytes(bcontent).decode('ascii'), 'base64'
def get(self, path, command):
notebook = self.contents_manager.get(path, content=True)
inspector = self._get_inspector(notebook)
if command == 'available':
self.set_status(201)
self.finish()
elif command == 'variables':
cell_index = self._int_argument('cellIdx')
if cell_index is None:
raise HTTPError(400, 'invalid_cell_idx')
variables = inspector.get_variables(notebook, cell_index)
self.finish(json.dumps({'data': variables}))
elif command == "inspector.html":
cell_index = self._int_argument('cellIdx')
if cell_index is None:
raise HTTPError(400, 'invalid_cell_idx')
self.render(
"inspector.html",
**{'variables': inspector.get_variables(notebook, cell_index)})
else:
raise HTTPError(400, 'no_cmd')
def post(self, image_name, command):
if command == 'run':
return self._run(image_name)
elif command == 'stop':
return self._stop(image_name)
raise HTTPError(400, 'no_cmd')
def _dir_model(self, path, content=True):
"""
Build a model to return all of the files in gridfs
if content is requested, will include a listing of the directory
"""
# this looks like a seemingly simple task, it's carefully crafted
path = unquote(path).strip('/')
model = self._base_model(path, kind='directory')
model['format'] = 'json'
contents = model['content']
# get existing entries from a pattern that matches either
# top-level files: ([\w -]+\.[\w -]*)
# directories (files in): ([\w ]+/([\w ]+\.[\w]*))
# does not work:
# pattern = r'([\w ]+/_placeholder\.[\w]*)|([\w ]+\.[\w]*)$'
# it is too restrictive as entries can be generated without a placeholder
# so we get all, which may include sub/sub/file
# and we need to include them because we need to find sub/sub directories
# note \w is any word character (letter, digit, underscore)
# \s is any white space
# \d is any digit
# :_ match literally
# [^\/] matches any character except /
#pattern = r'([\w\s\-.\d:()+]+\/)?([\w\s\-.\d:()+]+\.[\w]*)$'
pattern = r'([^\/]+\/)?([^\/]+\.[^\/]*)$'
# if we're looking in an existing directory, prepend that
if path:
pattern = r'{path}/{pattern}'.format(path=path, pattern=pattern)
pattern = r'^{}'.format(pattern)
entries = self.omega.jobs.list(regexp=pattern, raw=True, hidden=True, include_temp=True)
if path and not entries:
raise HTTPError(400, "Directory not found {}".format(path))
# by default assume the current path is listed already
directories = [path]
for meta in entries:
# get path of entry, e.g. sub/foo.ipynb => sub
entry_path = os.path.dirname(meta.name)
# if not part of listed directories yet, include
if entry_path not in directories:
entry = self._base_model(entry_path, kind='directory')
contents.append(entry)
directories.append(entry_path)
# ignore placeholder files
if meta.name.endswith(self._dir_placeholder):
continue
# only include files that are in the path we're listing
if entry_path != path:
continue
# include the actual file
try:
entry = self._notebook_model(meta.name, content=content, meta=meta)
except Exception as e:
msg = ('_dir_model error, cannot get {}, '
'removing from list, exception {}'.format(meta.name, str(e)))
self.log.warning(msg)
else:
contents.append(entry)
return model
def _save_file(self, os_path, content, format):
"""Save content of a generic file."""
if format not in {'text', 'base64'}:
raise HTTPError(
400,
"Must specify format of file contents as 'text' or 'base64'",
)
try:
if format == 'text':
bcontent = content.encode('utf8')
else:
b64_bytes = content.encode('ascii')
bcontent = decodebytes(b64_bytes)
except Exception as e:
raise HTTPError(400,
u'Encoding error saving %s: %s' % (os_path, e))
self.omega.datasets.put(BytesIO(bcontent), os_path)
def post(self, path):
notebook = self.contents_manager.get(path, content=True)
notebook_path = os.path.join(os.getcwd(), path)
notebook_name = "notebook.ipynb"
body = self.get_json_body()
image_name = body.get('imageName')
base_image = body.get('baseImage')
cell_index = int(body.get('cellIndex'))
variables = body.get('variables', {})
if image_name is None or base_image is None or cell_index is None:
raise HTTPError(400, 'abc')
requirements = body.get('environment', BASE_STRING)
# Create a temporary dir which will be our build context.
with tempfile.TemporaryDirectory() as tmpdir:
shutil.copyfile(notebook_path, tmpdir + "/" + notebook_name)
# Find the location of the FAIR-Cells module on disk
# So it can copy & install it in the container.
dirname = os.path.dirname(__file__)
nested_levels = len(__name__.split('.')) - 2
module_path = os.path.join(dirname + '/..' * nested_levels)
# Copy helper to build context.
shutil.copytree(module_path,
tmpdir + "/fair-cells/",
ignore=shutil.ignore_patterns(
'.ipynb_checkpoints', '__pycache__'))
with open(tmpdir + "/environment.yml", "a") as reqs:
reqs.write(requirements)
with open(tmpdir + "/nb_helper_config.json", "a") as cfg:
config = create_config(notebook_name, cell_index, variables)
cfg.write(config)
with open(tmpdir + "/.dockerignore", "a") as ignore:
ignore.write("**/backend\n")
ignore.write("**/frontend\n")
cc = ContainerCreator(tmpdir, image_name, base_image)
try:
_, log = cc.build_container(cc.get_dockerfile())
except docker.errors.BuildError as be:
log = be.build_log
logs = "".join([l['stream'] if 'stream' in l else '' for l in log])
self.finish(json.dumps({'logs': logs}))
def _run(self, image_name):
body = self.get_json_body()
port = self._int_body('port', 10000)
if port is None:
raise HTTPError(400, 'def')
cc = DockerService()
container = cc.run_container(port=port,image=image_name)
self.finish(json.dumps({
'data': container.status
}))
def _run(self, image_name):
body = self.get_json_body()
port = self._int_body('port', 10000)
if port is None:
raise HTTPError(400, 'def')
cc = ContainerCreator('.', image_name, None)
container = cc.run_container(port)
self.finish(json.dumps({
'data': container.status
}))
def _get_inspector(self, notebook):
kernel = notebook['content']['metadata']['kernelspec']['name']
try:
inspector_module = importlib.import_module(f'.inspection.{kernel}',
package="fair-cells")
inspector = inspector_module.inspector()
if inspector.available():
return inspector
except ModuleNotFoundError:
pass
raise HTTPError(501, 'inspector_unavailable')
def _notebook_model(self, path, content=True, meta=None):
"""
Build a notebook model
if content is requested, the notebook content will be populated
as a JSON structure (not double-serialized)
"""
path = unquote(path).strip('/')
model = self._base_model(path)
model['type'] = 'notebook'
# always add accurate created and modified
meta = meta or self.omega.jobs.metadata(path)
if meta is not None:
model['created'] = meta.created
model['last_modified'] = meta.modified
if content:
nb = self._read_notebook(path, as_version=4)
if nb is None:
raise HTTPError(400, "Cannot read non-file {}".format(path))
self.mark_trusted_cells(nb, path)
model['content'] = nb
model['format'] = 'json'
self.validate_notebook_model(model)
return model
def get(self, image_name, command):
if command == 'status':
return self._status(image_name)
raise HTTPError(400, 'no_cmd')
def post(self, path):
notebook = self.contents_manager.get(path, content=True)
notebook_path = os.path.join(os.getcwd(), path)
notebook_name = "notebook.ipynb"
body = self.get_json_body()
image_name = body.get('imageName')
base_image = body.get('baseImage')
cell_index = int(body.get('cellIndex'))
variables = body.get('variables', {})
logging.info("image_name: " + str(image_name))
logging.info("base_image: " + str(base_image))
logging.info("cell_index: " + str(cell_index))
logging.info("variables: " + str(variables))
if image_name is None or base_image is None or cell_index is None:
raise HTTPError(400, 'abc')
requirements = body.get('environment', BASE_STRING)
# Create a temporary dir which will be our build context.
with tempfile.TemporaryDirectory() as tmpdir:
shutil.copyfile(notebook_path, tmpdir + "/" + notebook_name)
# Find the location of the FAIR-Cells module on disk
# So it can copy & install it in the container.
dirname = os.path.dirname(__file__)
nested_levels = len(__name__.split('.')) - 2
module_path = os.path.join(dirname + '/..' * nested_levels)
# Copy helper to build context.
shutil.copytree(module_path,
tmpdir + "/fair-cells/",
ignore=shutil.ignore_patterns(
'.ipynb_checkpoints', '__pycache__'))
with open(tmpdir + "/environment.yml", "a") as reqs:
reqs.write(requirements)
with open(tmpdir + "/environment.yml") as file:
environment = yaml.load(file, Loader=yaml.FullLoader)
lines = ''
for requ in environment['dependencies'][0]['pip']:
lines += requ + '\n'
with open(tmpdir + '/requirements.txt', 'w') as f:
f.write(lines)
f.close()
with open(tmpdir + "/nb_helper_config.json", "a") as cfg:
config = create_config(notebook_name, cell_index, variables)
cfg.write(config)
with open(tmpdir + "/.dockerignore", "a") as ignore:
ignore.write("**/backend\n")
ignore.write("**/frontend\n")
cc = DockerService()
docker_file = cc.get_dockerfile(base_image)
self.finish(json.dumps({'dockerFile': docker_file}))
