def _patched_project_import_modules(project_path, ignores):
"""
copied form pigar req.project_import_modules
patching, os.getcwd() is incorrectly used
"""
from pigar.modules import ImportedModules
from pigar.reqs import file_import_modules
modules = ImportedModules()
try_imports = set()
local_mods = list()
ignore_paths = collections.defaultdict(set)
if not ignores:
ignore_paths[project_path].add('.git')
else:
for path in ignores:
parent_dir = os.path.dirname(path)
ignore_paths[parent_dir].add(os.path.basename(path))
if os.path.isfile(project_path):
fake_path = Path(project_path).name
with open(project_path, 'rb') as f:
fmodules, try_ipts = file_import_modules(fake_path, f.read())
modules |= fmodules
try_imports |= try_ipts
else:
cur_dir = project_path # os.getcwd()
for dirpath, dirnames, files in os.walk(project_path,
followlinks=True):
if dirpath in ignore_paths:
dirnames[:] = [
d for d in dirnames if d not in ignore_paths[dirpath]
]
py_files = list()
for fn in files:
# C extension.
if fn.endswith('.so'):
local_mods.append(fn[:-3])
# Normal Python file.
if fn.endswith('.py'):
local_mods.append(fn[:-3])
py_files.append(fn)
if '__init__.py' in files:
local_mods.append(os.path.basename(dirpath))
for file in py_files:
fpath = os.path.join(dirpath, file)
fake_path = fpath.split(cur_dir)[1][1:]
with open(fpath, 'rb') as f:
fmodules, try_ipts = file_import_modules(
fake_path, f.read())
modules |= fmodules
try_imports |= try_ipts
return ScriptRequirements.add_trains_used_packages(
modules), try_imports, local_mods
def _patched_project_import_modules(project_path, ignores):
"""
copied form pigar req.project_import_modules
patching, os.getcwd() is incorrectly used
"""
from pigar.modules import ImportedModules
from pigar.reqs import file_import_modules
modules = ImportedModules()
try_imports = set()
local_mods = list()
ignore_paths = collections.defaultdict(set)
if not ignores:
ignore_paths[project_path].add('.git')
else:
for path in ignores:
parent_dir = os.path.dirname(path)
ignore_paths[parent_dir].add(os.path.basename(path))
if os.path.isfile(project_path):
fake_path = Path(project_path).name
with open(project_path, 'rb') as f:
fmodules, try_ipts = file_import_modules(fake_path, f.read())
modules |= fmodules
try_imports |= try_ipts
else:
cur_dir = project_path # os.getcwd()
for dirpath, dirnames, files in os.walk(project_path,
followlinks=True):
if dirpath in ignore_paths:
dirnames[:] = [
d for d in dirnames if d not in ignore_paths[dirpath]
]
py_files = list()
for fn in files:
# C extension.
if fn.endswith('.so'):
local_mods.append(fn[:-3])
# Normal Python file.
if fn.endswith('.py'):
local_mods.append(fn[:-3])
py_files.append(fn)
if '__init__.py' in files:
local_mods.append(os.path.basename(dirpath))
for file in py_files:
fpath = os.path.join(dirpath, file)
fake_path = fpath.split(cur_dir)[1][1:]
with open(fpath, 'rb') as f:
fmodules, try_ipts = file_import_modules(
fake_path, f.read())
modules |= fmodules
try_imports |= try_ipts
# hack: forcefully insert storage modules if we have them
# noinspection PyBroadException
try:
import boto3
modules.add('boto3', 'trains.storage', 0)
except Exception:
pass
# noinspection PyBroadException
try:
from google.cloud import storage
modules.add('google_cloud_storage', 'trains.storage', 0)
except Exception:
pass
# noinspection PyBroadException
try:
from azure.storage.blob import ContentSettings
modules.add('azure_storage_blob', 'trains.storage', 0)
except Exception:
pass
return modules, try_imports, local_mods
def _daemon(cls, jupyter_notebook_filename):
from trains import Task
# load jupyter notebook package
# noinspection PyBroadException
try:
from nbconvert.exporters.script import ScriptExporter
_script_exporter = ScriptExporter()
except Exception:
return
# load pigar
# noinspection PyBroadException
try:
from pigar.reqs import get_installed_pkgs_detail, file_import_modules
from pigar.modules import ReqsModules
from pigar.log import logger
logger.setLevel(logging.WARNING)
except Exception:
file_import_modules = None
# load IPython
# noinspection PyBroadException
try:
from IPython import get_ipython
except Exception:
# should not happen
get_ipython = None
# setup local notebook files
if jupyter_notebook_filename:
notebook = Path(jupyter_notebook_filename)
local_jupyter_filename = jupyter_notebook_filename
else:
notebook = None
fd, local_jupyter_filename = mkstemp(suffix='.ipynb')
os.close(fd)
last_update_ts = None
counter = 0
prev_script_hash = None
# main observer loop, check if we need to exit
while not cls._exit_event.wait(timeout=0.):
# wait for timeout or sync event
cls._sync_event.wait(cls._sample_frequency if counter else cls.
_first_sample_frequency)
cls._sync_event.clear()
counter += 1
# noinspection PyBroadException
try:
# if there is no task connected, do nothing
task = Task.current_task()
if not task:
continue
# if we have a local file:
if notebook:
if not notebook.exists():
continue
# check if notebook changed
if last_update_ts is not None and notebook.stat(
).st_mtime - last_update_ts <= 0:
continue
last_update_ts = notebook.stat().st_mtime
else:
# serialize notebook to a temp file
# noinspection PyBroadException
try:
get_ipython().run_line_magic('notebook',
local_jupyter_filename)
except Exception as ex:
continue
# get notebook python script
script_code, resources = _script_exporter.from_filename(
local_jupyter_filename)
current_script_hash = hash(script_code)
if prev_script_hash and prev_script_hash == current_script_hash:
continue
requirements_txt = ''
conda_requirements = ''
# parse jupyter python script and prepare pip requirements (pigar)
# if backend supports requirements
if file_import_modules and Session.check_min_api_version(
'2.2'):
fmodules, _ = file_import_modules(notebook.parts[-1],
script_code)
fmodules = ScriptRequirements.add_trains_used_packages(
fmodules)
installed_pkgs = get_installed_pkgs_detail()
reqs = ReqsModules()
for name in fmodules:
if name in installed_pkgs:
pkg_name, version = installed_pkgs[name]
reqs.add(pkg_name, version, fmodules[name])
requirements_txt, conda_requirements = ScriptRequirements.create_requirements_txt(
reqs)
# update script
prev_script_hash = current_script_hash
data_script = task.data.script
data_script.diff = script_code
data_script.requirements = {
'pip': requirements_txt,
'conda': conda_requirements
}
task._update_script(script=data_script)
# update requirements
task._update_requirements(requirements=requirements_txt)
except Exception:
pass
def _daemon(cls, jupyter_notebook_filename):
from trains import Task
# load jupyter notebook package
# noinspection PyBroadException
try:
from nbconvert.exporters.script import ScriptExporter
_script_exporter = ScriptExporter()
except Exception:
return
# load pigar
# noinspection PyBroadException
try:
from pigar.reqs import get_installed_pkgs_detail, file_import_modules
from pigar.modules import ReqsModules
from pigar.log import logger
logger.setLevel(logging.WARNING)
except Exception:
file_import_modules = None
# main observer loop
notebook = Path(jupyter_notebook_filename)
last_update_ts = None
counter = 0
prev_script_hash = None
while True:
if cls._exit_event.wait(cls._sample_frequency if counter else cls.
_first_sample_frequency):
return
counter += 1
# noinspection PyBroadException
try:
if not notebook.exists():
continue
# check if notebook changed
if last_update_ts is not None and notebook.stat(
).st_mtime - last_update_ts <= 0:
continue
last_update_ts = notebook.stat().st_mtime
task = Task.current_task()
if not task:
continue
# get notebook python script
script_code, resources = _script_exporter.from_filename(
jupyter_notebook_filename)
current_script_hash = hash(script_code)
if prev_script_hash and prev_script_hash == current_script_hash:
continue
requirements_txt = ''
# parse jupyter python script and prepare pip requirements (pigar)
# if backend supports requirements
if file_import_modules and Session.api_version > '2.1':
fmodules, _ = file_import_modules(notebook.parts[-1],
script_code)
installed_pkgs = get_installed_pkgs_detail()
reqs = ReqsModules()
for name in fmodules:
if name in installed_pkgs:
pkg_name, version = installed_pkgs[name]
reqs.add(pkg_name, version, fmodules[name])
requirements_txt = ScriptRequirements.create_requirements_txt(
reqs)
# update script
prev_script_hash = current_script_hash
data_script = task.data.script
data_script.diff = script_code
data_script.requirements = {'pip': requirements_txt}
task._update_script(script=data_script)
# update requirements
if requirements_txt:
task._update_requirements(requirements=requirements_txt)
except Exception:
pass