def make_symlink(path, real_path):
real_path = Path(real_path)
real_path.parent.mkdir(parents=True, exist_ok=True)
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists():
os.rename(str(path), str(path.with_suffix('.bak')))
os.symlink(str(real_path), str(path))
def find_spec(self, name, path, target=None):
path = Path('./' + name.replace('.', r'/'))
if path.with_suffix('.c').exists():
return ModuleSpec(name,
CintegrateLoader(path, type='c'),
is_package=True)
if path.with_suffix('.cpp').exists():
return ModuleSpec(name,
CintegrateLoader(path, type='c++'),
is_package=True)
if path.with_suffix('.so').exists():
return ModuleSpec(name,
CintegrateLoader(path, type='so'),
is_package=True)
if path.is_dir():
return ModuleSpec(name, CintegrateLoader(path), is_package=True)
return None
def check_text_files(obtained_fn, expected_fn, fix_callback=lambda x: x, encoding=None):
"""
Compare two files contents. If the files differ, show the diff and write a nice HTML
diff file into the data directory.
:param Path obtained_fn: path to obtained file during current testing.
:param Path expected_fn: path to the expected file, obtained from previous testing.
:param str encoding: encoding used to open the files.
:param callable fix_callback:
A callback to "fix" the contents of the obtained (first) file.
This callback receives a list of strings (lines) and must also return a list of lines,
changed as needed.
The resulting lines will be used to compare with the contents of expected_fn.
"""
__tracebackhide__ = True
obtained_fn = Path(obtained_fn)
expected_fn = Path(expected_fn)
obtained_lines = fix_callback(obtained_fn.read_text(encoding=encoding).splitlines())
expected_lines = expected_fn.read_text(encoding=encoding).splitlines()
if obtained_lines != expected_lines:
diff_lines = list(
difflib.unified_diff(expected_lines, obtained_lines, lineterm="")
)
if len(diff_lines) <= 500:
html_fn = obtained_fn.with_suffix(".diff.html")
try:
differ = difflib.HtmlDiff()
html_diff = differ.make_file(
fromlines=expected_lines,
fromdesc=expected_fn,
tolines=obtained_lines,
todesc=obtained_fn,
)
except Exception as e:
html_fn = "(failed to generate html diff: %s)" % e
else:
html_fn.write_text(html_diff, encoding="UTF-8")
diff = ["FILES DIFFER:", str(expected_fn), str(obtained_fn)]
diff += ["HTML DIFF: %s" % html_fn]
diff += diff_lines
raise AssertionError("\n".join(diff))
else:
# difflib has exponential scaling and for thousands of lines it starts to take minutes to render
# the HTML diff.
msg = [
"Files are different, but diff is too big ({} lines)".format(
len(diff_lines)
),
"- obtained: {}".format(obtained_fn),
"- expected: {}".format(expected_fn),
]
raise AssertionError("\n".join(msg))
def check_text_files(obtained_fn, expected_fn, fix_callback=lambda x: x, encoding=None):
"""
Compare two files contents. If the files differ, show the diff and write a nice HTML
diff file into the data directory.
:param Path obtained_fn: path to obtained file during current testing.
:param Path expected_fn: path to the expected file, obtained from previous testing.
:param str encoding: encoding used to open the files.
:param callable fix_callback:
A callback to "fix" the contents of the obtained (first) file.
This callback receives a list of strings (lines) and must also return a list of lines,
changed as needed.
The resulting lines will be used to compare with the contents of expected_fn.
"""
__tracebackhide__ = True
obtained_fn = Path(obtained_fn)
expected_fn = Path(expected_fn)
obtained_lines = fix_callback(obtained_fn.read_text(encoding=encoding).splitlines())
expected_lines = expected_fn.read_text(encoding=encoding).splitlines()
if obtained_lines != expected_lines:
diff_lines = list(difflib.unified_diff(expected_lines, obtained_lines))
if len(diff_lines) <= 500:
html_fn = obtained_fn.with_suffix(".diff.html")
try:
differ = difflib.HtmlDiff()
html_diff = differ.make_file(
fromlines=expected_lines,
fromdesc=expected_fn,
tolines=obtained_lines,
todesc=obtained_fn,
)
except Exception as e:
html_fn = "(failed to generate html diff: %s)" % e
else:
html_fn.write_text(html_diff, encoding="UTF-8")
diff = ["FILES DIFFER:", str(expected_fn), str(obtained_fn)]
diff += ["HTML DIFF: %s" % html_fn]
diff += diff_lines
raise AssertionError("\n".join(diff))
else:
# difflib has exponential scaling and for thousands of lines it starts to take minutes to render
# the HTML diff.
msg = [
"Files are different, but diff is too big (%s lines)" % (len(diff_lines),),
"- obtained: %s" % (obtained_fn,),
"- expected: %s" % (expected_fn,),
]
raise AssertionError("\n".join(msg))
def which(
program,
find_all=False): # pragma: no cover -- TODO separate module for utils
EXECUTABLE_EXT = ['']
if "PATHEXT" in os.environ:
EXECUTABLE_EXT += os.environ["PATHEXT"].split(os.pathsep)
result = []
for path in [''] + os.environ["PATH"].split(os.pathsep):
path = path.strip('"')
exe_file = Path(path) / program
for ext in EXECUTABLE_EXT:
fullpath = exe_file.with_suffix(
ext) if ext and ext != '.' else exe_file
if fullpath.is_file() and os.access(str(fullpath), os.X_OK):
if find_all:
result.append(fullpath)
else:
return fullpath
return result or None
def modified_config(request):
link_gmx_mpi = request.config.getoption('link_gmx_mpi')
tools = str(Path('~/gmx_mpi').expanduser()) if link_gmx_mpi else ''
append_suffix = 'yes' if request.config.getoption('append_suffix') else 'no'
return tools, append_suffix, Path
path_config = Path('~/.gromacswrapper.cfg').expanduser()
gw_config = ConfigParser()
if path_config.exists():
gw_config.read(str(path_config.resolve()))
config_existed = True
else:
gw_config.read('gromacs/templates/gromacswrapper.cfg')
config_existed = False
config_backup = path_config.with_suffix('.bak')
def pytest_configure(config):
link_gmx_mpi = config.getoption('link_gmx_mpi')
append_suffix = 'yes' if config.getoption('append_suffix') else 'no'
if config_existed:
shutil.copy(str(path_config), str(config_backup))
tools = gmx_mpi_linked(link_gmx_mpi)
gw_config.set('Gromacs', 'tools', tools)
gw_config.set('Gromacs', 'append_suffix', append_suffix)
with open(str(path_config), 'w') as config_file:
gw_config.write(config_file)
def pytest_unconfigure(config):
parser.add_argument("-o",
"--out_files",
action="store",
dest="outFiles",
required=True,
help="output file or directory")
args = parser.parse_args()
inPath = Path(args.inFiles)
outPath = Path(args.outFiles)
# Case 1: Convert single file.
if (inPath.is_file()):
inSuff = inPath.suffix
inStem = inPath.stem
outDir = outPath.parent
outDir.mkdir(parents=True, exist_ok=True)
if (inSuff != tsurfSuffix):
msg = 'Only tsurf (*.ts) files are allowed as input.'
raise ValueError(msg)
outPath = outPath.with_suffix(vtkSuffix)
convertFile(inPath, outPath)
# Case 2: Convert directory.
elif (inPath.is_dir()):
convertDir(inPath, outPath)
# Case 3: Give up.
else:
msg = 'Unable to find %s.' % inPath
raise ValueError(msg)
def _get_jupyter_notebook_filename(cls):
# check if we are running in vscode, we have the jupyter notebook defined:
if 'IPython' in sys.modules:
# noinspection PyBroadException
try:
from IPython import get_ipython # noqa
ip = get_ipython()
# vscode-jupyter PR #8531 added this variable
local_ipynb_file = ip.__dict__.get('user_ns', {}).get('__vsc_ipynb_file__') if ip else None
if local_ipynb_file:
# now replace the .ipynb with .py
# we assume we will have that file available for monitoring
local_ipynb_file = Path(local_ipynb_file)
script_entry_point = local_ipynb_file.with_suffix('.py').as_posix()
# install the post store hook,
# notice that if we do not have a local file we serialize/write every time the entire notebook
cls._jupyter_install_post_store_hook(local_ipynb_file.as_posix(), log_history=False)
return script_entry_point
except Exception:
pass
if not (sys.argv[0].endswith(os.path.sep + 'ipykernel_launcher.py') or
sys.argv[0].endswith(os.path.join(os.path.sep, 'ipykernel', '__main__.py'))) \
or len(sys.argv) < 3 or not sys.argv[2].endswith('.json'):
return None
server_info = None
# we can safely assume that we can import the notebook package here
# noinspection PyBroadException
try:
# noinspection PyPackageRequirements
from notebook.notebookapp import list_running_servers
import requests
current_kernel = sys.argv[2].split(os.path.sep)[-1].replace('kernel-', '').replace('.json', '')
# noinspection PyBroadException
try:
server_info = next(list_running_servers())
except Exception:
# on some jupyter notebook versions this function can crash on parsing the json file,
# we will parse it manually here
# noinspection PyPackageRequirements
import ipykernel
from glob import glob
import json
for f in glob(os.path.join(os.path.dirname(ipykernel.get_connection_file()), '??server-*.json')):
# noinspection PyBroadException
try:
with open(f, 'r') as json_data:
server_info = json.load(json_data)
except Exception:
server_info = None
if server_info:
break
cookies = None
password = None
if server_info and server_info.get('password'):
# we need to get the password
from ....config import config
password = config.get('development.jupyter_server_password', '')
if not password:
cls._get_logger().warning(
'Password protected Jupyter Notebook server was found! '
'Add `sdk.development.jupyter_server_password=<jupyter_password>` to ~/clearml.conf')
return os.path.join(os.getcwd(), 'error_notebook_not_found.py')
r = requests.get(url=server_info['url'] + 'login')
cookies = {'_xsrf': r.cookies.get('_xsrf', '')}
r = requests.post(server_info['url'] + 'login?next', cookies=cookies,
data={'_xsrf': cookies['_xsrf'], 'password': password})
cookies.update(r.cookies)
auth_token = server_info.get('token') or os.getenv('JUPYTERHUB_API_TOKEN') or ''
try:
r = requests.get(
url=server_info['url'] + 'api/sessions', cookies=cookies,
headers={'Authorization': 'token {}'.format(auth_token), })
except requests.exceptions.SSLError:
# disable SSL check warning
from urllib3.exceptions import InsecureRequestWarning
# noinspection PyUnresolvedReferences
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
# fire request
r = requests.get(
url=server_info['url'] + 'api/sessions', cookies=cookies,
headers={'Authorization': 'token {}'.format(auth_token), }, verify=False)
# enable SSL check warning
import warnings
warnings.simplefilter('default', InsecureRequestWarning)
# send request to the jupyter server
try:
r.raise_for_status()
except Exception as ex:
cls._get_logger().warning('Failed accessing the jupyter server{}: {}'.format(
' [password={}]'.format(password) if server_info.get('password') else '', ex))
return os.path.join(os.getcwd(), 'error_notebook_not_found.py')
notebooks = r.json()
cur_notebook = None
for n in notebooks:
if n['kernel']['id'] == current_kernel:
cur_notebook = n
break
notebook_path = cur_notebook['notebook'].get('path', '')
notebook_name = cur_notebook['notebook'].get('name', '')
is_google_colab = False
# check if this is google.colab, then there is no local file
# noinspection PyBroadException
try:
# noinspection PyPackageRequirements
from IPython import get_ipython
if get_ipython() and 'google.colab' in get_ipython().extension_manager.loaded:
is_google_colab = True
except Exception:
pass
if is_google_colab:
script_entry_point = str(notebook_name or 'notebook').replace(
'>', '_').replace('<', '_').replace('.ipynb', '.py')
if not script_entry_point.lower().endswith('.py'):
script_entry_point += '.py'
local_ipynb_file = None
else:
# always slash, because this is from uri (so never backslash not even on windows)
entry_point_filename = notebook_path.split('/')[-1]
# now we should try to find the actual file
entry_point = (Path.cwd() / entry_point_filename).absolute()
if not entry_point.is_file():
entry_point = (Path.cwd() / notebook_path).absolute()
# fix for VSCode pushing uuid at the end of the notebook name.
if not entry_point.exists():
# noinspection PyBroadException
try:
alternative_entry_point = '-'.join(entry_point_filename.split('-')[:-5])+'.ipynb'
# now we should try to find the actual file
entry_point_alternative = (Path.cwd() / alternative_entry_point).absolute()
if not entry_point_alternative.is_file():
entry_point_alternative = (Path.cwd() / alternative_entry_point).absolute()
# If we found it replace it
if entry_point_alternative.exists():
entry_point = entry_point_alternative
except Exception as ex:
cls._get_logger().warning('Failed accessing jupyter notebook {}: {}'.format(notebook_path, ex))
# get local ipynb for observer
local_ipynb_file = entry_point.as_posix()
# now replace the .ipynb with .py
# we assume we will have that file available with the Jupyter notebook plugin
entry_point = entry_point.with_suffix('.py')
script_entry_point = entry_point.as_posix()
# install the post store hook,
# notice that if we do not have a local file we serialize/write every time the entire notebook
cls._jupyter_install_post_store_hook(local_ipynb_file, is_google_colab)
return script_entry_point
except Exception:
return None
class Video(object):
def __init__(self, path):
self.path = Path(path)
self.get_info()
# *that's* not a video format we care about
if self._format['format_name'] == 'tty':
raise NotAVideo('unable to transcode ansi videos')
# This checks that there is at least one video stream available
if not any(s.get('codec_type') == 'video' for s in self._streams):
raise NotAVideo('no video streams available')
def get_info(self):
cmd = ['ffprobe',
'-show_streams', '-show_format',
'-hide_banner',
'-print_format', 'json', str(self.path)]
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode != 0:
LOG.debug(stderr)
raise NotAVideo('failed to read video metadata')
res = json.loads(stdout)
self._streams = res.get('streams')
self._format = res.get('format')
def __str__(self):
return '<%s (%s)>' % (
self.path,
self._format['format_long_name'])
def stream_types(self):
return [(s['codec_type'], s['codec_name']) for s in self._streams]
def audio_codec(self):
'''Returns the codec of the first audio stream'''
return [s[1] for s in self.stream_types()
if s[0] == 'audio'][0]
def video_codec(self):
'''Returns the codec of the first video stream'''
return [s[1] for s in self.stream_types()
if s[0] == 'video'][0]
def video_size(self):
stream = [s for s in self._streams if s['codec_type'] == 'video'][0]
return (stream['width'], stream['height'])
def transcode(self,
video_codec=None, video_args=None,
audio_codec=None, audio_args=None,
height=None, width=None, scale=True,
profile=None, loglevel='warning',
copy_if_same=False, suffix=None,
output=None, progress=None):
cmd = [
'ffmpeg',
'-loglevel', loglevel, '-hide_banner', '-nostats',
'-y',
'-i', str(self.path),
]
if progress is not None:
cmd += [
'-progress', progress,
]
if profile is None:
profile = {
'video_codec': 'copy',
'audio_codec': 'copy',
'suffix': '.vid',
}
if video_codec is not None:
profile['video_codec'] = video_codec
if video_args is not None:
profile['video_args'] = video_args
if audio_codec is not None:
profile['audio_codec'] = audio_codec
if audio_args is not None:
profile['audio_args'] = audio_args
if height is not None:
profile['height'] = height
if suffix is not None:
profile['suffix'] = suffix
if width is not None:
profile['width'] = width
if copy_if_same and profile['video_codec'] == self.video_codec():
profile['video_codec'] = 'copy'
if copy_if_same and profile['audio_codec'] == self.audio_codec():
profile['audio_codec'] = 'copy'
if scale and ((profile['width'], profile['height'])
!= self.video_size()):
if profile['video_codec'] == 'copy':
raise ValueError('cannot resize with "copy" codec')
# why -2? https://trac.ffmpeg.org/ticket/309
cmd += ['-vf', 'scale=%d:-2' % profile['width']]
cmd += ['-c:a', profile['audio_codec']]
if profile.get('audio_args'):
cmd += shlex.split(profile['audio_args'])
cmd += ['-c:v', profile['video_codec']]
if profile.get('video_args'):
cmd += shlex.split(profile['video_args'])
if output is None:
output = self.path.with_suffix(profile['suffix'])
cmd += [str(output)]
LOG.debug('running %s', cmd)
try:
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode != 0:
LOG.debug(stderr)
LOG.error('%s: transcoding failed: removing output file "%s"',
self.path, output)
try:
output.unlink()
except OSError:
pass
raise TranscodingFailed('transcoding failed')
except KeyboardInterrupt:
LOG.error('%s: transcoding interrupted by user: '******'removing output file "%s"',
self.path, output)
try:
output.unlink()
except OSError:
pass
raise
link_gmx_mpi = request.config.getoption('link_gmx_mpi')
tools = str(Path('~/gmx_mpi').expanduser()) if link_gmx_mpi else ''
append_suffix = 'yes' if request.config.getoption(
'append_suffix') else 'no'
return tools, append_suffix, Path
path_config = Path('~/.gromacswrapper.cfg').expanduser()
gw_config = ConfigParser()
if path_config.exists():
gw_config.read(str(path_config.resolve()))
config_existed = True
else:
gw_config.read('gromacs/templates/gromacswrapper.cfg')
config_existed = False
config_backup = path_config.with_suffix('.bak')
def pytest_configure(config):
link_gmx_mpi = config.getoption('link_gmx_mpi')
append_suffix = 'yes' if config.getoption('append_suffix') else 'no'
if config_existed:
shutil.copy(str(path_config), str(config_backup))
tools = gmx_mpi_linked(link_gmx_mpi)
gw_config.set('Gromacs', 'tools', tools)
gw_config.set('Gromacs', 'append_suffix', append_suffix)
with open(str(path_config), 'w') as config_file:
gw_config.write(config_file)
def pytest_unconfigure(config):
