def globaltrace_lt(self, frame, why, arg):
"""
## Copied from trace module ##
Handler for call events.
If the code block being entered is to be ignored, returns `None',
else returns self.localtrace.
"""
if why == 'call':
code = frame.f_code
filename = frame.f_globals.get('__file__', None)
if filename:
# XXX modname() doesn't work right for packages, so
# the ignore support won't work right for packages
ignore_it = self.ignore.names(filename)
if not ignore_it:
if self.trace:
filename = Path(filename)
modulename = '.'.join(
filename.relative_to(
self.ignore.package_path).parts[:-1] +
(filename.stem, ))
self.__out(' --- modulename: %s, funcname: %s' %
(modulename, code.co_name))
return self.localtrace
else:
return None
def make_decision(path: pathlib2.Path):
match_sum = sum(
map(
lambda ignore: ignore.match(
str(path.relative_to(self.root_path))) is not None,
self.ignore_list))
return match_sum == 0 if self.mode == IGNORE_MODE else match_sum > 0
def is_resource(package, name):
"""True if name is a resource inside package.
Directories are *not* resources.
"""
package = _get_package(package)
_normalize_path(name)
try:
package_contents = set(contents(package))
except OSError as error:
if error.errno not in (errno.ENOENT, errno.ENOTDIR):
# We won't hit this in the Python 2 tests, so it'll appear
# uncovered. We could mock os.listdir() to return a non-ENOENT or
# ENOTDIR, but then we'd have to depend on another external
# library since Python 2 doesn't have unittest.mock. It's not
# worth it.
raise # pragma: nocover
return False
if name not in package_contents:
return False
# Just because the given file_name lives as an entry in the package's
# contents doesn't necessarily mean it's a resource. Directories are not
# resources, so let's try to find out if it's a directory or not.
path = Path(package.__file__).parent / name
if path.is_file():
return True
if path.is_dir():
return False
# If it's not a file and it's not a directory, what is it? Well, this
# means the file doesn't exist on the file system, so it probably lives
# inside a zip file. We have to crack open the zip, look at its table of
# contents, and make sure that this entry doesn't have sub-entries.
archive_path = package.__loader__.archive # type: ignore
package_directory = Path(package.__file__).parent
with ZipFile(archive_path) as zf:
toc = zf.namelist()
relpath = package_directory.relative_to(archive_path)
candidate_path = relpath / name
for entry in toc: # pragma: nobranch
try:
relative_to_candidate = Path(entry).relative_to(candidate_path)
except ValueError:
# The two paths aren't relative to each other so we can ignore it.
continue
# Since directories aren't explicitly listed in the zip file, we must
# infer their 'directory-ness' by looking at the number of path
# components in the path relative to the package resource we're
# looking up. If there are zero additional parts, it's a file, i.e. a
# resource. If there are more than zero it's a directory, i.e. not a
# resource. It has to be one of these two cases.
return len(relative_to_candidate.parts) == 0
# I think it's impossible to get here. It would mean that we are looking
# for a resource in a zip file, there's an entry matching it in the return
# value of contents(), but we never actually found it in the zip's table of
# contents.
raise AssertionError('Impossible situation')
def _local_url(path):
"""Copy a filepath into the static dir if required
"""
path = Path(path).resolve()
# if file is already below static in the hierarchy, don't do anything
if static in path.parents:
return path.relative_to(base)
# otherwise copy the file into static
static.mkdir(parents=True, exist_ok=True)
local = static / path.name
copyfile(str(path), str(local)) # only need str for py<3.6
return str(local.relative_to(base))
def _local_url(path):
"""Copy a filepath into the static dir if required
"""
path = Path(path).resolve()
# if file is already below static in the hierarchy, don't do anything
if static in path.parents:
return path.relative_to(base)
# otherwise copy the file into static
static.mkdir(parents=True, exist_ok=True)
local = static / path.name
copyfile(str(path), str(local)) # only need str for py<3.6
return str(local.relative_to(base))
def contents(package):
"""Return an iterable of entries in `package`.
Note that not all entries are resources. Specifically, directories are
not considered resources. Use `is_resource()` on each entry returned here
to check if it is a resource or not.
"""
package = _get_package(package)
package_directory = Path(package.__file__).parent
try:
return os.listdir(str(package_directory))
except OSError as error:
if error.errno not in (errno.ENOENT, errno.ENOTDIR):
# We won't hit this in the Python 2 tests, so it'll appear
# uncovered. We could mock os.listdir() to return a non-ENOENT or
# ENOTDIR, but then we'd have to depend on another external
# library since Python 2 doesn't have unittest.mock. It's not
# worth it.
raise # pragma: nocover
# The package is probably in a zip file.
archive_path = getattr(package.__loader__, 'archive', None)
if archive_path is None:
raise
relpath = package_directory.relative_to(archive_path)
with ZipFile(archive_path) as zf:
toc = zf.namelist()
subdirs_seen = set()
subdirs_returned = []
for filename in toc:
path = Path(filename)
# Strip off any path component parts that are in common with the
# package directory, relative to the zip archive's file system
# path. This gives us all the parts that live under the named
# package inside the zip file. If the length of these subparts is
# exactly 1, then it is situated inside the package. The resulting
# length will be 0 if it's above the package, and it will be
# greater than 1 if it lives in a subdirectory of the package
# directory.
#
# However, since directories themselves don't appear in the zip
# archive as a separate entry, we need to return the first path
# component for any case that has > 1 subparts -- but only once!
if path.parts[:len(relpath.parts)] != relpath.parts:
continue
subparts = path.parts[len(relpath.parts):]
if len(subparts) == 1:
subdirs_returned.append(subparts[0])
elif len(subparts) > 1: # pragma: nobranch
subdir = subparts[0]
if subdir not in subdirs_seen:
subdirs_seen.add(subdir)
subdirs_returned.append(subdir)
return subdirs_returned
def pytest_collection_modifyitems(items):
"""Mark collected test types based on their location"""
for item in items:
type_marker = item.get_closest_marker("type")
if type_marker and type_marker.args:
continue # Do not modify manually set marks
file_path = Path("%s" % item.reportinfo()[0])
repo_rel_path = file_path.relative_to(repo_path())
ty = repo_rel_path.parts[1]
if ty not in test_types:
raise Exception("Test in %s not TYPE marked: %r (%r)" %
(repo_rel_path, item, ty))
item.add_marker(pytest.mark.type.with_args(ty))
def create_task(self):
# type: () -> Task
"""
Create the new populated Task
:return: newly created Task object
"""
local_entry_file = None
repo_info = None
if self.folder or (self.script and Path(self.script).is_file()
and not self.repo):
self.folder = os.path.expandvars(os.path.expanduser(
self.folder)) if self.folder else None
self.script = os.path.expandvars(os.path.expanduser(
self.script)) if self.script else None
self.cwd = os.path.expandvars(os.path.expanduser(
self.cwd)) if self.cwd else None
if Path(self.script).is_file():
entry_point = self.script
else:
entry_point = (Path(self.folder) / self.script).as_posix()
entry_point = os.path.abspath(entry_point)
if not os.path.isfile(entry_point):
raise ValueError(
"Script entrypoint file \'{}\' could not be found".format(
entry_point))
local_entry_file = entry_point
repo_info, requirements = ScriptInfo.get(
filepaths=[entry_point],
log=getLogger(),
create_requirements=self.packages is True,
uncommitted_from_remote=True,
detect_jupyter_notebook=False)
# check if we have no repository and no requirements raise error
if self.raise_on_missing_entries and (not self.requirements_file and not self.packages) \
and not self.repo and (
not repo_info or not repo_info.script or not repo_info.script.get('repository')):
raise ValueError(
"Standalone script detected \'{}\', but no requirements provided"
.format(self.script))
if self.base_task_id:
if self.verbose:
print('Cloning task {}'.format(self.base_task_id))
task = Task.clone(source_task=self.base_task_id,
project=Task.get_project_id(self.project_name))
else:
# noinspection PyProtectedMember
task = Task._create(task_name=self.task_name,
project_name=self.project_name,
task_type=self.task_type
or Task.TaskTypes.training)
# if there is nothing to populate, return
if not any([
self.folder, self.commit, self.branch, self.repo,
self.script, self.cwd, self.packages,
self.requirements_file, self.base_task_id, self.docker
]):
return task
task_state = task.export_task()
if 'script' not in task_state:
task_state['script'] = {}
if repo_info:
task_state['script']['repository'] = repo_info.script['repository']
task_state['script']['version_num'] = repo_info.script[
'version_num']
task_state['script']['branch'] = repo_info.script['branch']
task_state['script']['diff'] = repo_info.script['diff'] or ''
task_state['script']['working_dir'] = repo_info.script[
'working_dir']
task_state['script']['entry_point'] = repo_info.script[
'entry_point']
task_state['script']['binary'] = repo_info.script['binary']
task_state['script']['requirements'] = repo_info.script.get(
'requirements') or {}
if self.cwd:
self.cwd = self.cwd
cwd = self.cwd if Path(self.cwd).is_dir() else (
Path(repo_info.script['repo_root']) / self.cwd).as_posix()
if not Path(cwd).is_dir():
raise ValueError(
"Working directory \'{}\' could not be found".format(
cwd))
cwd = Path(cwd).relative_to(
repo_info.script['repo_root']).as_posix()
entry_point = \
Path(repo_info.script['repo_root']) / repo_info.script['working_dir'] / repo_info.script[
'entry_point']
entry_point = entry_point.relative_to(cwd).as_posix()
task_state['script']['entry_point'] = entry_point
task_state['script']['working_dir'] = cwd
elif self.repo:
# normalize backslashes and remove first one
entry_point = '/'.join(
[p for p in self.script.split('/') if p and p != '.'])
cwd = '/'.join(
[p for p in (self.cwd or '.').split('/') if p and p != '.'])
if cwd and entry_point.startswith(cwd + '/'):
entry_point = entry_point[len(cwd) + 1:]
task_state['script']['repository'] = self.repo
task_state['script']['version_num'] = self.commit or None
task_state['script']['branch'] = self.branch or None
task_state['script']['diff'] = ''
task_state['script']['working_dir'] = cwd or '.'
task_state['script']['entry_point'] = entry_point
else:
# standalone task
task_state['script']['entry_point'] = self.script
task_state['script']['working_dir'] = '.'
# update requirements
reqs = []
if self.requirements_file:
with open(self.requirements_file.as_posix(), 'rt') as f:
reqs = [line.strip() for line in f.readlines()]
if self.packages and self.packages is not True:
reqs += self.packages
if reqs:
# make sure we have clearml.
clearml_found = False
for line in reqs:
if line.strip().startswith('#'):
continue
package = reduce(lambda a, b: a.split(b)[0], "#;@=~<>",
line).strip()
if package == 'clearml':
clearml_found = True
break
if not clearml_found:
reqs.append('clearml')
task_state['script']['requirements'] = {'pip': '\n'.join(reqs)}
elif not self.repo and repo_info and not repo_info.script.get(
'requirements'):
# we are in local mode, make sure we have "requirements.txt" it is a must
reqs_txt_file = Path(
repo_info.script['repo_root']) / "requirements.txt"
if self.raise_on_missing_entries and not reqs_txt_file.is_file():
raise ValueError("requirements.txt not found [{}] "
"Use --requirements or --packages".format(
reqs_txt_file.as_posix()))
if self.add_task_init_call:
script_entry = os.path.abspath(
'/' + task_state['script'].get('working_dir', '.') + '/' +
task_state['script']['entry_point'])
idx_a = 0
# find the right entry for the patch if we have a local file (basically after __future__
if local_entry_file:
with open(local_entry_file, 'rt') as f:
lines = f.readlines()
future_found = self._locate_future_import(lines)
if future_found >= 0:
idx_a = future_found + 1
task_init_patch = ''
if self.repo or task_state.get('script', {}).get('repository'):
# if we do not have requirements, add clearml to the requirements.txt
if not reqs:
task_init_patch += \
"diff --git a/requirements.txt b/requirements.txt\n" \
"--- a/requirements.txt\n" \
"+++ b/requirements.txt\n" \
"@@ -0,0 +1,1 @@\n" \
"+clearml\n"
# Add Task.init call
task_init_patch += \
"diff --git a{script_entry} b{script_entry}\n" \
"--- a{script_entry}\n" \
"+++ b{script_entry}\n" \
"@@ -{idx_a},0 +{idx_b},3 @@\n" \
"+from clearml import Task\n" \
"+Task.init()\n" \
"+\n".format(
script_entry=script_entry, idx_a=idx_a, idx_b=idx_a + 1)
else:
# Add Task.init call
task_init_patch += \
"from clearml import Task\n" \
"Task.init()\n\n"
# make sure we add the dif at the end of the current diff
task_state['script']['diff'] = task_state['script'].get('diff', '')
if task_state['script']['diff'] and not task_state['script'][
'diff'].endswith('\n'):
task_state['script']['diff'] += '\n'
task_state['script']['diff'] += task_init_patch
# set base docker image if provided
if self.docker:
task.set_base_docker(
docker_cmd=self.docker.get('image'),
docker_arguments=self.docker.get('args'),
docker_setup_bash_script=self.docker.get('bash_script'),
)
if self.verbose:
if task_state['script']['repository']:
repo_details = {
k: v
for k, v in task_state['script'].items()
if v and k not in ('diff', 'requirements', 'binary')
}
print('Repository Detected\n{}'.format(
json.dumps(repo_details, indent=2)))
else:
print('Standalone script detected\n Script: {}'.format(
self.script))
if task_state['script'].get('requirements') and \
task_state['script']['requirements'].get('pip'):
print('Requirements:{}{}'.format(
'\n Using requirements.txt: {}'.format(
self.requirements_file.as_posix())
if self.requirements_file else '',
'\n {}Packages: {}'.format(
'Additional ' if self.requirements_file else '',
self.packages) if self.packages else ''))
if self.docker:
print('Base docker image: {}'.format(self.docker))
# update the Task
task.update_task(task_state)
self.task = task
return task
