def exclusion(path):
relative_to_search_base = path.relative_to(search_base)
if relative_to_search_base.as_posix() in non_special_excludes:
return True
for pattern in match_excludes:
result = relative_to_search_base.match(pattern, match_entire=True)
if result:
return True
relative_to_project_root = path.relative_to(project_root)
for pattern in project_root_relative_excludes:
result = relative_to_project_root.match(pattern, match_entire=True)
if result:
return True
return False
def exclusion(path):
relative_to_search_base = path.relative_to(search_base)
if relative_to_search_base.as_posix() in non_special_excludes:
return True
for pattern in match_excludes:
result = relative_to_search_base.match(pattern, match_entire=True)
if result:
return True
relative_to_project_root = path.relative_to(project_root)
for pattern in project_root_relative_excludes:
result = relative_to_project_root.match(pattern, match_entire=True)
if result:
return True
return False
def template_url(context, template_name):
'''
Return a GitHub URL to the source of the given template.
'''
path = get_template_path(template_name)
return github_url_for_path(path.relative_to(ROOT_DIR))
def run(self):
global CMAKE_BUILD_DIR
self.jobs = multiprocessing.cpu_count()
plat_specifier = '.{0}-{1}.{2}'.format(self.plat_name, *sys.version_info[:2])
self.build_temp = os.path.join(self.build_base, 'temp' + plat_specifier, self.config)
# if setup.py is directly called use CMake to build product
if CMAKE_BUILD_DIR == '.':
# set path to the root of OpenVINO CMakeList file
openvino_root_dir = Path(__file__).resolve().parents[4]
self.announce(f'Configuring cmake project: {openvino_root_dir}', level=3)
self.spawn(['cmake', '-H' + str(openvino_root_dir), '-B' + self.build_temp,
'-DCMAKE_BUILD_TYPE={type}'.format(type=self.config),
'-DENABLE_PYTHON=ON',
'-DENABLE_OV_ONNX_FRONTEND=ON'])
self.announce('Building binaries', level=3)
self.spawn(['cmake', '--build', self.build_temp,
'--config', self.config, '-j', str(self.jobs)])
CMAKE_BUILD_DIR = self.build_temp
self.run_command('build_clib')
build.run(self)
# Copy extra package_data content filtered by find_packages
dst = Path(self.build_lib)
src = Path(get_package_dir(PY_INSTALL_CFG))
exclude = ignore_patterns('*ez_setup*', '*__pycache__*', '*.egg-info*')
for path in src.glob('**/*'):
if path.is_dir() or exclude(str(path)):
continue
path_rel = path.relative_to(src)
(dst / path_rel.parent).mkdir(exist_ok=True, parents=True)
copyfile(path, dst / path_rel)
def _matches_recursive_blob(path, pattern):
split = pattern.split('**', maxsplit=1)
try:
remaining_path = path.relative_to(split[0])
except ValueError:
return False
if len(split) == 1:
return False
remaining_pattern = split[1]
if remaining_pattern.startswith(os_utils.path_sep()):
remaining_pattern = remaining_pattern[len(os_utils.path_sep()):]
if remaining_pattern == '': # this can happen if pattern ends with **
return True
if remaining_path.match(remaining_pattern):
return True
elif '**' not in remaining_pattern:
return False
(remaining_head, remaining_tail
) = SingleFileMatcher._split_first_parent(remaining_pattern)
for parent in remaining_path.parents:
if parent.name == '':
continue
if fnmatch(parent.name, remaining_head):
if SingleFileMatcher._matches_recursive_blob(
remaining_path.relative_to(parent), remaining_tail):
return True
return False
def _naive_relative_to(path: PosixPath, root: PosixPath) -> PurePosixPath:
"""
Compute relative PurePosixPath, result may include '..' parts.
Both arguments must be absolute PurePosixPath's and lack '..' parts.
Possibility of symlinks is ignored, i. e. arguments are interpreted
as resolved paths.
"""
if not path.is_absolute():
raise ValueError(path)
if not root.is_absolute():
raise ValueError(root)
if '..' in path.parts:
raise ValueError(path)
if '..' in root.parts:
raise ValueError(root)
upstairs = 0
while root not in path.parents:
parent = root.parent
assert parent != root
root = parent
upstairs += 1
return PurePosixPath(
* (('..',) * upstairs),
path.relative_to(root) )
def includes_iterator():
for pattern in includes:
for path in search_base.glob(pattern):
# TODO(bhamiltoncx): Handle hidden files on Windows.
if path.is_file() and (include_dotfiles
or not path.name.startswith('.')):
yield path.relative_to(search_base)
def template_map(source: Path, target: Path, maps: dict):
loader = FileSystemLoader(str(source))
environment = Environment(loader=loader, undefined=DebugUndefined)
for path in [x for x in source.rglob('*') if not x.is_dir()]:
relative = path.relative_to(source)
if str(relative).startswith('host_vars'):
continue
if str(relative).startswith('custom_role'):
continue
context = maps.get(str(relative))
dest = target / relative
if not dest.parent.exists():
dest.parent.mkdir(parents=True)
if context:
if isinstance(context, Validators) or isinstance(
context, Bootnode):
context.write(environment, target, relative)
else:
template = environment.get_template(str(relative))
dest.write_text(template.render(context))
else:
copy(str(path), str(dest))
def get_outputfolder_given_path(
path: Path, dervo_root: Path, output_root: Path):
"""Create output folder, create symlink to it """
# Create output folder (name defined by relative path wrt root_dervo)
output_foldername = str(path.relative_to(dervo_root)).replace('/', '.')
workfolder = vst.mkdir(output_root/output_foldername)
return workfolder
def template_url(context, template_name):
'''
Return a GitHub URL to the source of the given template.
'''
path = get_template_path(template_name)
return github_url_for_path(path.relative_to(ROOT_DIR))
def find_prebuilt_extensions(search_dirs):
"""Collect prebuilt python extensions."""
extensions = []
ext_pattern = ""
if sys.platform == "linux":
ext_pattern = "**/*.so"
elif sys.platform == "win32":
ext_pattern = "**/*.pyd"
elif sys.platform == "darwin":
ext_pattern = "**/*.so"
for base_dir in search_dirs:
for path in Path(base_dir).glob(ext_pattern):
if path.match("openvino/libs/*"):
continue
relpath = path.relative_to(base_dir)
if relpath.parent != ".":
package_names = str(relpath.parent).split(os.path.sep)
else:
package_names = []
package_names.append(path.name.split(".", 1)[0])
name = ".".join(package_names)
extensions.append(PrebuiltExtension(name, sources=[str(path)]))
if not extensions:
extensions.append(
PrebuiltExtension("openvino", sources=[str("setup.py")]))
return extensions
def find_prebuilt_extensions(search_dirs):
"""collect prebuilt python extensions"""
extensions = []
ext_pattern = ''
if sys.platform == 'linux':
ext_pattern = '**/*.so'
elif sys.platform == 'win32':
ext_pattern = '**/*.pyd'
elif sys.platform == 'darwin':
ext_pattern = '**/*.so'
for base_dir in search_dirs:
for path in Path(base_dir).glob(ext_pattern):
if path.match('openvino/libs/*'):
continue
relpath = path.relative_to(base_dir)
if relpath.parent != '.':
package_names = str(relpath.parent).split(os.path.sep)
else:
package_names = []
package_names.append(path.name.split('.', 1)[0])
name = '.'.join(package_names)
extensions.append(PrebuiltExtension(name, sources=[str(path)]))
if not extensions:
extensions.append(
PrebuiltExtension('openvino', sources=[str('setup.py')]))
return extensions
def _try_relativize_path(path):
"""Make the path relative to current directory if it's under that directory."""
path = pathlib.Path(path)
try:
path = path.relative_to(os.getcwd())
except ValueError:
pass
return str(path)
def find_dandiset_and_subpath(path: Path) -> tuple[Dandiset, Path]:
"""
Find the Dandiset rooted at ``path`` or one of its parents, and return the
Dandiset along with ``path`` made relative to the Dandiset root
"""
path = path.absolute()
ds = Dandiset.find(path)
if ds is None:
raise ValueError(f"{path}: not a Dandiset")
return (ds, path.relative_to(ds.path))
def html_zip(self, tmp_path, html_path):
if not hasattr(zipfile, 'Path'):
pytest.skip("Needs zipfile.Path")
zip_path = tmp_path / 'qutebrowser.zip'
with zipfile.ZipFile(zip_path, 'w') as zf:
for path in html_path.iterdir():
zf.write(path, path.relative_to(tmp_path))
yield zipfile.Path(zip_path) / 'qutebrowser'
def as_route(dirpath, filepath):
path = (Path(dirpath) / filepath).absolute()
if ".py" in path.suffix:
return
else:
url_path = Path(base_url_path) / path.relative_to(topdir)
target = (url_path.parent if strip_stem is not None
and strip_stem == url_path.stem else url_path)
handler = lambda eg: get(eg, path=path)
return ((b"GET", str(target).encode("utf-8")),
update_wrapper(handler, get))
def find_prebuilt_extensions(base_dir, ext_pattern):
extensions = []
for path in Path(base_dir).glob(ext_pattern):
relpath = path.relative_to(base_dir)
if relpath.parent != ".":
package_names = str(relpath.parent).split(os.path.sep)
else:
package_names = []
package_names.append(path.name.split(".", 1)[0])
name = ".".join(package_names)
extensions.append(PrebuiltExtension(name, sources=[str(path)]))
return extensions
def run(self):
self.run_command('build_clib')
build.run(self)
# Copy extra package_data content filtered by find_packages
dst = Path(self.build_lib)
src = Path(get_package_dir(PY_INSTALL_CFG))
exclude = ignore_patterns('*ez_setup*', '*__pycache__*', '*.egg-info*')
for path in src.glob('**/*'):
if path.is_dir() or exclude(str(path)):
continue
path_rel = path.relative_to(src)
(dst / path_rel.parent).mkdir(exist_ok=True, parents=True)
copyfile(path, dst / path_rel)
def _build_sitemap(site_dir: str) -> None:
"""Builds a minimal sitemap.xml for drake.mit.edu.
This helps Google, Bing, and other search engines decide which pages on the
generated drake.mit.edu site should be crawled, and helps determine the
canonical version of each page.
https://developers.google.com/search/docs/advanced/sitemaps/build-sitemap
Args:
site_dir: The absolute path to the root directory of the generated
website and the directory to which the built sitemap.xml will be
written.
Raises:
OSError: If the directory to which site_dir refers is not readable or
writable.
"""
print("Building sitemap.xml...")
root_path = Path(site_dir)
assert root_path.is_absolute(), \
"Path to generated website is not an absolute path"
paths = root_path.glob("**/*.html")
XML_NAMESPACE = "http://www.sitemaps.org/schemas/sitemap/0.9"
ROOT_URL = "https://drake.mit.edu"
urlset = ET.Element("urlset", xmlns=XML_NAMESPACE)
for path in sorted(paths):
relative_path = path.relative_to(root_path)
url = ET.SubElement(urlset, "url")
if relative_path.name == "index.html":
# sitemap.xml should only include canonical urls.
location = relative_path.parent.as_posix() + "/"
else:
location = relative_path.as_posix()
location = urllib.parse.urljoin(ROOT_URL,
urllib.parse.quote(location))
loc = ET.SubElement(url, "loc")
loc.text = location
sitemap = ET.ElementTree(urlset)
sitemap.write(os.path.join(site_dir, "sitemap.xml"),
encoding="utf-8",
pretty_print=True,
xml_declaration=True)
def addAllFiles(targetDir, sourceDir):
#Get everything recursively.
rootpath = Path(sourceDir)
paths = list(rootpath.glob("**/*"))
print(targetDir)
print(sourceDir)
for path in paths:
rel = str(path.relative_to(sourceDir))
if(path.is_dir()):
ensureFolder(targetDir+slash+rel)
else:
ensureFolder((targetDir+slash+rel).rpartition(slash)[0])
if not (Path(targetDir+slash+rel).exists()):
#in dst, src order
makeHardLink(targetDir+slash+rel, str(path.absolute()))
addToLinkManifest(targetDir+slash+rel, str(path.absolute()))
def html_zip(self, tmp_path, html_path):
if not hasattr(zipfile, 'Path'):
pytest.skip("Needs zipfile.Path")
zip_path = tmp_path / 'qutebrowser.zip'
with zipfile.ZipFile(zip_path, 'w') as zf:
for path in html_path.rglob('*'):
zf.write(path, path.relative_to(tmp_path))
assert sorted(zf.namelist()) == [
'qutebrowser/html/README',
'qutebrowser/html/subdir/',
'qutebrowser/html/subdir/subdir-file.html',
'qutebrowser/html/test1.html',
'qutebrowser/html/test2.html',
'qutebrowser/html/unrelatedhtml',
]
yield zipfile.Path(zip_path) / 'qutebrowser'
def get_md_links(folder):
MD_LINK_RE = r'\[.+?\]\((.+?)(#.+)?\)'
idf_path = Path(os.getenv('IDF_PATH'))
links = []
for path in (idf_path / folder).rglob('*.md'):
if any([path.relative_to(idf_path).match(exclude_doc) for exclude_doc in EXCLUDE_DOCS_LIST]):
print('{} - excluded'.format(path))
continue
with path.open(encoding='utf8') as f:
content = f.read()
for url in re.findall(MD_LINK_RE, content):
link = Link(path, url[0].lstrip())
# Ignore "local" links
if not link.url.startswith('#'):
links.append(link)
return links
def get_job_files(uuid):
"""Get the list of output files. ``GET /task/{task_id}/files``
:param uuid: task identifier
:return: JSON response with list of files produced by the task.
"""
job = documents.JobMetadata.find_one(mongo.slivkadb, uuid=uuid)
if job is None:
raise abort(404)
if job.status == JobStatus.PENDING:
return JsonResponse({'statuscode': 200, 'files': []}, status=200)
service_conf = slivka.settings.get_service_configuration(job.service)
work_dir = pathlib.Path(job.work_dir)
output_files = [
OutputFile(uuid='%s/%s' % (job.uuid, path.name),
title=path.name,
label=key,
location=path.relative_to(
slivka.settings.TASKS_DIR).as_posix(),
media_type=val.get('media-type'))
for key, val in service_conf.execution_config['outputs'].items()
for path in work_dir.glob(val['path'])
]
return JsonResponse(
{
'statuscode':
200,
'files':
[{
'uuid': file.uuid,
'title': file.title,
'label': file.label,
'mimetype': file.media_type,
'URI': url_for('.get_file_metadata', uid=file.uuid),
'contentURI': url_for('root.outputs', location=file.location)
} for file in output_files]
},
status=200)
def relative_to(path1: Path, path2: Path) -> Path:
common_path = path1
relative = ''
path = path2.resolve()
result = path
while True:
try:
result = path.relative_to(common_path)
except ValueError:
_common_path = common_path.parent
relative += '../'
else:
if not relative:
relative = './'
return Path(relative + str(result))
if _common_path == common_path:
break
common_path = _common_path
return result
def refresh_last_update_metadata(logger, session, meta):
"""Update the ``last_update`` and ``status`` meta data value."""
_ = logger
# Get path in irods that corresponds to root and update the meta data there.
path = pathlib.Path(meta["path"])
root = pathlib.Path(meta["root"])
target = pathlib.Path(meta["target"])
logger.info("meta = %s" % meta)
print("meta = %s" % meta)
rel_root_path = path.relative_to(root) # relative to root
print("... %s" % rel_root_path)
rel_folder_path = "/".join(
str(rel_root_path).split("/")[1:]) # relative to run folder
print("... %s" % rel_folder_path)
root_target = str(target)[:-(len(str(rel_folder_path)) + 1)]
print("... %s" % root_target)
with cleanuping(session) as wrapped_session:
coll = wrapped_session.collections.get(root_target)
# Replace ``last_update`` and ``status`` meta data.
coll.metadata[KEY_LAST_UPDATE] = iRODSMeta(
KEY_LAST_UPDATE,
datetime.datetime.now().isoformat(), "")
coll.metadata[KEY_STATUS] = iRODSMeta(KEY_STATUS, "running", "")
def shortPath(path):
'''Returns a 'prettier' path: relative to the cwd rather than root'''
try:
return path.relative_to(Path.cwd())
except Exception:
return path
def resolvePath(self, path):
'''
Returns the equivalent path, but in the temporary copy of the nuclide
root.
'''
return self._nuclideTranspilePath / path.relative_to(self.nuclidePath)
def _group_out_path(group_num, path, out_dir, relative_to, fmt='%d'):
path = pathlib.Path(path)
out_dir = pathlib.Path(out_dir)
group_name = fmt % (group_num, )
return out_dir / group_name / path.relative_to(relative_to)
def reroot_path(self, path: Path) -> Path:
"Submitters should translate paths to be relative to execution environment"
return Path(self.remote_root) / path.relative_to(
Config['nanopore']['path'])
def list_templates(self):
return set(
str(path.relative_to(base))
for base, path in yield_all_sources(*self.search_paths)
if not path.is_symlink() and path.is_file()
and path.suffix == self.template_suffix)
def __init__(self, file_with_info):
path, inputbase, args = file_with_info
self.path = path
self.args = args
self.cuesheet = cuesheet(path)
self.tags = flactags(path)
self.picture = picture(path)
self.jobs = []
if args.verbose:
print("%s: %d tracks in file." % (path, self.cuesheet.lasttrack))
outdir = args.outputdir
if args.keepdirs:
relative = path.relative_to(inputbase) if inputbase \
else path
# If the fatsafe flag is specified, every component that we add
# to the output path must be fatsafe
if args.fatsafe:
for part in relative.parent.parts:
outdir = outdir / fatsafe(part)
else:
outdir = outdir.joinpath(relative.parent)
if args.subdir:
if args.fatsafe:
outdir = outdir / fatsafe(path.stem)
else:
outdir = outdir / path.stem
self.badtracks = []
for track in range(1, self.cuesheet.lasttrack + 1):
if track not in self.cuesheet.tracks:
self.badtracks.append(
"track %d not present" % track)
continue
title = self.tags.get_tag('TITLE', track)
artist = self.tags.get_tag('ARTIST', track)
if not title:
self.badtracks.append("track %d is missing TITLE tag" % track)
continue
if not artist:
self.badtracks.append("track %d is missing ARTIST tag" % track)
continue
if args.verbose:
print("%02d: %s by %s" % (
track, title, artist))
outfilename = "%02d %s (%s)" % (
track, title, artist)
outfilename = outfilename.replace(os.sep, '')
if args.fatsafe:
outfilename = fatsafe(outfilename)
if args.max_filename_length:
# This is somewhat clunky, but hopefully won't come up
# too often! The limit on filename length is on the
# number of bytes the filename encodes to, but we can
# only remove a whole number of characters
while len(outfilename.encode('utf-8')) > args.max_filename_length:
outfilename = outfilename[:-1]
outfilename = outfilename + ".mp3"
outputfile = outdir / outfilename
try:
output_mtime = outputfile.stat().st_mtime
except:
output_mtime = 0
if args.update:
input_mtime = self.path.stat().st_mtime
if output_mtime > input_mtime:
if args.verbose:
print(" - skipping %s because it is newer" % outputfile)
continue
self.jobs.append((self, track, outputfile))
def list_templates(self):
return set(str(path.relative_to(base))
for base, path in yield_all_sources(*self.search_paths)
if not path.is_symlink() and path.is_file()
and path.suffix == self.template_suffix)
def cmd_make_dist(self) -> None:
print("Downloading dependencies")
def download_dependency(name: str) -> Path:
url = self.project.get_conf("dependencies", f"{name}_url")
filename = self.project.get_conf("dependencies", f"{name}_file")
download_path = self.project.download_dir / filename
if not download_path.exists():
downloaded_size = self.download_file(
name,
lambda: self.http_client.request(HTTPRequest(url)),
lambda: download_path.open("wb"),
)
print(
f"{name} downloaded - net {self.format_bytes(downloaded_size)}"
)
else:
print(f"{name} already downloaded")
return download_path
localize_me_file = download_dependency("localize_me")
ccloader_file = download_dependency("ccloader")
ultimate_ui_file = download_dependency("ultimate_ui")
print("Collecting metadata")
with (self.project.root_dir / "ccmod.json").open(
"r", encoding="utf8") as manifest_file:
manifest = json.load(manifest_file)
mod_id: str = manifest["id"]
mod_version: str = manifest["version"]
mod_files: list[Path] = []
for pattern in self.project.get_conf("distributables",
"mod_files_patterns",
self.project.get_conf_list):
mod_files.extend(
path.relative_to(self.project.root_dir)
for path in self.project.root_dir.glob(pattern))
# Note that paths here are sorted as lists of their components and not as
# strings, so the path separators will not be taken into account when
# sorting.
mod_files.sort()
commiter_time = int(
subprocess.run(
[
"git", "log", "--max-count=1", "--date=unix",
"--pretty=format:%cd"
],
check=True,
stdout=subprocess.PIPE,
cwd=self.project.root_dir,
).stdout)
print("Making packages")
def archive_add_mod_files(archived_prefix: Path) -> None:
print("Adding mod files")
for file in mod_files:
archive.add_real_file(
str(self.project.root_dir / file),
str(archived_prefix / file),
recursive=False,
mtime=commiter_time,
)
def archive_add_dependency(archived_prefix: Path,
dependency_path: Path,
strip_components: int) -> None:
print(f"Adding files from {dependency_path.name}")
with TarFile.gzopen(dependency_path) as dependency_archive:
for file_info in dependency_archive:
archived_path = str(
Path(
archived_prefix,
*Path(file_info.name).parts[strip_components:],
))
if file_info.isreg():
file_reader = dependency_archive.extractfile(file_info)
assert file_reader is not None
archive.add_file_entry(archived_path,
file_reader.read(),
mtime=file_info.mtime)
elif file_info.issym():
archive.add_symlink_entry(archived_path,
file_info.linkname,
mtime=file_info.mtime)
elif file_info.isdir():
# Directories are deliberately ignored because the previous setup
# didn't put them into resulting archives, and their entries are
# useless to us anyway.
pass
else:
# Other file types (character devices, block devices and named
# pipes) are UNIX-specific and can't be handled by Zip, but it's
# not like they are used in dependencies anyway. Correction: well,
# after checking APPNOTE.TXT section 4.5.7 I noticed that these
# exotic file types may be supported, but it's not like any modding
# projects use those.
pass
all_archive_adapters: list[type[ArchiveAdapter]] = [
TarGzArchiveAdapter, ZipArchiveAdapter
]
for archive_cls in all_archive_adapters:
archive_name = f"{mod_id}_v{mod_version}{archive_cls.DEFAULT_EXTENSION}"
print(f"Making {archive_name}")
with archive_cls.create(self.project.dist_archives_dir /
archive_name) as archive:
archive_add_mod_files(Path(mod_id))
# TODO: Sort all files in quick install archives
archive_name = f"{mod_id}_quick-install_v{mod_version}{archive_cls.DEFAULT_EXTENSION}"
print(f"Making {archive_name}")
with archive_cls.create(self.project.dist_archives_dir /
archive_name) as archive:
archive_add_mod_files(Path("assets", "mods", mod_id))
archive_add_dependency(Path("assets", "mods", "Localize-me"),
localize_me_file, 1)
archive_add_dependency(Path("assets", "mods"),
ultimate_ui_file, 0)
archive_add_dependency(Path(), ccloader_file, 0)
def includes_iterator():
for pattern in includes:
for path in search_base.glob(pattern):
# TODO(bhamiltoncx): Handle hidden files on Windows.
if path.is_file() and (include_dotfiles or not path.name.startswith('.')):
yield path.relative_to(search_base)
def __LoadTemplateFiles(self):
files = []
for path in self.TemplateDir.glob('*/**/*.*'):
files.append(str(path.relative_to(self.TemplateDir)))
files.sort()
return files
