def run_experiment(compiler, opt_levels, directory, number_of_experiments,
csv_file, csv_writer, paths, configs):
for config in configs:
for opt_level in opt_levels:
for path in paths:
try:
exe_path = Path(str(path).replace('.c', '.exe'))
name, permutation = exe_path.stem.rsplit('_', 1)
print(
f"Running: {name}, permutation: {permutation}, optimization: {opt_level}, config: {config}"
)
run(f'{compiler} {path} -O{opt_level} -o {exe_path}',
shell=True,
check=True)
runtimes = []
for i in range(1, number_of_experiments + 1):
output = run(f'{exe_path} {config}',
shell=True,
check=True,
capture_output=True)
runtimes.append(float(output.stdout.decode('ascii')))
csv_writer.writerow(
[name, config, permutation, opt_level] + runtimes +
[median(runtimes), mean(runtimes)])
csv_file.flush()
# fsync()
except Exception as e:
print(f"{name} failed with exception: {e}. Skipping")
finally:
if exe_path.exists():
exe_path.unlink()
for path in paths:
path.unlink()
def _do_generate_directory(self, sources: Tuple[pathlib.Path, ...],
destination: pathlib.Path):
destination_base = destination
logger.info("Generating %s from %s", destination,
', '.join(map(str, sources)))
for base, source in yield_all_sources(*sources):
destination = destination_base / source.relative_to(base)
if source.is_symlink():
self._create_symlink(source, destination)
elif source.is_dir():
if destination.exists():
logger.debug("Clearing directory %s", destination)
# Clear destination directory contents
if destination.is_symlink() or not destination.is_dir():
raise GeneratorError(
"Destination {} is not a directory".format(
destination))
for path in destination.iterdir():
if path.is_dir():
shutil.rmtree(str(path))
else:
path.unlink()
else:
logger.debug("Creating directory %s", destination)
destination.mkdir(self.dir_mode, exist_ok=True)
shutil.copystat(str(source),
str(destination),
follow_symlinks=False)
self._setstat(destination, self.dir_mode)
else:
if source.suffix == self.TEMPLATE_SUFFIX:
destination = destination.with_name(destination.stem)
self._generate_template_to_file(base, source, destination)
else:
self._copy_to_file(source, destination)
def serve_content(self, content, alias=None, as_file=True):
uri = None
if as_file:
filepath, filename = self.get_random_path_at_webroot()
try:
with open(filepath, 'w') as out:
out.write(content)
self.served_files.add(filepath)
except:
if path.isfile(filepath):
path.unlink(filepath)
raise
uri = self.wwwroot + '/' + filename
else:
uri = self.random_path()
self.app.add_handlers('.*', [(uri, StaticTextHandler, {
'content': content
})])
if alias:
self.aliases[uri] = alias
return uri
def remove_file(path: Union[str, Path]):
path = make_path(path)
if path.exists() and path.is_file():
LOGGER.debug("delete file: %s", str(path))
path.unlink()
else:
LOGGER.debug("file does not exist/path is not a file, do nothing")
def _do_generate_directory(self, sources: Tuple[pathlib.Path, ...],
destination: pathlib.Path):
destination_base = destination
logger.info("Generating %s from %s", destination,
', '.join(map(str, sources)))
for base, source in yield_all_sources(*sources):
destination = destination_base / source.relative_to(base)
if source.is_symlink():
self._create_symlink(source, destination)
elif source.is_dir():
if destination.exists():
logger.debug("Clearing directory %s", destination)
# Clear destination directory contents
if destination.is_symlink() or not destination.is_dir():
raise GeneratorError("Destination {} is not a directory"
.format(destination))
for path in destination.iterdir():
if path.is_dir():
shutil.rmtree(str(path))
else:
path.unlink()
else:
logger.debug("Creating directory %s", destination)
destination.mkdir(self.dir_mode, exist_ok=True)
shutil.copystat(str(source), str(destination),
follow_symlinks=False)
self._setstat(destination, self.dir_mode)
else:
if source.suffix == self.TEMPLATE_SUFFIX:
destination = destination.with_name(destination.stem)
self._generate_template_to_file(base, source, destination)
else:
self._copy_to_file(source, destination)
def delete_empty_file(local_filename):
path = Path(local_filename)
try:
size = path.stat().st_size
except FileNotFoundError:
return
if size == 0:
path.unlink()
def _postsetup_autoremove_kernels(self) -> None:
"""Allow apt to autoremove kernels."""
# this file is one APT creates to make sure we don't "autoremove" our
# currently in-use kernel, which doesn't really apply to
# debootstraps/Docker images that don't even have kernels installed
path = self._target / 'etc' / 'apt' / 'apt.conf.d' / '01autoremove-kernels'
try:
path.unlink()
except FileNotFoundError:
pass
def download_file(self, url: str, sha256sum: str) -> Optional[Path]:
path = self._file_url_to_local_path(url)
# Avoid downloading again if we have the file and it matches the hash.
if path.exists():
existing_hash = utils.hash(str(path))
if existing_hash == sha256sum:
return None
else:
logger.info(
"Checksum mismatch with local file {}: "
"expected {} got {}, will re-download.".format(
path, sha256sum, existing_hash
)
)
path.unlink()
# Hack: download from another host
if self.customized_host is not None and url.startswith("https://files.pythonhosted.org/"):
url = url.replace("https://files.pythonhosted.org/", self.customized_host)
logger.info(f"Downloading: {url}")
dirname = path.parent
if not dirname.exists():
dirname.mkdir(parents=True)
# Even more special handling for the serial of package files here:
# We do not need to track a serial for package files
# as PyPI generally only allows a file to be uploaded once
# and then maybe deleted. Re-uploading (and thus changing the hash)
# is only allowed in extremely rare cases with intervention from the
# PyPI admins.
# Py3 sometimes has requests lib return bytes. Need to handle that
r = self.mirror.master.get(url, required_serial=None, stream=True)
checksum = hashlib.sha256()
with utils.rewrite(path, "wb") as f:
for chunk in r.iter_content(chunk_size=64 * 1024):
checksum.update(chunk)
f.write(chunk)
existing_hash = checksum.hexdigest()
if existing_hash == sha256sum:
# Good case: the file we got matches the checksum we expected
pass
else:
# Bad case: the file we got does not match the expected
# checksum. Even if this should be the rare case of a
# re-upload this will fix itself in a later run.
raise ValueError(
"Inconsistent file. {} has hash {} instead of {}.".format(
url, existing_hash, sha256sum
)
)
return path
def unlink_parent_dir(path: Path) -> None:
""" Remove a file and if the dir is empty remove it """
logger.info(f"unlink {str(path)}")
path.unlink()
parent_path = path.parent
try:
parent_path.rmdir()
logger.info(f"rmdir {str(parent_path)}")
except OSError as oe:
logger.debug(f"Did not remove {str(parent_path)}: {str(oe)}")
async def download_file(self,
url: str,
sha256sum: str,
chunk_size: int = 64 * 1024) -> Optional[Path]:
path = self._file_url_to_local_path(url)
# Avoid downloading again if we have the file and it matches the hash.
if path.exists():
existing_hash = utils.hash(str(path))
if existing_hash == sha256sum:
return None
else:
logger.info(
f"Checksum mismatch with local file {path}: expected {sha256sum} "
+ f"got {existing_hash}, will re-download.")
path.unlink()
logger.info(f"Downloading: {url}")
dirname = path.parent
if not dirname.exists():
dirname.mkdir(parents=True)
# Even more special handling for the serial of package files here:
# We do not need to track a serial for package files
# as PyPI generally only allows a file to be uploaded once
# and then maybe deleted. Re-uploading (and thus changing the hash)
# is only allowed in extremely rare cases with intervention from the
# PyPI admins.
r_generator = self.mirror.master.get(url, required_serial=None)
response = await r_generator.asend(None)
checksum = hashlib.sha256()
with utils.rewrite(path, "wb") as f:
while True:
chunk = await response.content.read(chunk_size)
if not chunk:
break
checksum.update(chunk)
f.write(chunk)
existing_hash = checksum.hexdigest()
if existing_hash != sha256sum:
# Bad case: the file we got does not match the expected
# checksum. Even if this should be the rare case of a
# re-upload this will fix itself in a later run.
raise ValueError(
f"Inconsistent file. {url} has hash {existing_hash} " +
f"instead of {sha256sum}.")
return path
async def run(self) -> None:
path = self.node.path
if os.path.lexists(str(path)):
path.unlink()
self.logger.debug(
"create directory <ITALIC>%(path)s<UPRIGHT>%(parents)s",
dict(
path=self.node.relative_path,
parents=' with parents' if self.parents else '')
)
# rwxr-xr-x
path.mkdir(mode=0b111101101, parents=self.parents)
self.node.modified = True
self.node.updated = True
def delete_files(directory, filenames, files_to_keep=()):
ensure_overwritable(*[directory.joinpath(f) for f in filenames])
# We implement the "files to keep" logic using inodes rather than names so
# we can safely handle case-insenstiive filesystems
inodes_to_keep = set()
for filename in files_to_keep:
try:
stat = directory.joinpath(filename).stat()
inodes_to_keep.add((stat.st_dev, stat.st_ino))
except FileNotFoundError:
pass
for filename in filenames:
path = directory / filename
try:
stat = path.stat()
except FileNotFoundError:
continue
inode = (stat.st_dev, stat.st_ino)
if inode not in inodes_to_keep:
path.unlink()
def main(verbose, web, db):
"""
remove blacklisted projects from the mirror
"""
web = os.path.expanduser(web)
if db is None:
db = os.path.join(web, "pypi.db")
else:
db = os.path.expanduser(db)
bl = pickle.load(open("blacklist.cache", "rb"))[0]
conn = sqlite3.connect(db)
web = pathlib.Path(web)
total = 0
count = 0
for name, url, size in conn.execute("select name,url,size from file"):
if name not in bl:
continue
url = urlparse(url).path[1:]
path = web / url
if path.exists():
path.unlink()
try:
path.parent.rmdir()
except OSError:
# not empty, should not occur
pass
total += size
count += 1
if verbose:
print(f"removed: {path} ({size} bytes)")
print(f"files removed: {count}")
print(f"space freed: {hf.format_size(total)}")
def record_video(self):
print('Recording')
url = 'http://127.0.0.1:8080/stream/video.mjpeg'
local_filename = url.split('/')[-1]
# NOTE the stream=True parameter
r = requests.get(url, stream=True)
filename = time.strftime("%Y%m%d-%H%M%S") + '.mp4'
save_path = '/home/pi/Downloads/tesis/video'
completed_video = os.path.join(save_path, filename)
i = 0
with open(completed_video, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
i = i + 1
#f.flush() commented by recommendation from J.F.Sebastian
if (self.recording == False or i == 10000):
print('Sending')
f.close()
with open(completed_video, 'rb') as f:
video = f.read()
encode_video = base64.encodebytes(video)
json = {
'ip_address': '10.10.10.110',
'date': time.strftime('%Y-%m-%d %H:%M:%S'),
'video': encode_video
}
r = self.api.post(json, 'createvideo')
a = r.json()
print(a)
path = pathlib.Path(completed_video)
path.unlink()
f.close()
break
return local_filename
if db_file is None:
db_file = p.abspath(p.join(p.dirname(__file__), "sounds.db"))
global engine, db_session
engine = create_engine("sqlite:////%s" % (db_file,), convert_unicode=True)
# check if db file existst, else create db
if p.isfile(db_file) is False:
create_db(engine)
db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine))
def create_db(engine):
Base.metadata.create_all(bind=engine, checkfirst=True)
if __name__ == "__main__":
db_file = p.abspath(p.join(os.getcwd(), "sounds.db"))
if p.isfile(db_file):
p.unlink(db_file)
connect_db(db_file)
print("Engine: ", engine)
create_db(engine)
store_poetry("to be or not to be")
db_session.commit()
all_poetry = db_session.query(Poetry).all()
for pp in all_poetry:
print("%s spoken at %s" % (pp.poetry, pp.sounded_at))
def sync_handler(sync_val: str, repository_val=None, requested_bundle_name=None):
sync_val = sync_val.lower()
if sync_val == "all" or sync_val == "force":
# Firstly, check our remote connection before doing anything
remote_state, remote_url = pbgit.check_remote_connection()
if not remote_state:
pbtools.error_state(
f"Remote connection was not successful. Please verify that you have a valid git remote URL & internet connection. Current git remote URL: {remote_url}")
else:
pblog.info("Remote connection is up")
pblog.info("------------------")
pblog.info(f"Executing {sync_val} sync command")
pblog.info(f"PBpy Library Version: {pbpy_version.ver}")
pblog.info(f"PBSync Program Version: {pbsync_version.ver}")
pblog.info("------------------")
detected_git_version = pbgit.get_git_version()
needs_git_update = False
if detected_git_version == pbconfig.get('supported_git_version'):
pblog.info(f"Current Git version: {detected_git_version}")
else:
pblog.error("Git is not updated to the supported version in your system")
pblog.error(f"Supported Git Version: {pbconfig.get('supported_git_version')}")
pblog.error(f"Current Git Version: {detected_git_version}")
pblog.error("Please install the supported Git version from https://github.com/microsoft/git/releases")
pblog.error("Visit https://github.com/ProjectBorealisTeam/pb/wiki/Prerequisites for installation instructions")
if os.name == "nt":
webbrowser.open(f"https://github.com/microsoft/git/releases/download/v{pbconfig.get('supported_git_version')}/Git-{pbconfig.get('supported_git_version')}-64-bit.exe")
needs_git_update = True
if os.name == "nt":
# find Git/cmd/git.exe
git_paths = [path for path in pbtools.whereis("git") if "cmd" in path.parts]
if len(git_paths) > 0:
bundled_git_lfs = False
is_admin = pbuac.isUserAdmin()
delete_paths = []
for git_path in git_paths:
# find Git from Git/cmd/git.exe
git_root = git_path.parents[1]
possible_lfs_paths = ["cmd/git-lfs.exe", "mingw64/bin/git-lfs.exe", "mingw64/libexec/git-core/git-lfs.exe"]
for possible_lfs_path in possible_lfs_paths:
path = git_root / possible_lfs_path
if path.exists():
try:
if is_admin:
path.unlink()
else:
delete_paths.append(str(path))
except FileNotFoundError:
pass
except OSError:
pblog.error(f"Git LFS is bundled with Git, overriding your installed version. Please remove {path}.")
bundled_git_lfs = True
if not is_admin and len(delete_paths) > 0:
pblog.info("Requesting permission to delete bundled Git LFS which is overriding your installed version...")
quoted_paths = [f'"{path}"' for path in delete_paths]
delete_cmdline = ["cmd.exe", "/c", "DEL", "/q", "/f"] + quoted_paths
try:
ret = pbuac.runAsAdmin(delete_cmdline)
except OSError:
pblog.error("User declined permission. Automatic delete failed.")
for delete_path in delete_paths:
path = pathlib.Path(delete_path)
if path.exists():
bundled_git_lfs = True
pblog.error(f"Git LFS is bundled with Git, overriding your installed version. Please remove {path}.")
if bundled_git_lfs:
pbtools.error_state()
detected_lfs_version = pbgit.get_lfs_version()
if detected_lfs_version == pbconfig.get('supported_lfs_version'):
pblog.info(f"Current Git LFS version: {detected_lfs_version}")
else:
pblog.error("Git LFS is not updated to the supported version in your system")
pblog.error(f"Supported Git LFS Version: {pbconfig.get('supported_lfs_version')}")
pblog.error(f"Current Git LFS Version: {detected_lfs_version}")
pblog.error("Please install the supported Git LFS version from https://git-lfs.github.com")
if os.name == "nt":
supported_lfs_version = pbconfig.get('supported_lfs_version').split("/")[1]
webbrowser.open(f"https://github.com/git-lfs/git-lfs/releases/download/v{supported_lfs_version}/git-lfs-windows-v{supported_lfs_version}.exe")
needs_git_update = True
if needs_git_update:
pbtools.error_state()
pblog.info("------------------")
# Do not execute if Unreal Editor is running
if pbtools.get_running_process("UE4Editor") is not None:
pbtools.error_state("Unreal Editor is currently running. Please close it before running PBSync. It may be listed only in Task Manager as a background process. As a last resort, you should log off and log in again.")
current_branch = pbgit.get_current_branch_name()
# repo was already fetched in StartProject.bat
if current_branch != "promoted":
pblog.info("Fetching recent changes on the repository...")
fetch_base = ["git", "fetch", "origin"]
branches = {"promoted", "master", "trunk", current_branch}
fetch_base.extend(branches)
pbtools.get_combined_output(fetch_base)
# Do some housekeeping for git configuration
pbgit.setup_config()
# Check if we have correct credentials
pbgit.check_credentials()
pblog.info("------------------")
# Execute synchronization part of script if we're on the expected branch, or force sync is enabled
is_on_expected_branch = pbgit.compare_with_current_branch_name(pbconfig.get('expected_branch_name'))
if sync_val == "force" or is_on_expected_branch:
pbtools.resolve_conflicts_and_pull()
pblog.info("------------------")
project_version = pbunreal.get_project_version()
if project_version is not None:
pblog.info(f"Current project version: {project_version}")
else:
pbtools.error_state(
"Something went wrong while fetching project version. Please request help in #tech-support.")
if pbhub.is_pull_binaries_required():
pblog.info("Binaries are not up to date, trying to pull new binaries...")
ret = pbhub.pull_binaries(project_version)
if ret == 0:
pblog.info("Binaries were pulled successfully")
elif ret < 0:
pbtools.error_state("Binaries pull failed, please view log for instructions.")
elif ret > 0:
pbtools.error_state("An error occurred while pulling binaries. Please request help in #tech-support to resolve it, and please do not run StartProject.bat until the issue is resolved.", True)
else:
pblog.info("Binaries are up-to-date")
else:
pblog.warning(f"Current branch is not supported for repository synchronization: {pbgit.get_current_branch_name()}. Auto synchronization "
"will be disabled")
pblog.info("------------------")
pblog.info("Checking for engine updates...")
if pbgit.sync_file("ProjectBorealis.uproject") != 0:
pbtools.error_state(
"Something went wrong while updating the .uproject file. Please request help in #tech-support.")
engine_version = pbunreal.get_engine_version(False)
pblog.info("Trying to register current engine build if it exists. Otherwise, the build will be downloaded...")
symbols_needed = pbunreal.is_versionator_symbols_enabled()
bundle_name = pbconfig.get("ue4v_default_bundle")
if pbunreal.run_ue4versionator(bundle_name, symbols_needed) != 0:
pblog.error(f"Something went wrong while registering engine build {bundle_name}-{engine_version}. Please request help in #tech-support.")
sys.exit(1)
else:
pblog.info(f"Engine build {bundle_name}-{engine_version} successfully registered")
# Clean old engine installations, do that only in expected branch
if is_on_expected_branch:
if pbunreal.clean_old_engine_installations():
pblog.info("Old engine installations are successfully cleaned")
else:
pblog.warning("Something went wrong while cleaning old engine installations. You may want to clean them manually.")
pblog.info("------------------")
if pbunreal.check_ue4_file_association():
try:
os.startfile(os.path.normpath(os.path.join(os.getcwd(), "ProjectBorealis.uproject")))
except NotImplementedError:
pblog.info("You may now launch ProjectBorealis.uproject with Unreal Engine 4.")
else:
pbtools.error_state(".uproject extension is not correctly set into Unreal Engine. Make sure you have Epic Games Launcher installed. If problem still persists, please get help in #tech-support.")
elif sync_val == "engineversion":
if repository_val is None:
pblog.error("--repository <URL> argument should be provided with --sync engine command")
sys.exit(1)
engine_version = pbunreal.get_latest_available_engine_version(str(repository_val))
if engine_version is None:
pblog.error("Error while trying to fetch latest engine version")
sys.exit(1)
if not pbunreal.set_engine_version(engine_version):
pblog.error("Error while trying to update engine version in .uproject file")
sys.exit(1)
pblog.info(f"Successfully changed engine version as {str(engine_version)}")
elif sync_val == "ddc":
pbunreal.generate_ddc_data()
elif sync_val == "binaries":
project_version = pbunreal.get_project_version()
ret = pbhub.pull_binaries(project_version, True)
if ret == 0:
pblog.info(f"Binaries for {project_version} pulled & extracted successfully")
else:
pblog.error(f"Failed to pull binaries for {project_version}")
sys.exit(1)
elif sync_val == "engine":
# Pull engine build with ue4versionator & register it
if requested_bundle_name is None:
requested_bundle_name = pbconfig.get("ue4v_default_bundle")
engine_version = pbunreal.get_engine_version(False)
if pbunreal.run_ue4versionator(requested_bundle_name) != 0:
pblog.error(f"Something went wrong while registering engine build {requested_bundle_name}-{engine_version}")
sys.exit(1)
else:
pblog.info(f"Engine build {requested_bundle_name}-{engine_version} successfully registered")
async def EnsureDeleteFile(ver: AbstractFileVersion, fb: AbstractFeedbackUI):
path = pathlib.Path(version.GetAbsolutePath())
if path.exists():
fb.feedback("Deleting file...")
path.unlink()
fb.feedback("Done.")
