def collect_windows(info_add):
try:
import ctypes
except ImportError:
return
if not hasattr(ctypes, 'WinDLL'):
return
ntdll = ctypes.WinDLL('ntdll')
BOOLEAN = ctypes.c_ubyte
try:
RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled
except AttributeError:
res = '<function not available>'
else:
RtlAreLongPathsEnabled.restype = BOOLEAN
RtlAreLongPathsEnabled.argtypes = ()
res = bool(RtlAreLongPathsEnabled())
info_add('windows.RtlAreLongPathsEnabled', res)
try:
import _winapi
dll_path = _winapi.GetModuleFileName(sys.dllhandle)
info_add('windows.dll_path', dll_path)
except (ImportError, AttributeError):
pass
def _find_w9xpopen(self):
w9xpopen = os.path.join(os.path.dirname(_winapi.GetModuleFileName(0)), 'w9xpopen.exe')
if not os.path.exists(w9xpopen):
w9xpopen = os.path.join(os.path.dirname(sys.base_exec_prefix), 'w9xpopen.exe')
if not os.path.exists(w9xpopen):
raise RuntimeError('Cannot locate w9xpopen.exe, which is needed for Popen to work with your shell or platform.')
return w9xpopen
def parallel_optimize_t1(
self,
ir_data,
ti_list,
mask,
im_dims,
n_max,
ir_dim,
lower_bounds,
upper_bounds,
processes=4,
):
ir_data = ir_data.reshape(-1, ir_dim)
mask = mask.reshape(-1, 1)
if sys.platform == "win32":
# mp.spawn.set_executable(_winapi.GetModuleFileName(0))
import _winapi
mp.set_executable(_winapi.GetModuleFileName(0))
pool = mp.Pool(processes=processes) # initialize multiprocess
# run multiprocess and convert to numpy array
results = np.array(
pool.starmap(
self.single_voxel_parallel_optimize,
[
(ir_data[i, :], ti_list, mask[i, :], n_max, lower_bounds, upper_bounds)
for i in range(mask.shape[0])
],
)
)
pool.close()
pool.terminate()
return results.reshape((im_dims[0], im_dims[1], im_dims[2], 2 * n_max))
def start(self):
print(f'Running scene with {len(self.scene_class.values)} instances, on {mp.cpu_count()} processes...')
mp.set_executable(_winapi.GetModuleFileName(0))
start_time = time.perf_counter()
with mp.Pool(mp.cpu_count()) as pool:
self.results = pool.map(self.scene_class.run_scene, self.scene_class.values)
end_time = time.perf_counter()
print(f'Finished in {np.round(end_time - start_time)} seconds.')
self.end()
def venv(known_paths):
global PREFIXES, ENABLE_USER_SITE
env = os.environ
if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env:
executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__']
elif sys.platform == 'win32' and '__PYVENV_LAUNCHER__' in env:
executable = sys.executable
import _winapi
sys._base_executable = _winapi.GetModuleFileName(0)
# bpo-35873: Clear the environment variable to avoid it being
# inherited by child processes.
del os.environ['__PYVENV_LAUNCHER__']
else:
executable = sys.executable
exe_dir, _ = os.path.split(os.path.abspath(executable))
site_prefix = os.path.dirname(exe_dir)
sys._home = None
conf_basename = 'pyvenv.cfg'
candidate_confs = [
conffile for conffile in (
os.path.join(exe_dir, conf_basename),
os.path.join(site_prefix, conf_basename)
)
if os.path.isfile(conffile)
]
if candidate_confs:
virtual_conf = candidate_confs[0]
system_site = "true"
# Issue 25185: Use UTF-8, as that's what the venv module uses when
# writing the file.
with open(virtual_conf, encoding='utf-8') as f:
for line in f:
if '=' in line:
key, _, value = line.partition('=')
key = key.strip().lower()
value = value.strip()
if key == 'include-system-site-packages':
system_site = value.lower()
elif key == 'home':
sys._home = value
sys.prefix = sys.exec_prefix = site_prefix
# Doing this here ensures venv takes precedence over user-site
addsitepackages(known_paths, [sys.prefix])
# addsitepackages will process site_prefix again if its in PREFIXES,
# but that's ok; known_paths will prevent anything being added twice
if system_site == "true":
PREFIXES.insert(0, sys.prefix)
else:
PREFIXES = [sys.prefix]
ENABLE_USER_SITE = False
return known_paths
def test_libc_ver(self):
# check that libc_ver(executable) doesn't raise an exception
if os.path.isdir(sys.executable) and \
os.path.exists(sys.executable+'.exe'):
# Cygwin horror
executable = sys.executable + '.exe'
elif sys.platform == "win32" and not os.path.exists(sys.executable):
# App symlink appears to not exist, but we want the
# real executable here anyway
import _winapi
executable = _winapi.GetModuleFileName(0)
else:
executable = sys.executable
platform.libc_ver(executable)
filename = support.TESTFN
self.addCleanup(support.unlink, filename)
with mock.patch('os.confstr', create=True, return_value='mock 1.0'):
# test os.confstr() code path
self.assertEqual(platform.libc_ver(), ('mock', '1.0'))
# test the different regular expressions
for data, expected in (
(b'__libc_init', ('libc', '')),
(b'GLIBC_2.9', ('glibc', '2.9')),
(b'libc.so.1.2.5', ('libc', '1.2.5')),
(b'libc_pthread.so.1.2.5', ('libc', '1.2.5_pthread')),
(b'', ('', '')),
):
with open(filename, 'wb') as fp:
fp.write(b'[xxx%sxxx]' % data)
fp.flush()
# os.confstr() must not be used if executable is set
self.assertEqual(platform.libc_ver(executable=filename),
expected)
# binary containing multiple versions: get the most recent,
# make sure that 1.9 is seen as older than 1.23.4
chunksize = 16384
with open(filename, 'wb') as f:
# test match at chunk boundary
f.write(b'x' * (chunksize - 10))
f.write(b'GLIBC_1.23.4\0GLIBC_1.9\0GLIBC_1.21\0')
self.assertEqual(platform.libc_ver(filename, chunksize=chunksize),
('glibc', '1.23.4'))
def _create_underpth_exe(self, lines, exe_pth=True):
import _winapi
temp_dir = tempfile.mkdtemp()
self.addCleanup(os_helper.rmtree, temp_dir)
exe_file = os.path.join(temp_dir, os.path.split(sys.executable)[1])
dll_src_file = _winapi.GetModuleFileName(sys.dllhandle)
dll_file = os.path.join(temp_dir, os.path.split(dll_src_file)[1])
shutil.copy(sys.executable, exe_file)
shutil.copy(dll_src_file, dll_file)
if exe_pth:
_pth_file = os.path.splitext(exe_file)[0] + '._pth'
else:
_pth_file = os.path.splitext(dll_file)[0] + '._pth'
with open(_pth_file, 'w') as f:
for line in lines:
print(line, file=f)
return exe_file
def test_symlink(self):
if sys.platform == "win32" and not os.path.exists(sys.executable):
# App symlink appears to not exist, but we want the
# real executable here anyway
import _winapi
real = _winapi.GetModuleFileName(0)
else:
real = os.path.realpath(sys.executable)
link = os.path.abspath(TESTFN)
os.symlink(real, link)
# On Windows, the EXE needs to know where pythonXY.dll is at so we have
# to add the directory to the path.
env = None
if sys.platform == "win32":
env = {k.upper(): os.environ[k] for k in os.environ}
env["PATH"] = "{};{}".format(os.path.dirname(real),
env.get("PATH", ""))
# Requires PYTHONHOME as well since we locate stdlib from the
# EXE path and not the DLL path (which should be fixed)
env["PYTHONHOME"] = os.path.dirname(real)
if sysconfig.is_python_build(True):
env["PYTHONPATH"] = os.path.dirname(os.__file__)
# Issue 7880
def get(python, env=None):
cmd = [
python, '-c',
'import sysconfig; print(sysconfig.get_platform())'
]
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
out, err = p.communicate()
if p.returncode:
print((out, err))
self.fail('Non-zero return code {0} (0x{0:08X})'.format(
p.returncode))
return out, err
try:
self.assertEqual(get(real), get(link, env))
finally:
unlink(link)
def test_architecture_via_symlink(self): # issue3762
if sys.platform == "win32" and not os.path.exists(sys.executable):
# App symlink appears to not exist, but we want the
# real executable here anyway
import _winapi
real = _winapi.GetModuleFileName(0)
else:
real = os.path.realpath(sys.executable)
link = os.path.abspath(support.TESTFN)
os.symlink(real, link)
# On Windows, the EXE needs to know where pythonXY.dll and *.pyd is at
# so we add the directory to the path, PYTHONHOME and PYTHONPATH.
env = None
if sys.platform == "win32":
env = {k.upper(): os.environ[k] for k in os.environ}
env["PATH"] = "{};{}".format(os.path.dirname(real),
env.get("PATH", ""))
env["PYTHONHOME"] = os.path.dirname(real)
if sysconfig.is_python_build(True):
env["PYTHONPATH"] = os.path.dirname(os.__file__)
def get(python, env=None):
cmd = [
python, '-c', 'import platform; print(platform.architecture())'
]
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
r = p.communicate()
if p.returncode:
print(repr(r[0]))
print(repr(r[1]), file=sys.stderr)
self.fail('unexpected return code: {0} (0x{0:08X})'.format(
p.returncode))
return r
try:
self.assertEqual(get(sys.executable), get(link, env=env))
finally:
os.remove(link)
def test_libc_ver(self):
if os.path.isdir(sys.executable) and \
os.path.exists(sys.executable+'.exe'):
# Cygwin horror
executable = sys.executable + '.exe'
elif sys.platform == "win32" and not os.path.exists(sys.executable):
# App symlink appears to not exist, but we want the
# real executable here anyway
import _winapi
executable = _winapi.GetModuleFileName(0)
else:
executable = sys.executable
res = platform.libc_ver(executable)
self.addCleanup(support.unlink, support.TESTFN)
with open(support.TESTFN, 'wb') as f:
f.write(b'x' * (16384 - 10))
f.write(b'GLIBC_1.23.4\0GLIBC_1.9\0GLIBC_1.21\0')
self.assertEqual(platform.libc_ver(support.TESTFN),
('glibc', '1.23.4'))
def collect_windows(info_add):
try:
import ctypes
except ImportError:
return
if not hasattr(ctypes, 'WinDLL'):
return
ntdll = ctypes.WinDLL('ntdll')
BOOLEAN = ctypes.c_ubyte
try:
RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled
except AttributeError:
res = '<function not available>'
else:
RtlAreLongPathsEnabled.restype = BOOLEAN
RtlAreLongPathsEnabled.argtypes = ()
res = bool(RtlAreLongPathsEnabled())
info_add('windows.RtlAreLongPathsEnabled', res)
try:
import _winapi
dll_path = _winapi.GetModuleFileName(sys.dllhandle)
info_add('windows.dll_path', dll_path)
except (ImportError, AttributeError):
pass
import subprocess
try:
# When wmic.exe output is redirected to a pipe,
# it uses the OEM code page
proc = subprocess.Popen(
["wmic", "os", "get", "Caption,Version", "/value"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="oem",
text=True)
output, stderr = proc.communicate()
if proc.returncode:
output = ""
except OSError:
pass
else:
for line in output.splitlines():
line = line.strip()
if line.startswith('Caption='):
line = line.removeprefix('Caption=').strip()
if line:
info_add('windows.version_caption', line)
elif line.startswith('Version='):
line = line.removeprefix('Version=').strip()
if line:
info_add('windows.version', line)
try:
proc = subprocess.Popen(["ver"],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
output = proc.communicate()[0]
if proc.returncode:
output = ""
except OSError:
return
else:
output = output.strip()
line = output.splitlines()[0]
if line:
info_add('windows.ver', line)
if sys.platform != 'win32':
WINEXE = False
WINSERVICE = False
_WINENV = False
else:
WINEXE = getattr(sys, 'frozen', False)
WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
_WINENV = '__PYVENV_LAUNCHER__' in os.environ
if WINSERVICE:
_python_exe = os.path.join(sys.exec_prefix, 'python.exe')
elif _WINENV:
# bpo-35797: When running in a venv, we need to bypass the redirect
# executor and launch our base Python.
import _winapi
_python_exe = _winapi.GetModuleFileName(0)
else:
_python_exe = sys.executable
def set_executable(exe):
global _python_exe
_python_exe = exe
def get_executable():
return _python_exe
#
#
import nltk
#nltk.download('stopwords')
from nltk import pos_tag
from nltk.corpus import stopwords
from collections import Counter
from bs4 import BeautifulSoup
from xgboost import XGBClassifier
# fix for bug in venv on windows, probably starting from python 3.7.2: https://bugs.python.org/issue35797
# this manifests as "PermissionError: [WinError 5] Access is denied" errors.
# this is a workaround
in_virtual_env = sys.prefix != sys.base_prefix
if sys.platform == 'win32' and in_virtual_env and sys.version_info.major == 3 and sys.version_info.minor == 7:
import _winapi
sys.executable = _winapi.GetModuleFileName(0)
this_file_path = os.path.abspath(__file__)
project_root = os.path.split(
os.path.split(os.path.split(this_file_path)[0])[0])[0]
path_root = os.path.join(project_root, "data") + '/'
path_to_cadrs = path_root + 'cadrs/'
path_to_pretrained_wv = path_root
path_to_plot = path_root
path_to_save = path_root
crs_cat = pd.read_csv(os.path.join(path_to_cadrs, 'cadrs_training_rsd.csv'),
delimiter=',')
print('The shape: %d x %d' % crs_cat.shape)
crs_cat.columns