def pathlib_func():
'''
适合于不同操作系统的类来对文件系统路径进行操作
'''
# === PurePath (纯路径) ===
# PurePath(*pathsegments) // 系统地灵分割
path_pure = pathlib.PurePath('hello.py')
path_pure = pathlib.PurePath('python', 'hello.py')
path_pure = pathlib.PurePath(pathlib.Path('python'),
pathlib.Path('hello.py'))
# PurePosixPath(*pathsegments) // 非windows文件系统路径
# PureWindowsPath(*pathsegments) // windows文件系统路径
# 路径是不可变哈希的, 同风格的路径是比较有序的
pathlib.PurePosixPath('hello') == pathlib.PurePosixPath(
'Hello') # => Flase (PurePosixPath分大小写)
pathlib.PureWindowsPath('hello') == pathlib.PureWindowsPath(
'hello') # => True (PureWindowsPath不分大小写)
pathlib.PureWindowsPath('hello') in {pathlib.PureWindowsPath('Hello')
} # => True
pathlib.PureWindowsPath('C:') < pathlib.PureWindowsPath('d:') # => True
# 不同风格的路径不能比较有序
pathlib.PureWindowsPath('hello') == pathlib.PurePosixPath(
'hello') # => False
# pathlib.PureWindowsPath('hello') < pathlib.PurePosixPath('hello')
# 运算符
path_pure = pathlib.PurePath('c:/')
path_pure = path_pure / 'python' / 'hello.py' # 拼接路径
tups = path_pure.parts # 返回组件元组 => ('c:\\', 'python', 'hello.py')
drive = path_pure.drive # 盘符 => 'c:'
root = path_pure.root # 根目录 => '\\'
driroot = path_pure.anchor # 盘符 + 根目录 => 'c:\\'
parents = path_pure.parents # 父路径列表 parents[0] => PureWindowsPath('c:/python')
parent = path_pure.parent # 父路径 => PureWindowsPath('c:/python')
name = path_pure.name # 文件(夹)名
suffix = path_pure.suffix # 扩展名
suffixs = path_pure.suffixes # 扩展名列表
name = path_pure.stem # 不带后缀名的文件(夹)名 => 'hello'
path = path_pure.as_posix() # 以/返回路径
path = path_pure.as_uri(
) # 以file URL方式返回路径, 必须是绝对路径 => 'file:///c:/python/hello.py'
path = path_pure.joinpath(
"a.txt",
"b.txt") # 拼接路径 => PureWindowsPath('c:/python/hello.py/a.txt/b.txt')
path = path_pure.with_name(
"world.py") # 更改路径文件(夹)名 => PureWindowsPath('c:/python/world.py')
path = path_pure.with_suffix(".txt") # 更改扩展名,没有则添加
boolean = path_pure.match("*.py") # 匹配通配符 (从右侧进行匹配) (大小写的区分见上方比较代码)
boolean = path_pure.is_absolute() # 是否是绝对路径 (/ // c:/ 开头都被认为是绝对路径)
# === Path (具体路径) ===
# Path(*pathsegments)
path = pathlib.Path('hello.py')
# PosixPath(*pathsegments)
# WindowsPath(*pathsegments)
path_s = path.cwd() # 当前路径
home = path.home() # 用户主目录
state = path.stat() # 该路径的状态信息 同os.stat()
state = path.lstat() # 同stat(), 目标是软链接将返回软链接信息
path.chmod(777) # 修改权限模式 同os.chmod()
path.lchmod(777) # 同chmod(), 目标是软链接将更改软链接模式
lists = path.glob(r'*.py') # 迭代器, 所有py文件 (windows失效)
group = path.group() # 文件的组 (windows不可用)
iter = path.iterdir() # 遍历目录 (必须是目录)
# mkdir(mode=0o777, parents=False, exist_ok=False)
path.mkdir() # 创建文件夹
path.rmdir() # 删除文件夹
# open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) // 打开文件
f = path.open()
bytes = path.read_bytes() # 读取文件内容
num = path.write_bytes(b"hello") # (覆盖)写入文件内容
# read_text(encoding=None, errors=None)
strs = path.read_text(encoding='utf-8')
# write_text(data, encoding=None, errors=None)
num = path.write_text("hello")
ow = path.owner() # 该文件拥有者 (windows不可用)
path.rename("new.txt") # 重命名,目标存在则替换
path.replace("new.txt") # 重命名,目标存在则替换
path_s = path.resolve() # 绝对路径
path.symlink_to(
"hello.py", target_is_directory=False
) # 创建软链接, 如果目标是目录, 则(windows下)target_is_directory=True (windows失败)
path.unlink() # 删除文件/文件软链接
# touch(mode=0o666, exist_ok=True)
path.touch() # 创建文件
boolean = path.exists() # 文件(夹)是否存在
boolean = path.is_dir() # 是否是文件夹
boolean = path.is_file() # 是否是文件
boolean = path.is_symlink() # 是否是硬链接
boolean = path.is_block_device() # 是否是块设备
boolean = path.is_char_device # 是否是字符设备
boolean = path.samefile(path) # path是否是同一文件
def resolve_wdir(wdir, path):
rel_wdir = relpath(wdir, os.path.dirname(path))
return pathlib.PurePath(rel_wdir).as_posix() if rel_wdir != "." else None
def initialize_config_and_execute(config_values):
global configs
configs = config_values
# Applying checks for paths
p1 = pathlib.PurePath(configs['DICOMHome'])
dicom_home = p1.as_posix() # the folder containing your dicom files
p2 = pathlib.PurePath(configs['OutputDirectory'])
output_directory = p2.as_posix()
print_images = configs['PrintImages']
print_only_common_headers = bool(configs['CommonHeadersOnly'])
depth = int(configs['Depth'])
processes = int(configs['UseProcesses']) # how many processes to use.
flattened_to_level = configs['FlattenedToLevel']
email = configs['YourEmail']
send_email = configs['SendEmail']
no_splits = int(configs['SplitIntoChunks'])
is16Bit = bool(configs['is16Bit'])
metadata_col_freq_threshold = 0.1
nifti_destination = output_directory + '/extracted-images/'
failed = output_directory + '/failed-dicom/'
maps_directory = output_directory + '/maps/'
meta_directory = output_directory + '/meta/'
LOG_FILENAME = output_directory + '/ImageExtractor.out'
pickle_file = output_directory + '/ImageExtractor.pickle'
dict_pickle_file = output_directory + '/ImageExtractork_dict.pickle'
# record the start time
t_start = time.time()
if not os.path.exists(output_directory):
os.makedirs(output_directory)
logging.basicConfig(filename=LOG_FILENAME, level=logging.DEBUG)
if not os.path.exists(maps_directory):
os.makedirs(maps_directory)
if not os.path.exists(meta_directory):
os.makedirs(meta_directory)
if not os.path.exists(nifti_destination):
os.makedirs(nifti_destination)
if not os.path.exists(failed):
os.makedirs(failed)
if not os.path.exists(failed + "/1"):
os.makedirs(failed + "/1")
if not os.path.exists(failed + "/2"):
os.makedirs(failed + "/2")
if not os.path.exists(failed + "/3"):
os.makedirs(failed + "/3")
if not os.path.exists(failed + "/4"):
os.makedirs(failed + "/4")
logging.info("------- Values Initialization DONE -------")
final_res = execute(pickle_file, dicom_home, output_directory, print_images, print_only_common_headers, depth,
processes, flattened_to_level, email, send_email, no_splits, is16Bit, nifti_destination,
failed, maps_directory, meta_directory, LOG_FILENAME, metadata_col_freq_threshold, t_start,dict_pickle_file)
return final_res
def pre_spray_info(self):
"""
Display spray config table
"""
spray_info = Table(
show_header=False,
show_footer=False,
min_width=61,
title=f"Module: {self.module.upper()}",
title_justify="left",
title_style="bold reverse",
)
spray_info.add_row("Target", f"{self.target.url}")
if self.domain:
spray_info.add_row("Domain", f"{self.domain}")
if self.attempts:
spray_info.add_row("Interval", f"{self.interval} minutes")
spray_info.add_row("Attempts", f"{self.attempts} per interval")
if self.jitter:
spray_info.add_row("Jitter",
f"{self.jitter_min}-{self.jitter} seconds")
if self.notify:
spray_info.add_row("Notify", f"True ({self.notify})")
log_name = pathlib.PurePath(self.log_name)
out_name = pathlib.PurePath(self.output)
spray_info.add_row("Logfile", f"{log_name.name}")
spray_info.add_row("Results", f"{out_name.name}")
console.print(spray_info)
print()
Confirm.ask(
"[blue]Press enter to begin",
default=True,
show_choices=False,
show_default=False,
)
print()
if self.module == "Smb":
console.print(f"[*] Initiaing SMB connection to {self.host} ...",
style="warning")
if self.target.get_conn():
console.print(
f'[+] Connected to {self.host} over {"SMBv1" if self.target.smbv1 else "SMBv3"}',
style="good",
)
console.print(f"\t[>] Hostname: {self.target.hostname} ",
style="info")
console.print(f"\t[>] Domain: {self.target.domain} ",
style="info")
console.print(f"\t[>] OS: {self.target.os} ", style="info")
print()
else:
console.print(f"[!] Failed to connect to {self.host} over SMB",
style="danger")
exit()
self.target.print_headers(self.output)
def prepare_repository(
package,
version,
package_path,
source_repo,
dest_repo,
http_root,
skip_images,
skip_cli
):
dest_path = dest_repo / package_path.relative_to(source_repo)
shutil.copytree(str(package_path), str(dest_path))
dest_resource = dest_path / 'resource.json'
with dest_resource.open('w', encoding='utf-8') as dest_file:
resource = load_json(package_path / 'resource.json')
# Change the root for images (ignore screenshots)
if not skip_images and 'images' in resource:
resource["images"] = {
n: urllib.parse.urljoin(
http_root, str(pathlib.PurePath(
package, version, "images", pathlib.Path(uri).name)))
for n, uri in resource.get("images", {}).items() if 'icon' in n}
# Change the root for asset uris.
if 'assets' in resource:
resource["assets"]["uris"] = {
n: urllib.parse.urljoin(
http_root, str(pathlib.PurePath(
package, version, "uris", pathlib.Path(uri).name)))
for n, uri in resource["assets"].get("uris", {}).items()}
# Change the root for cli uris.
if not skip_cli and 'cli' in resource:
for os_type, arch_dict in \
resource.get('cli', {}).get('binaries', {}).items():
for arch in arch_dict.items():
uri = arch[1]["url"]
arch[1]["url"] = urllib.parse.urljoin(
http_root,
str(
pathlib.PurePath(
package,
version,
"uris",
os_type,
pathlib.Path(uri).name)))
# Add the local docker repo prefix.
if 'assets' in resource:
if 'container' in resource["assets"]:
resource["assets"]["container"]["docker"] = {
n: format_image_name(DOCKER_ROOT, image_name)
for n, image_name in resource["assets"]["container"].get(
"docker", {}).items()}
json.dump(resource, dest_file, indent=4)
command_path = (package_path / 'command.json')
if not command_path.exists():
return
dest_command = dest_path / 'command.json'
with dest_command.open('w', encoding='utf-8') as dest_file:
command = load_json(command_path)
command['pip'] = [
urllib.parse.urljoin(
http_root,
str(
pathlib.PurePath(
package,
version,
"commands",
pathlib.Path(uri).name
)
)
)
for uri in command.get("pip", [])
]
json.dump(command, dest_file, indent=4)
def issubdir(path):
"""Whether path is equal to or is a subdirectory of root."""
path = pathlib.PurePath(path)
return path == root or any(parent == root for parent in path.parents)
import os, glob, pathlib, stat
from pwd import getpwuid
from pathlib import Path
for path, subdirs, files in os.walk("/"):
for name in files:
current = str(pathlib.PurePath(path, name))
if Path(current).is_file():
size = os.stat(current).st_size
if size == 33:
print(getpwuid(stat(current).st_uid).pw_name)
from pathlib import Path
import pathlib
# 列出子目录
p = Path(".")
a = [x for x in p.iterdir() if x.is_dir()]
# print(a)
# 列出当前f盘的所有py文件
abc = list(p.glob("**\*.py"))
# print(abc)
# 查看快捷方式
p = Path(r"C:\Users\Administrator\Desktop\Everything.exe")
# print(p)
# print("="*30)
# print(p.resolve())
# 查询路径属性
q = Path(r'F:\PythonProject')
# print(q.exists())
# print(q.is_dir())
# 拼接路径
pin = pathlib.PurePath("test", "abc/dfa", "tes")
# print(pin)
# print(pathlib.PurePath(Path("目录1"),Path("子目录1")))
# 使用当前目录
# currentpath = pathlib.PurePath()
def get_root():
root = pathlib.PurePath(
input("What's the full path where you'd like the project? "))
if not root.is_absolute():
return os.path.abspath(root)
return root
def join(*args):
return pathlib.PurePath(oldjoin(*args)).as_posix()
ALL_LOGS = LOGS_DIR + "/logs"
shutil.rmtree(ALL_LOGS, ignore_errors=True)
LOG_DIR = ALL_LOGS + "/" + EXP_ID
os.makedirs(LOG_DIR)
CONTROLLER_PID = "CONTROLLER_PID"
### By default, we allocate 3 CPUs per run.
### Recall that we are running 3 processes, and they are multithreaded.
CPUS = 3
TIMEOUT_SUT_START_MINUTES = 20
if not CLUSTER:
REPORT_DIR = str(pathlib.PurePath(REPORT_DIR).as_posix())
SCRIPT_DIR = str(pathlib.PurePath(SCRIPT_DIR).as_posix())
TEST_DIR = str(pathlib.PurePath(TEST_DIR).as_posix())
LOG_DIR = str(pathlib.PurePath(LOG_DIR).as_posix())
def createRunallScript():
script_path = BASE_DIR + "/runall.sh"
script = open(script_path, "w")
script.write("#!/bin/bash \n\n")
script.write("DIR=`dirname \"$0\"` \n\n")
script.write("for s in `ls $DIR/scripts/*.sh`; do\n")
def get_root():
root = pathlib.PurePath(
input("What's the full path where you'd like the project? "))
if not root.is_absolute():
return get_root() # we recall the function till we get what we want
return root
def __init__(self, parent = None):
"""Create a wizard or the mainwindow"""
self._parent = parent
super().__init__(self._parent)
print("runner/Main parent: ", self._parent, " -> self: ", self) if oPB.PRINTHIER else None
self.logger = None
self.args = self.get_args()
self._log_level = None
self._log_file = None
self.translator = None
# make it really quiet, part 1
if self.args.quiet:
self.args.nogui = True
# instantiate configuration class
confighandler.ConfigHandler(oPB.CONFIG_INI)
# redirect system exception hook
if not self.args.noexcepthook:
sys.excepthook = self.excepthook
# pre-instantiating the application, avoid some nasty OpenGL messages
QApplication.setAttribute(QtCore.Qt.AA_UseOpenGLES, on = True)
# create new application and install stylesheet
self.app = QApplication(sys.argv)
#self.install_stylesheet()
# Create and display the splash screen, if in ui mode
if not self.args.nogui:
splash_pix = QPixmap(':/images/splash.png')
self.splash = QSplashScreen(splash_pix, QtCore.Qt.WindowStaysOnTopHint)
self.splash.setMask(splash_pix.mask())
# splash.showMessage("opsi Package Builder " + oPB.PROGRAM_VERSION + " " + translate("Main", "is loading..."), QtCore.Qt.AlignCenter, QtCore.Qt.white)
self.splash.show()
self.app.processEvents()
# Application name
self.app.setOrganizationName("opsi Package Builder")
self.app.setApplicationName("opsi Package Builder " + oPB.PROGRAM_VERSION)
# save ourselves in main instance property to be easily accessd via qApp
# i.e.
# from PyQt5.QtWidgets import qApp
# main = qApp.property("main")
self.app.setProperty("main", self)
if confighandler.ConfigHandler.cfg.log_always == "True":
if self.args.log_file is not None:
self._log_file = self.args.log_file
else:
self._log_file = confighandler.ConfigHandler.cfg.log_file
if self.args.log_level.upper() != "NOTSET":
self._log_level = self.args.log_level.upper()
else:
self._log_level = confighandler.ConfigHandler.cfg.log_level
else:
self._log_file = self.args.log_file
if self.args.log_level.upper() == "NOTSET":
self._log_level = "CRITICAL"
else:
self._log_level = self.args.log_level.upper()
if self._log_file is not None:
if not pathlib.Path(self._log_file).is_absolute():
if platform.system() == "Windows":
self._log_file = str(pathlib.PurePath(oPB.WIN_TMP_PATH, self._log_file))
if platform.system() in ["Darwin", "Linux"]:
self._log_file = str(pathlib.PurePath(oPB.UNIX_TMP_PATH, self._log_file))
# overwrite development directory from config with command line arg
if self.args.dev_dir is not None:
confighandler.ConfigHandler.cfg.dev_dir = self.args.dev_dir
# Initialize the logger and reroute QtCore messages to it
self.logWindow = oPB.gui.logging.LogDialog(None, self, self._log_level)
self.instantiate_logger(False)
QtCore.qInstallMessageHandler(self.qt_message_handler)
# log program version and user
self.logger.info(80 * "-")
self.logger.info("opsi PackageBuilder (MIT licensed) " + oPB.PROGRAM_VERSION)
self.logger.info("Current user: "******"Command line arguments given:")
for key, val in vars(self.args).items():
self.logger.info(" " + key + ": " + str(val))
if self._log_level not in ["DEBUG", "INFO", "SSHINFO", "WARNING", "ERROR", "CRITICAL", "SSH"]:
self.logger.error(" Undefined log level: " + self._log_level)
self.logger.error(" Log level has been set to ERROR")
for elem in self.app.libraryPaths():
self.logger.debug("QT5 library path: " + elem)
# write config to log, if necessary
confighandler.ConfigHandler.cfg.log_config()
self.check_online_status()
# -----------------------------------------------------------------------------------------
# main ui dispatching
# startup gui variant
if not self.args.nogui:
# install stylesheet
self.install_stylesheet()
# hide console window, but only under Windows and only if app is frozen
if sys.platform.lower().startswith('win'):
if getattr(sys, 'frozen', False):
hideConsole()
# installing translators
self.translator = Translator(self.app, "opsipackagebuilder")
self.translator.install_translations(confighandler.ConfigHandler.cfg.language)
# retranslate logWindow, as it is loaded before the translations
self.logWindow.retranslateUi(self.logWindow)
# create app icon
app_icon = QtGui.QIcon()
app_icon.addFile(':images/prog_icons/opb/package_16x16.png', QtCore.QSize(16, 16))
app_icon.addFile(':images/prog_icons/opb/package_24x24.png', QtCore.QSize(24, 24))
app_icon.addFile(':images/prog_icons/opb/package_32x32.png', QtCore.QSize(32, 32))
app_icon.addFile(':images/prog_icons/opb/package_48x48.png', QtCore.QSize(48, 48))
app_icon.addFile(':images/prog_icons/opb/package_64x64.png', QtCore.QSize(64, 64))
app_icon.addFile(':images/prog_icons/opb/package_92x92.png', QtCore.QSize(92, 92))
app_icon.addFile(':images/prog_icons/opb/package_128x128.png', QtCore.QSize(128, 128))
app_icon.addFile(':images/prog_icons/opb/package_256x256.png', QtCore.QSize(256, 256))
self.app.setProperty("prog_icon", app_icon)
# startup program window
self.mainWindow = main.MainWindowController(self.args)
self.mainWindow.ui.showLogRequested.connect(self.logWindow.show)
self.mainWindow.closeAppRequested.connect(self.logWindow.close)
self.splash.finish(self.mainWindow.ui)
# check for updates if configured
if confighandler.ConfigHandler.cfg.updatecheck == "True":
self.mainWindow.update_check()
# run main app loop
self.app.exec_()
# only process commandline
else:
self.logger.info("No-GUI parameter set")
# startup program window
self.console = console.ConsoleController(self.args)
# -----------------------------------------------------------------------------------------
# unmount drive (if exist) after end of program
if (oPB.NETDRV is not None) and oPB.NETDRV != "offline":
ret = MapDrive.unMapDrive(oPB.NETDRV)
if ret[0] != 0:
self.logger.error("Error unmounting path: " + str(ret))
else:
self.logger.info("Network drive successfully unmounted")
# exit and set return code
self.logger.info("Exit code: " + str(oPB.EXITCODE))
# show console window
if not self.args.nogui:
if sys.platform.lower().startswith('win'):
if getattr(sys, 'frozen', False):
showConsole()
sys.exit(oPB.EXITCODE)
import requests
import pathlib
from win32com import client as win_client
# 工作目录（当前路径调试时需加上.parent）
BASE_DIR = str(pathlib.Path.cwd())
# BASE_DIR = str(pathlib.Path.cwd().parent)
CHROME_DRIVER_BASE_URL = "https://chromedriver.storage.googleapis.com"
EDGE_DRIVER_BASE_URL = "https://msedgedriver.azureedge.net"
CHROME_DRIVER_ZIP = "chromedriver_win32.zip"
EDGE_DRIVER_ZIP = "edgedriver_win64.zip"
CHROME_DRIVER = "chromedriver.exe"
EDGE_DRIVER = "msedgedriver.exe"
BROWSER_DRIVER_DIR = str(pathlib.PurePath(BASE_DIR, "driver"))
DRIVER_MAPPING_FILE = os.path.join(BASE_DIR, "config", "mapping.json")
def get_browser_version(file_path):
"""
获取浏览器版本
:param file_path: 浏览器文件路径
:return: 浏览器大版本号
"""
# 判断路径文件是否存在
if not os.path.isfile(file_path):
raise FileNotFoundError(f"{file_path} is not found.")
win_obj = win_client.Dispatch('Scripting.FileSystemObject')
version = win_obj.GetFileVersion(file_path)
class DataAnalyser(object):
'''
Helper object for reading some common data types and exploring them.
'''
__readTypes = {
'csv' : _loadCSV
,'hdf5' : _loadHDF
,'hdf' : _loadHDF
,'rand' : _loadRAND
}
__writeTypes = {
'csv' : _writeCSV
,'hdf' : _writeHDF
,'hdf5' : _writeHDF
}
configDef = { # Configuration Definition
'chopDirectory' : {
'default' : pathlib.PurePath(os.path.curdir)
,'func' : lambda p: isinstance(p,pathlib.PurePath)
}
,'chopFilenamePrefix' : {
'default' : 'chop'
,'func' : lambda p: isinstance(p,str) and (len(p) >= 1 and len(p) <= 256)
}
,'hdfKey' : {
'default' : 'chop'
,'func' : lambda p: isinstance(p,str) and (len(p) >= 1 and len(p) <= 64)
}
,'chopFileFormat' : {
'default' : 'csv'
,'func' : lambda p: isinstance(p,str) and (p in DataAnalyser.getSupprtedFormats())
}
,'indexSpecialName' : {
'default' : 'Internal Index'
,'func' : lambda s: isinstance(s,str) and (len(s) >= 1 and len(s) <= 32)
}
}
"""Transformation Options"""
__transforms = {
'fixed' : {
'label' : 'Fixed Translation Transform'
,'xTrans' : 0.0
,'yTrans' : 0.0
,'Enabled' : False
,'func' : fixedTranslation
}
,'tagcentered' : {
'label' : 'Active View-Centered Translation'
,'Enabled' : False
,'func' : viewCenteredTranslation
}
,'angle' : {
'label' : 'Active Angle Translation'
,'Enabled' : False
,'func' : angleTransform
}
}
""" Curve Fit Options """
__fits = {
'linear' : {
'label' : 'Linear Curve Fit'
,'Enabled' : False
,'func' : fitLinear
}
}
def getDefaultConfig(self) :
defaultDict = dict()
for key in self.configDef :
defaultDict[key] = self.configDef[key]['default']
debug("DataAnalyser: getDefaultConfig: "+str(defaultDict))
return defaultDict
@staticmethod
def getSupprtedFormats() :
""" Get a list of supported file extensions that DataAnalyser can read
and write.
"""
return DataAnalyser.__readTypes.keys()
@staticmethod
def getFitOptions() :
return DataAnalyser.__fits
@staticmethod
def getTransformOptions() :
return DataAnalyser.__transforms
def __init__(self, initObj=None, *args, **kwargs):
self.isLoaded = False
self.__config = configer.Configer(DataAnalyser.configDef)
if initObj :
''' Initialize the data object with the first passed argument '''
self.df = pd.DataFrame(initObj)
self.isLoaded = True
self.__setDefaultView()
else :
''' Otherwise, empty'''
pass
if kwargs is not None :
for key,val in kwargs.items() :
self.__config[key] = val
self.indexName = self.__config['indexSpecialName']
def __setDefaultView(self):
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: __setDefaultView: no data loaded")
self.windowStart = 0
self.windowSize = self.df.shape[0]
self.windowType = 'index'
self.altIndexCol = 'index'
""" Create a default view of the fist column only. One column is
technically okay, but, rest of implementation may assume otherwise.
"""
"""At minimum, there must be and index and one column, so we know
that a good default is:"""
currentView = [ [self.indexName, self.plotColumns[0] ] ]
"""If there is more than one plot-able column, choose the first two"""
if len(self.plotColumns) > 2 :
currentView = [self.plotColumns[0:2] ]
"""Keep forgetting the view is a list of lists!"""
assert(isinstance(currentView[0],list))
self.currentView = currentView
def getConfig(self) :
return self.__config.getConfig()
def load(self, filetype='csv', filename=None, *args, **kwargs):
""" Determine which file to load """
if filename == None :
raise Exception('No filename provided.')
else :
self.filename = filename
''' Load the file '''
loadFunc = DataAnalyser.__readTypes[filetype]
self.df = loadFunc(filename, *args, **kwargs)
debug("load:Loaded new data file: head:"+str(self.df.head()))
self.indexCol = kwargs['index_col']
self.isLoaded = True
self.cleanData()
self.__setDefaultView()
debug("load:default view:"+str(self.currentView))
def cleanData(self) :
#data.dropna(inplace=True)
#data.reset_index(drop=True)
"""Pull out column names for plot-able column data"""
#self.plotColumns = self.df.columns[ self.columnDataIsPlotable(e) for e in self.df.columns.tolist() ]
self.plotColumns = list()
for label in self.df.columns.tolist() :
"""For each lable, check against function, store"""
if self.columnDataIsPlotable(label):
self.plotColumns.append(label)
debug("Got {} plot-able columns: {}".format(len(self.plotColumns),self.plotColumns))
info("Not yet fully implemented.")
def getColumnList(self) :
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: getColumnList: no data loaded")
return self.getLabels()
def getLabels(self):
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: getLabels: no data loaded")
return self.plotColumns
def columnDataIsPlotable(self, name):
"""Check data in the column named 'name' to make sure we can plot it
Cannot plot anything but ints and floats.
"""
return isinstance(self.df[name].iloc[0], Number)
def isValidColumn(self, name):
"""Check if 'name' is in the DataFrame."""
return (name in self.df.columns.tolist())
def isIndexable(self, name):
""" Determine whether or not the given column can be used as an index. """
retVal = False
if isValidColumn(name):
series = self.df[name]
""" Are all values unique? """
if series.duplicated().any():
""" Are the values ordered? """
#TODO sort it?
retVal = True
return retVal
def setAltIndexColumn(self, value):
warn('DataAnalyser:setAltIndexColumn: Not implemented.')
return
debug("setAltIndexColumn: got new index name:'"+str(value)+"'")
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: setAltIndexColumn: no data loaded")
""" Check for special option, index """
if str(value) == str('index') :
self.altIndexCol = 'index'
return
""" Otherwise, check value. """
if not self.isValidColumn(value):
raise Exception('Invalid column name:'+str(value))
self.altIndexCol = value
def validateView(self, viewList):
"""TODO Move view validation code here"""
pass
def setView(self, viewList=None, windowStart = None, windowSize=None, windowType=None) :
""" Set the analyser view
Parameters:
viewList : list of views, each view is a tuple of x,y data columns/names
"""
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: no data loaded")
if windowStart + windowSize > self.df.index.size :
raise Exception("ERROR: window out of range")
debug("Got new view list:"+str(viewList))
goodLabels = self.getLabels()
if viewList is not None :
""" Okay, this is real data """
newViewList = list()
if isinstance(viewList, list) or isinstance(viewList,tuple) :
for view in viewList:
debug("new view:"+str(view))
debug("new view type:"+str(type(view)))
if isinstance(view, list) or isinstance(view,tuple) :
""" viewList must be a list of lists """
validView = True
for item in view :
""" Validate each name in new view """
if item not in goodLabels :
warnwarn("View contains invalid labels for this data set, ignoring")
validView = False
else :
debug("item okay:"+str(item))
""" Okay, new view is validated, set currentview if passed."""
if validView :
newViewList.append(view)
else :
error("Was passed invalid view, ignoring it:"+str(view))
elif view is None :
"""View maybe none if input not sanitized. Fine, just
ignore it."""
warnwarn("View is 'None', ignoring.")
else :
raise TypeError("view must be a list or tuple, also.")
else :
raise TypeError("viewList must be a list or tuple.")
"""At this point, the new views have been validated."""
"""Accept the current view list, unless it's empty!"""
if len(newViewList) < 1 :
"""New views failed validation"""
debug("setView: All requested views failed validation.")
if len(self.currentView) >= 1 :
"""Old views exist, so, just keep that"""
debug("setView: Keeping old views:"+str(self.currentView))
else :
raise TypeError("New view did not pass validation:"+str(viewList))
else :
"""Okay, we can accept the new views that have passed validation"""
self.currentView = newViewList
debug("setView: Views updated: "+str(self.currentView))
if windowStart is not None :
self.windowStart = windowStart
if windowSize is not None :
self.windowSize = windowSize
if windowType is not None :
if not windowType == 'index' :
warnwarn("Cannot set index type: " +str(windowType))
warnwarn("The only supported index type is: index.")
#self.windowType = windowType
def getIndexLimits(self) :
#TODO use alternative index
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: no data loaded")
return { 'max' : self.df.index.max(),
'min' : self.df.index.min() }
def getView(self) :
""" Return the set of labels/parameters in current view
"""
return self.currentView
def getViewLimits(self) :
""" Returns: min/max values for currently shown data, all view pairs
"""
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: getWindowLimits: no data loaded")
viewLimitDictList = list()
for view in self.view :
x,y = self.view
viewDict = {'xmin' : self.df[x].min(),
'ymin' : self.df[y].min(),
'xmax' : self.df[x].max(),
'ymax' : self.df[y].max() }
viewLimitDictList.append(viewDict)
return viewLimitDictList
def setWindow(self, start, size) :
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: setWindow: no data loaded")
self.windowStart = start
self.windowSize = size
def getWindow(self) :
""" Returns: window start and window size
"""
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: getView: no data loaded")
return ( self.windowStart , self.windowSize )
def getStartEnd(self) :
start,size = self.getWindow()
end = start+size
debug("start,end:%s,%s" % (str(start),str(end)))
return (start, end)
def getViewData(self) :
""" Returns df of data in the current view
"""
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: getViewData: no data loaded")
if self.windowType == 'index' :
start = self.windowStart
end = self.windowStart + self.windowSize
mySlice = slice(start,end)
dfList = list()
debug("currentView:"+ str(self.currentView))
"""Make a list of DF Views to sent back to caller"""
for viewpair in self.currentView :
"""Create DF views to return to caller"""
df = self.df[ list(viewpair) ] # make view
df = df[mySlice] # get slice of the data requested
dfList.append(df)
"""Now, pass all views through transforms"""
newDFlist = self.doTransforms(dfList)
"""Now, pass all views through fitter"""
#newDFlist = self.doFits(dfList)
return newDFlist
else:
raise Exception('DataAnalyser window type ' + self.windowType + ' not implemented')
def doTransforms(self, dfList):
"""Run all active transforms on the given DataFrame"""
newDFlist = dfList
#debug("doTransforms: available transforms: %s" , self.__transforms.keys())
for transName in self.__transforms.keys():
transConfig = self.__transforms[transName]
if not transConfig['Enabled']:
continue
debug("Running transformation type: %s", transName)
transFunc = transConfig['func']
debug("Transformation Config: %s", transConfig)
newDFlist = transFunc(newDFlist, **transConfig)
#debug("Got transformation back: %s", newDFlist[0].head())
return newDFlist
def get2DData(self) :
if not self.isLoaded :
raise _DataNotLoaded("ERROR: DataAnalyser: get2DData: no data loaded")
raise Exception('Not implemented yet.')
def chop(self, dirpath=pathlib.PurePath(os.path.curdir), fmt='csv'
,hdfKey='chop', prefix='chop', **kwargs) :
""" Cut out the current view and make a new file with those data points
"""
if not self.isLoaded :
raise _DataNotLoaded("Data not loaded.")
xMin, xMax = (self.windowStart, self.windowStart + self.windowSize)
filename = "{0}_{1:d}-{2:d}.{3}".format(prefix,xMin,xMax,fmt)
chopFilePath = os.path.join( dirpath , filename )
if fmt not in self.__writeTypes.keys() :
raise TypeError('Unsupported data format: '+str(fmt))
writeFunc = self.__writeTypes[fmt]
info("DataAnalyser: chop: writing to file:"+chopFilePath)
df = self.df
altIndex = self.altIndexCol
#TODO finish alternative indexes
try :
if altIndex == 'index' :
writeFunc(df[xMin:xMax+1], chopFilePath, **kwargs)
else :
dfView = df[ (df[altIndex] >= xMin) & (df[altIndex] <= xMax) ]
writeFunc( dfView ,chopFilePath, **kwargs)
except Exception as e :
error("ERROR: DataAnalyser: chop: failed writing to file:"+chopFilePath)
print(e)
def getStats(self, quantiles) :
""" Show stats
"""
#TODO make this just one result for all labels in all views
dfList = self.getViewData()
statList = list()
for df in dfList :
# Create a DF of statistics data by combining the describe nad quantile features of pandas DF
statDF = pd.concat( [df.describe(),df.quantile(quantiles)] )
statList.append( statDF )
# pull off the first stat DF in the list
statDF = statList.pop(0)
for df in statList :
# mash them together into one DF for simpler output
newdf = statDF.T.append( df.T )
statDF = newdf.T
return [statDF] #TODO need to return list for caller, could be changed
def getCDFall(self, num_bins=100) :
""" Return data for CDF
"""
debug("DA: getCDF: starting...")
debug("DA: getCDF: bins = %d" % num_bins)
viewDFlist = self.getViewData()
viewList = self.getView()
debug("DA: viewDFlist type:" + str(type(viewDFlist)))
debug("DA: viewList type:" + str(type(viewList)))
cdfInfoLst = list()
for viewpair in viewList :
# for each "current view" of column/label pairs, do CDF of the ordinate
yLabel = viewpair[1] # ordinate
debug("getCDF: got ordinate label: {}".format(yLabel))
ser = set()
for df in viewDFlist :
if yLabel in df.columns :
ser = df[yLabel] # get Series data
break
else :
continue
#debug("DA: Series: " + str(ser))
counts, bin_edges = np.histogram(ser, bins=num_bins)
#debug("DA: counts: " + str(counts))
cdf = np.cumsum(counts)
debug("DA: cdf: " + str(cdf))
#assert not (cdf[-1] == 0)
cdfInfoT = (yLabel, cdf, counts, bin_edges)
debug("DA: getCDF: got CDF data: format: {}".format(type(cdfInfoT)))
cdfInfoLst.append(cdfInfoT)
return cdfInfoLst
import pathlib, datetime
print("{:30} : {}".format("pathlib.PurePath('.')",
type(pathlib.PurePath("."))))
print("{:30} : {}".format("pathlib.PurePosixPath('.')",
type(pathlib.PurePosixPath("."))))
print("{:30} : {}".format("pathlib.PureWindowsPath('.')",
type(pathlib.PureWindowsPath("."))))
print()
print("{:30} : {}".format("pathlib.Path('.')", type(pathlib.Path("."))))
#print("{:30} : {}".format("pathlib.PosixPath('.')" , type(pathlib.PosixPath("."))))
print("{:30} : {}".format("pathlib.WindowsPath('.')",
type(pathlib.WindowsPath("."))))
_path = pathlib.Path(".")
print(_path.cwd())
print()
for _dir in _path.iterdir():
print("{} {:30} {:10} {:20} {}".format(
"D" if _dir.is_dir() else "F", _dir.name,
_dir.stat().st_size,
datetime.datetime.fromtimestamp(
_dir.stat().st_mtime).strftime("%Y/%m/%d %H:%M:%S"),
_path.joinpath(_path.absolute(), _dir)))
"""day13.py
"""
import pathlib
cwd = pathlib.Path(__file__).parent.absolute()
data = pathlib.PurePath(cwd, 'data')
test = pathlib.PurePath(cwd, 'test')
test2 = pathlib.PurePath(cwd, 'test2')
class MineCart:
def __init__(self, fname):
self.map = []
self.carts = {}
self.dindex = '^>v<'
self.directions = {
'^': (0, -1),
'>': (1, 0),
'v': (0, 1),
'<': (-1, 0)
}
self.process(fname)
def get(self, x, y):
return self.map[y][x]
def set(self, x, y, z):
self.map[y][x] = z
def next_cart(self, cart, curve, turn_count):
out = cart
def as_path(self):
""" returns the relative path construction for the child so that local can make use of it """
return pathlib.PurePath(*self.parts)
def isvalidpath_win(path):
"""Whether an element of PATH is "clean" on Windows."""
patterns = "*/cplex/*", "*/guirobi/*", "/windows/system32/*"
return any(map(pathlib.PurePath(path).match, patterns))
ext)).resolve()), **kwargs)
# f.export("res/CMajor.mp3", format="mp3", bitrate='32k')
# f.export("res/CMajor.ogg", format="ogg")
# f.export("res/CMajor.flac", format="flac")
if __name__ == "__main__":
import pathlib
import Sampler
import BaseWaveMaker
import MusicTheory.EqualTemperament
import MusicTheory.Scale
import MusicTheory.tempo
basepath = pathlib.PurePath('../res/scales/wav/')
create_dirs(basepath)
wm = BaseWaveMaker.BaseWaveMaker()
sampler = Sampler.Sampler()
scale = MusicTheory.Scale.Scale()
timebase = MusicTheory.tempo.TimeBase()
timebase.BPM = 120
timebase.Metre = (4, 4)
nv = MusicTheory.tempo.NoteValue(timebase)
for key in ['C']:
# for key in ['C','C+','D','D+','E','F','F+','G','G+','A','A+','B']:
print(key, 'メジャー・スケール')
scale.Major(key=key)
waves = []
for f0 in scale.Frequencies:
def initialize_gui():
"""Initializes the GUI
---
If user has not initialized a profile, the GUI will ask for the following data:
- Username
- Password
- Plan
- Protocol
sudo protonvpn-gui
- Will start the GUI without invoking cli()
"""
check_root()
change_file_owner(os.path.join(CONFIG_DIR, "protonvpn-gui.log"))
gui_logger.debug(
"\n______________________________________\n\n\tINITIALIZING NEW GUI WINDOW\n______________________________________\n"
)
interface = Gtk.Builder()
posixPath = pathlib.PurePath(
pathlib.Path(__file__).parent.absolute().joinpath(
"resources/main.glade"))
glade_path = ''
for path in posixPath.parts:
if path == '/':
glade_path = glade_path + path
else:
glade_path = glade_path + path + "/"
interface.add_from_file(glade_path[:-1])
interface.connect_signals(Handler(interface))
if len(get_gui_processes()) > 1:
gui_logger.debug(
"[!] Two processes were found. Displaying MessageDialog to inform user."
)
messagedialog_window = interface.get_object("MessageDialog")
messagedialog_label = interface.get_object("message_dialog_label")
messagedialog_spinner = interface.get_object("message_dialog_spinner")
messagedialog_label.set_markup(
"Another GUI process was found, attempting to end it...")
messagedialog_spinner.show()
messagedialog_window.show()
time.sleep(1)
# thread = Thread(target=kill_duplicate_gui_process, args=[interface, messagedialog_label, messagedialog_spinner])
# thread.daemon = True
# thread.start()
response = kill_duplicate_gui_process()
if not response['success']:
messagedialog_label.set_markup(response['message'])
messagedialog_spinner.hide()
time.sleep(3)
sys.exit(1)
messagedialog_label.set_markup(response['message'])
messagedialog_spinner.hide()
if not os.path.isfile(CONFIG_FILE):
gui_logger.debug(">>> Loading LoginWindow")
window = interface.get_object("LoginWindow")
dashboard = interface.get_object("DashboardWindow")
dashboard.connect("destroy", Gtk.main_quit)
window.show()
else:
window = interface.get_object("DashboardWindow")
gui_logger.debug(">>> Loading DashboardWindow")
window.connect("destroy", Gtk.main_quit)
messagedialog_window = interface.get_object("MessageDialog")
messagedialog_label = interface.get_object("message_dialog_label")
interface.get_object("message_dialog_sub_label").hide()
messagedialog_spinner = interface.get_object("message_dialog_spinner")
messagedialog_label.set_markup("Loading...")
messagedialog_spinner.show()
messagedialog_window.show()
objects = {
"interface": interface,
"messagedialog_window": messagedialog_window,
"messagedialog_label": messagedialog_label,
"messagedialog_spinner": messagedialog_spinner,
}
thread = Thread(target=load_content_on_start, args=[objects])
thread.daemon = True
thread.start()
window.show()
# Gdk.threads_init()
# Gdk.threads_enter()
GObject.threads_init()
Gtk.main()
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default=
"/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--ckpt",
help=
"The path to the checkpoint for test, default is the latest checkpoint.",
default=None,
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = cfg.OUTPUT_DIR # ""
filename = f'test_log_{pathlib.PurePath(args.ckpt) if args.ckpt is not None else "last"}.txt'
test_result_filename = os.path.join(
save_dir,
pathlib.PurePath(filename).stem + '.pickle')
logger = setup_logger("maskrcnn_benchmark",
save_dir,
get_rank(),
filename=filename)
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
# Initialize mixed-precision if necessary
use_mixed_precision = cfg.DTYPE == 'float16'
amp_handle = amp.init(enabled=use_mixed_precision, verbose=cfg.AMP_VERBOSE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
ckpt = cfg.MODEL.WEIGHT if args.ckpt is None else os.path.join(
save_dir, args.ckpt)
_ = checkpointer.load(ckpt, use_latest=args.ckpt is None)
iou_types = ()
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
if cfg.MODEL.KEYPOINT_ON:
iou_types = iou_types + ("keypoints", )
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=False if cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY,
bbox_aug=cfg.TEST.BBOX_AUG.ENABLED,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
test_result_filename=test_result_filename,
)
synchronize()
def __attrs_post_init__(self):
u = urllib.parse.urlparse(self.url)
self._basename = pathlib.PurePath(u.path).name
def build(cls, yaml_node):
return cls(pathlib.PurePath(yaml_node.value), yaml_node.loc)
async def gdrive_upload(bot, update):
dl_url, custom_file_name, _, _ = await extract_link(
update.reply_to_message, "GLEECH")
txt = update.text
logger.info("command is : " + txt)
if txt.find("rename") > -1 and len(txt[txt.find("rename") + 7:]) > 0:
custom_file_name = txt[txt.find("rename") + 7:]
custom_file_name = await sanitize_file_name(custom_file_name)
custom_file_name = await sanitize_text(custom_file_name)
logger.info(dl_url)
logger.info(custom_file_name)
reply_message = await bot.send_message(
text=Translation.DOWNLOAD_START,
chat_id=update.chat.id,
reply_to_message_id=update.message_id)
tmp_directory_for_each_user = f"{Config.DOWNLOAD_LOCATION}{update.message_id}"
if not os.path.isdir(tmp_directory_for_each_user):
os.makedirs(tmp_directory_for_each_user)
if custom_file_name is None:
if dl_url.find('workers.dev') > -1 or dl_url.find('uploadbot') > -1:
custom_file_name = dl_url[dl_url.rindex("/") + 1:]
elif dl_url.find('seedr') > -1:
custom_file_name = dl_url[int(dl_url.rindex("/")) +
1:int(dl_url.rindex("?"))]
else:
if dl_url.find("/") > -1 and dl_url.find("?") > -1:
m_url = dl_url[:dl_url.rindex("?")]
custom_file_name = m_url[int(m_url.rindex("/")) + 1:]
else:
custom_file_name = dl_url[dl_url.rindex("/") + 1:]
custom_file_name = urllib.parse.unquote(custom_file_name)
download_directory = tmp_directory_for_each_user + "/" + custom_file_name
async with aiohttp.ClientSession() as session:
c_time = time.time()
try:
await download_coroutine(bot, session, dl_url, download_directory,
reply_message.chat.id,
reply_message.message_id, c_time)
except:
await bot.edit_message_text(text=Translation.SLOW_URL_DECED,
chat_id=reply_message.chat.id,
message_id=reply_message.message_id)
return False
if os.path.exists(download_directory):
end_one = datetime.now()
up_name = pathlib.PurePath(download_directory).name
size = get_readable_file_size(get_path_size(download_directory))
try:
await bot.edit_message_text(
text="¡¡¡Descarga completa!!!\n Subida en curso...",
chat_id=reply_message.chat.id,
message_id=reply_message.message_id)
except Exception as e:
logger.info(str(e))
pass
logger.info(f"Subiendo nombre: {up_name}")
drive = gdriveTools.GoogleDriveHelper(up_name)
gd_url, index_url = drive.upload(download_directory)
button = []
button.append([
pyrogram.types.InlineKeyboardButton(text="☁️ Enlace de la nube",
url=f"{gd_url}")
])
if Config.INDEX_URL:
logger.info(index_url)
button.append([
pyrogram.types.InlineKeyboardButton(text="ℹ️ Enlace de índice",
url=f"{index_url}")
])
button_markup = pyrogram.types.InlineKeyboardMarkup(button)
await bot.send_message(
text=
f"🤖: <b>{up_name}</b> se ha subido con éxito a su nube 🤒 \n📀 Tamaño: {size}",
chat_id=update.chat.id,
reply_to_message_id=update.message_id,
reply_markup=button_markup)
if Config.INDEX_URL:
await generate_short_link(reply_message, index_url,
custom_file_name)
await reply_message.delete()
def __init__(self,
*,
name: Optional[str],
python_file: Optional[str],
comment: Optional[str],
writers: Set[str] = None):
"""
### Create the experiment
:param name: name of the experiment
:param python_file: `__file__` that invokes this. This is stored in
the experiments list.
:param comment: a short description of the experiment
The experiments log keeps track of `python_file`, `name`, `comment` as
well as the git commit.
Experiment maintains the locations of checkpoints, logs, etc.
"""
if python_file is None:
python_file = self.__get_caller_file()
self.lab = Lab(python_file)
if name is None:
file_path = pathlib.PurePath(python_file)
name = file_path.stem
if comment is None:
comment = ''
self.name = name
self.experiment_path = self.lab.experiments / name
self.check_repo_dirty = self.lab.check_repo_dirty
self.configs_processor = None
experiment_path = pathlib.Path(self.experiment_path)
if not experiment_path.exists():
experiment_path.mkdir(parents=True)
self.run = Run.create(experiment_path=self.experiment_path,
python_file=python_file,
trial_time=time.localtime(),
comment=comment)
repo = git.Repo(self.lab.path)
self.run.commit = repo.head.commit.hexsha
self.run.commit_message = repo.head.commit.message.strip()
self.run.is_dirty = repo.is_dirty()
self.run.diff = repo.git.diff()
checkpoint_saver = self._create_checkpoint_saver()
logger.internal().set_checkpoint_saver(checkpoint_saver)
if writers is None:
writers = {'sqlite', 'tensorboard'}
if 'sqlite' in writers:
logger.internal().add_writer(sqlite.Writer(self.run.sqlite_path))
if 'tensorboard' in writers:
logger.internal().add_writer(
tensorboard.Writer(self.run.tensorboard_log_path))
# This is a sample Python script.
# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
import pathlib
import re
keyword = "apple"
# 获取索引文件路径
current_path = pathlib.PurePath(__file__).parent
dbfile = current_path.joinpath("search.db")
# 在索引文件中搜索关键字
with open(dbfile, encoding='utf-8') as f:
for line in f.readlines():
if re.search(keyword, line):
print(line.rstrip())
def print_hi(name):
# Use a breakpoint in the code line below to debug your script.
print(f'Hi, {name}') # Press Ctrl+F8 to toggle the breakpoint.
# Press the green button in the gutter to run the script.
if __name__ == '__main__':
print_hi('PyCharm')
# See PyCharm help at https://www.jetbrains.com/help/pycharm/
def __init__(self, path):
self.path = path
super().__init__(pathlib.PurePath(path).name, [])
def Data_Validation_Main():
init()
root_dir = "C:\\Seronet_Data_Validation"
passing_msg = (
"File is a valid Zipfile. No errors were found in submission. " +
"Files are good to proceed to Data Validation")
#############################################################################################
summary_path = root_dir + file_sep + "Summary_of_all_Submissions.xlsx"
summary_file = []
if os.path.exists(summary_path):
xls = pd.ExcelFile(summary_path, engine='openpyxl')
for iterZ in xls.sheet_names:
if len(summary_file) == 0:
summary_file = pd.read_excel(summary_path,
sheet_name=iterZ,
engine='openpyxl')
else:
new_file = pd.read_excel(summary_path,
sheet_name=iterZ,
engine='openpyxl')
summary_file = pd.concat([summary_file, new_file])
summary_file.reset_index(inplace=True)
summary_file.drop(["index"], inplace=True, axis=1)
else:
summary_file = pd.DataFrame(columns=[
"Submission_Status", "Date_Of_Last_Status", "Folder_Location",
"CBC_Num", "Date_Timestamp", "Submission_Name",
"Validation_Status", "JIRA_Ticket", "Ticket_Status"
])
#############################################################################################
if "Submission_Status" not in summary_file.columns:
print(
"Excel file Exists but is empty, will cause errors during validation"
)
print("Delete File and re-run program. Terminating Data Validation")
clear_empty_folders(root_dir)
input("\n\nPress Enter to close window...")
return {}
Support_Files = get_subfolder(root_dir, "Support_Files")
assay_folders = [i for i in Support_Files if os.path.isdir(i)]
assay_data, assay_target = populate_assay_data(assay_folders)
create_sub_folders(root_dir, "00_Uploaded_Submissions")
create_sub_folders(root_dir, "01_Failed_File_Validation")
create_sub_folders(root_dir, "02_Data_Validation_No_Errors")
create_sub_folders(root_dir, "03_Data_Validation_Column_Errors")
create_sub_folders(root_dir, "04_Data_Validation_Data_Errors")
create_sub_folders(root_dir, "05_Data_Validation_Major_Errors")
create_sub_folders(root_dir, "06_Data_Validation_Minor_Errors")
summary_file = check_for_typo(summary_file)
summary_file = move_minor_errors(summary_file, root_dir)
summary_file = move_major_errors(summary_file, root_dir)
summary_file = move_updated(summary_file, root_dir)
summary_file = move_folder_to_uploaded(summary_file, root_dir)
CBC_Folders = get_subfolder(root_dir, "Files_To_Validate")
if len(CBC_Folders) == 0:
print(
"\nThe Files_To_Validate Folder is empty, no Submissions Downloaded to Process\n"
)
#############################################################################################
rename_CBC_folders(root_dir, CBC_Folders)
CBC_Folders = get_subfolder(root_dir, "Files_To_Validate")
sort_order = [int(i[-2:]) for i in CBC_Folders]
sort_list = sorted(range(len(sort_order)), key=lambda k: sort_order[k])
CBC_Folders = [CBC_Folders[i] for i in sort_list]
all_res = []
for iterT in CBC_Folders:
date_folders = os.listdir(iterT)
cbc_name = pathlib.PurePath(iterT).name
res = [cbc_name, 0, 0]
if len(date_folders) == 0:
print("There are no submitted files for " + cbc_name)
clear_dir(iterT)
all_res.append(res)
continue
for iterD in date_folders:
Date_path = iterT + file_sep + iterD
Submissions_Names = os.listdir(Date_path)
file_count = 0
for iterS in Submissions_Names:
file_str = (Date_path + file_sep + iterS)
if os.path.isfile(file_str):
os.remove(file_str)
file_str = file_str.replace(
(root_dir + file_sep + "Files_To_Validate"), "")
print(
colored(
"\n## File Validation has NOT been run for " +
file_str + " ##", 'yellow'))
continue
current_sub_object = File_Submission_Object.Submission_Object(
iterS[15:])
curr_dict = populate_dict(validation_date, cbc_name, iterD,
current_sub_object)
file_check = summary_file.query(
"CBC_Num == @cbc_name and Date_Timestamp == @iterD and " +
" Submission_Name ==@current_sub_object.File_Name")
if len(file_check) > 0:
curr_status = file_check['Submission_Status'].tolist()[0]
if curr_status in ["Updated"]:
curr_dict = file_check.to_dict('records')
curr_dict = curr_dict[0]
curr_dict["Submission_Status"] = "Pending Review"
else:
if curr_status in ["Downloaded"]:
pass
elif curr_status in ["Pending Review"]:
print(
"Submission Previously Updated and Reprocssed"
+ " - Pending Manual Review of changes")
elif curr_status not in "Unknown":
print("Submission Status (" + curr_status +
") is Unknown." + " Possible mistyped")
print(
"Defaulting Status to Unknowm and skipping File"
)
shutil.rmtree(Date_path + file_sep + iterS)
continue
orgional_path = Date_path + file_sep + iterS
print("\n## Starting the Data Validation Proccess for " +
current_sub_object.File_Name + " ##")
file_count = file_count + 1
list_of_folders = os.listdir(Date_path + file_sep + iterS)
result_message = get_result_message(list_of_folders, Date_path,
orgional_path, iterS)
if (result_message == ''):
continue
if result_message != passing_msg:
print("Submitted File FAILED the File-Validation Process")
print("With Error Message: " + result_message + "\n")
error_str = "Submission Failed File Validation"
move_file_and_update(orgional_path, root_dir,
current_sub_object, curr_dict,
"01_Failed_File_Validation",
error_str)
curr_dict["Validation_Status"] = result_message
summary_file = summary_file.append(curr_dict,
ignore_index=True)
summary_file.drop_duplicates(
["CBC_Num", "Date_Timestamp", "Submission_Name"],
keep='last',
inplace=True)
continue
#############################################################################################################################
list_of_files = []
if "UnZipped_Files" in list_of_folders:
list_of_files = os.listdir(Date_path + file_sep + iterS +
file_sep + "Unzipped_Files")
if len(list_of_files) == 0:
print(
"There are no files found within this submission to process"
)
continue
Subpath = Date_path + file_sep + iterS
current_sub_object = populate_object(current_sub_object,
Subpath, list_of_files,
Support_Files)
current_sub_object.update_object(assay_data, "assay.csv")
current_sub_object.update_object(assay_target,
"assay_target.csv")
#############################################################################################################################
col_err_count = len(current_sub_object.Column_error_count)
if col_err_count > 0:
print(
colored(
"There are (" + str(col_err_count) +
") Column Names in the submission that are wrong/missing",
'red'))
print(
colored(
"Not able to process this submission, please correct and resubmit \n",
'red'))
current_sub_object.write_col_errors((Subpath + file_sep))
error_str = "Submission has Column Errors, Data Validation NOT Preformed"
move_file_and_update(orgional_path, root_dir,
current_sub_object, curr_dict,
"03_Data_Validation_Column_Errors",
error_str)
summary_file = summary_file.append(curr_dict,
ignore_index=True)
continue
valid_cbc_ids = str(current_sub_object.CBC_ID)
for file_name in current_sub_object.Data_Object_Table:
if file_name not in [
"submission.csv", "shipping_manifest.csv",
"assay.csv", "assay_target.csv"
]:
if "Data_Table" in current_sub_object.Data_Object_Table[
file_name]:
try:
data_table = current_sub_object.Data_Object_Table[
file_name]['Data_Table']
data_table, drop_list = current_sub_object.correct_var_types(
file_name)
current_sub_object = Validation_Rules(
re, datetime, current_sub_object,
data_table, file_name, valid_cbc_ids,
drop_list)
except Exception as e:
display_error_line(e)
else:
print(file_name +
" was not included in the submission")
#############################################################################################################################
check_ID_Cross_Sheet(current_sub_object, re)
compare_tests(current_sub_object)
try:
create_sub_folders(
Date_path,
iterS + file_sep + "Data_Validation_Results", True)
Data_Validation_Path = Date_path + file_sep + iterS + file_sep + "Data_Validation_Results"
current_sub_object.write_error_file(Data_Validation_Path +
file_sep)
if len(current_sub_object.Error_list) == 0:
error_str = "No Errors were Found during Data Validation"
move_file_and_update(orgional_path, root_dir,
current_sub_object, curr_dict,
"02_Data_Validation_No_Errors",
error_str)
else:
error_str = ("Data Validation found " +
str(len(current_sub_object.Error_list)) +
" errors in the submitted files")
move_file_and_update(orgional_path, root_dir,
current_sub_object, curr_dict,
"04_Data_Validation_Data_Errors",
error_str)
except Exception as err:
print("An Error Occured when trying to write output file")
display_error_line(err)
print("Validation for this File is complete")
summary_file = summary_file.append(curr_dict,
ignore_index=True)
summary_file.drop_duplicates(
["CBC_Num", "Date_Timestamp", "Submission_Name"],
keep='last',
inplace=True)
if file_count > 0:
res[1] = res[1] + 1
res[2] = res[2] + file_count
if len(os.listdir(Date_path)) == 0:
shutil.rmtree(Date_path)
print(
colored(
"\nEnd of Current CBC Folder (" + cbc_name +
"), moving to next CBC Folder", 'blue'))
all_res.append(res)
clear_dir(iterT)
print("\nALl folders have been checked")
print("Closing Validation Program")
summary_file.sort_values(
by=['CBC_Num', 'Date_Of_Last_Status', 'Date_Timestamp'], inplace=True)
writer = pd.ExcelWriter(summary_path, engine='xlsxwriter')
writer = write_excel_sheets(writer, summary_file,
"01_Failed_File_Validation",
"Failed_File_Validation")
writer = write_excel_sheets(writer, summary_file,
"03_Data_Validation_Column_Errors",
"Column_Errors_Found")
writer = write_excel_sheets(writer, summary_file,
"04_Data_Validation_Data_Errors",
"Failed_Data_Validation")
writer = write_excel_sheets(writer, summary_file,
"06_Data_Validation_Minor_Errors",
"Pending_Feedback")
writer = write_excel_sheets(writer, summary_file,
"05_Data_Validation_Major_Errors",
"Major_Errors_Found")
writer = write_excel_sheets(writer, summary_file,
"02_Data_Validation_No_Errors",
"Passed_Data_Validation")
writer = write_excel_sheets(writer, summary_file,
"00_Uploaded_Submissions",
"Uploaded_Submissions")
writer.save()
print("\n#### Validation Summary ####\n")
for iterZ in range(len(all_res)):
print("CBC :: " + all_res[iterZ][0] + ", Had " +
str(all_res[iterZ][1]) + " date folders checked." +
" And processed " + str(all_res[iterZ][2]) +
" unique Submissions")
clear_empty_folders(root_dir)
input("\n\nPress Enter to close window...")
# coding: utf-8
# 2019/12/30 @ tongshiwei
from longling import path_append, loading
import pathlib
from .constants import CHAR, STROKE, RADICAL, GLYPH, PRON
from .glyph import character_glyph
from .stroke import token2stroke
__all__ = ["load_dict", "CDict", "token2radical", "char_features"]
DEFAULT_DICT = path_append(pathlib.PurePath(__file__).parents[2],
"meta_data",
"cdict.csv",
to_str=True)
DEFAULT_CDICT = [None]
def load_dict(dict_path):
cdict = {}
for line in loading(dict_path):
cdict[line[CHAR]] = {}
cdict[line[CHAR]][RADICAL] = line[RADICAL]
return cdict
class CDict(object):
def __init__(self, dict_obj=None, allow_missing=True):
self._dict = dict_obj
self._allow_missing = allow_missing
