#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author: xurongzhong#126.com
# 技术支持qq群： 144081101 591302926 567351477 钉钉免费群：21745728
# 作者博客：https://www.jianshu.com/u/9bc194fde100
# CreateDate: 2018-1-29
# from https://pymotw.com/3/pathlib/index.html
# pathlib_from_existing.py
import pathlib
ind = pathlib.PurePosixPath('source/pathlib/index.rst')
print(ind)
py = ind.with_name('pathlib_from_existing.py')
print(py)
pyc = py.with_suffix('.pyc')
print(pyc)
def populate(self):
for obj in AndroidDecoder.get_subclasses():
if obj.exclude_from_registry:
continue
self.decoders[obj] = pathlib.PurePosixPath(obj.RETARGET
or obj.TARGET)
def meta(self, pathmeta):
if not self.exists():
# if the path does not exist write even temporary to disk
if self.is_symlink():
meta = self.meta
if meta == pathmeta:
log.debug(
f'Metadata unchanged for {meta.id}. Not updating.')
return
if meta.id != pathmeta.id:
msg = ('Existing cache id does not match new id!\n'
f'{self!r}\n'
f'{meta.id} != {pathmeta.id}\n'
f'{meta.as_pretty()}\n'
f'{pathmeta.as_pretty()}')
log.critical(msg)
meta_newer = 'Meta newer. Not updating.'
pathmeta_newer = 'Other meta newer.'
msg = '{}' # apparently I was out of my mind when I wrote this originally ...
if meta.updated is None and pathmeta.updated is None:
log.warning(
'no change since either has an updated value (wat)'
)
return #FIXME
if meta.updated > pathmeta.updated:
log.info(msg.format(meta_newer))
return # this is the right thing to do for a sane filesystem
elif meta.updated < pathmeta.updated:
log.info(msg.format(pathmeta_newer))
# THIS IS EXPLICITLY ALLOWED
else: # they are equal
extra = 'Both updated at the same time '
if meta.created is not None and pathmeta.created is not None:
if meta.created > pathmeta.created:
log.info(msg.format(extra + meta_newer))
return
elif meta.created < pathmeta.created:
log.info(msg.format(extra + pathmeta_newer))
# THIS IS EXPLICITLY ALLOWED
else: # same created
log.info(
msg.format(
'Identical timestamps. Not updating.'))
return
elif meta.created is not None:
log.info(
msg.format(
extra +
'Meta has datetime other does not. Not updating.'
))
return
elif pathmeta.created is not None:
msg = msg.format(
extra + 'Meta has no datetime other does.')
log.info(msg)
raise exc.MetadataIdMismatchError(msg)
else: # both none
log.info(
msg.format(extra + (
'Identical update time both missing created time. '
'Not updating.')))
return
# equality
# id mismatch all cases above should return or raise except for other metadata newer
if meta.size is not None and pathmeta.size is None:
log.error('new meta has no size so will not overwrite')
return
# FIXME do the timestamp dance above here
log.debug('Metadata exists, but ids match so will update')
# trash old versions instead of just unlinking
pc = self.local.cache
trash = pc.trash
self.rename(trash / f'{pc.parent.id}-{meta.id}-{self.name}')
#self.unlink()
# FIXME if an id starts with / then the local name is overwritten due to pathlib logic
# we need to error if that happens
#symlink = pathlib.PurePosixPath(self.local.name, pathmeta.as_symlink().as_posix().strip('/'))
symlink = pathlib.PurePosixPath(
self.local.name) / pathmeta.as_symlink()
self.local.symlink_to(symlink)
else:
raise exc.PathExistsError(f'Path exists {self}')
#!/usr/bin/env python3
# encoding: utf-8
#
# Copyright (c) 2016 Doug Hellmann. All rights reserved.
# Written for https://pymotw.com
#
"""Building paths with joinpath
"""
#end_pymotw_header
import pathlib
root = pathlib.PurePosixPath('/')
subdirs = ['usr', 'local']
usr_local = root.joinpath(*subdirs)
print(usr_local)
def start_download_server(file_path, **kwargs):
"""Start the download web server.
This function will display a QR code to the terminal that directs a user's
cell phone to browse to this web server. Once connected, the web browser
will download the file, or display the file in the browser depending on the
options set.
Args:
file_path (str): The file path to serve.
**kwargs: Keyword Arguements.
Keyword Arguments:
debug (bool): Indication whether to output the encoded URL to the terminal.
custom_port (str): String indicating which custom port the user wants to use.
ip_addr (str): The IP address to bind web server to.
auth (str): Base64 encoded 'username:password'.
no_force_download (bool): Allow web browser to handle the file served
instead of forcing the browser to download it.
"""
PORT = int(
kwargs["custom_port"]) if kwargs.get("custom_port") else random_port()
LOCAL_IP = kwargs["ip_addr"] if kwargs["ip_addr"] else get_local_ip()
SSID = get_ssid()
auth = kwargs.get("auth")
debug = kwargs.get("debug", False)
if not os.path.exists(file_path):
print("No such file or directory")
clean_exit()
# Variable to mark zip for deletion, if the user uses a folder as an argument
delete_zip = 0
abs_path = os.path.normpath(os.path.abspath(file_path))
file_dir = os.path.dirname(abs_path)
file_path = os.path.basename(abs_path)
# change to directory which contains file
os.chdir(file_dir)
# Checking if given file name or path is a directory
if os.path.isdir(file_path):
zip_name = pathlib.PurePosixPath(file_path).name
try:
# Zips the directory
path_to_zip = make_archive(zip_name, "zip", file_path)
file_path = os.path.basename(path_to_zip)
delete_zip = file_path
except PermissionError:
print("Permission denied")
clean_exit()
# Tweaking file_path to make a perfect url
file_path = file_path.replace(" ", "%20")
handler = FileTransferServerHandlerClass(
file_path, auth, debug, kwargs.get("no_force_download", False))
httpd = socketserver.TCPServer(("", PORT), handler)
# This is the url to be encoded into the QR code
address = "http://" + str(LOCAL_IP) + ":" + str(PORT) + "/" + file_path
print("Scan the following QR code to start downloading.")
if SSID:
print(
"Make sure that your smartphone is connected to \033[1;94m{}\033[0m"
.format(SSID))
# There are many times where I just need to visit the url
# and cant bother scaning the QR code everytime when debugging
if debug:
print(address)
print_qr_code(address)
try:
httpd.serve_forever()
except KeyboardInterrupt:
pass
# If the user sent a directory, a zip was created
# this deletes the first created zip
if delete_zip != 0:
os.remove(delete_zip)
clean_exit()
def keyfile_img_path(self) -> pathlib.PurePosixPath:
"""Path to the public key of the key used to sign the packages, inside
the Docker image/container
"""
return pathlib.PurePosixPath("/tmp/gpg/signing.key")
def remote_uri_api(self, endpoint=''):
# technically this is for the whole repo
repo, rnprefix, url_base, netloc, path = self._remote_helper()
p = pathlib.PurePosixPath(path).with_suffix('')
return f'https://api.github.com/repos{p}' + endpoint
args = init_args(sys.argv[1:])
# Configure logging. One logfile per day, e.g. 'log/2021-03-12.log'.
today = datetime.now().strftime(r"%Y-%m-%d")
logpath = pathlib.Path(config["project"]["logdir"]) / f"{today}.log"
loglevel = logging.getLevelName(args.loglevel.upper())
init_logging(logpath, loglevel)
log = logging.getLogger("stockbro")
# Dispatch commands.
if args.command == "rss":
if args.rss_command == "fetch":
urls = config["rss"]["feeds"]
# Convert paths from Posix used in the config to whatever the OS is using.
feedsdb_path = pathlib.Path(pathlib.PurePosixPath(config["rss"]["feedsdb-path"]))
feedsdb_schema = pathlib.Path(pathlib.PurePosixPath(config["rss"]["feedsdb-schema"]))
# Count rows in database table before insertion
util.create_db(feedsdb_path, feedsdb_schema)
log.info(f"Fetching {len(urls)} RSS feeds ...")
for ii, url in enumerate(urls, 1):
try:
rss.feeds_to_database([url], feedsdb_path, tablename="items",
tags={"guid": "rss_guid", "link": "rss_link", "pubDate": "rss_pubdate",
"title": "rss_title", "description": "rss_description"},
keys=["rss_guid", "rss_link"])
log.info(f" - {url} ... success.")
except Exception as e:
log.error(f" - {url} ... error: {e}")
def join_path(a, b):
return os.path.join(a, str(pathlib.PurePosixPath(b)))
def test_round_trip(tmp_path: pathlib.Path):
""" Test that a serialized and then deserialized dictionary is unchanged. """
# Arrange
data = {
"bool":
True,
"none":
None,
"int":
12,
"float":
3.14,
"decimal_1":
Decimal(3.1415),
"decimal_2":
Decimal("3.1415"),
"inf":
inf,
"complex":
3 + 4j,
"string":
"string",
"date":
datetime.date(2020, 8, 9),
"time":
datetime.time(10, 11, 12, 13),
"datetime_with_tz":
datetime.datetime(2020,
8,
9,
10,
11,
12,
13,
tzinfo=datetime.timezone.utc),
"datetime_without_tz":
datetime.datetime(2020, 8, 9, 10, 11, 12, 13),
"timedelta":
datetime.timedelta(seconds=12, microseconds=13),
"list": [1, 2, 3],
"set": {1, 2, 3},
"tuple": (1, 2, 3),
"nested_list": [1, 2, [3, 4, []]],
"nested_tuple": (1, 2, (3, 4, ())),
"complex_list": [1, 2, (3, 4, {5})],
"my_test": {(1, 2), (3, 4)},
"my_test2": {"one", "two"},
"path":
pathlib.Path(),
"pure_windows_path":
pathlib.PureWindowsPath(),
"pure_posix_Path":
pathlib.PurePosixPath(),
"bytes":
b"bytes",
}
test_file = tmp_path / "test.json"
# Act
serialized = json.dumps(data, cls=JsonSerializer)
deserialized = json.loads(serialized, cls=JsonDeserializer)
json.dump(data, test_file.open("w"), cls=JsonSerializer)
deserialized_from_file = json.load(test_file.open(), cls=JsonDeserializer)
# Assert
for key, value in deserialized.items():
expected = data.get(key)
assert expected == value
assert deserialized == deserialized_from_file
def remove_extensions(cls, filename):
if not filename:
return None
for ext in pathlib.PurePosixPath(filename).suffixes:
filename = filename.replace(ext, '')
return filename
class TestPathEncoding:
""" Tests for custom encoding of Path types. """
PATH_DECODING_TEST_CASES = [
(
pathlib.Path("/Users/"),
CustomEncodedPathTypes.path.json_encoding(["/", "Users"]),
),
(
pathlib.PosixPath("/Users/Shared/"),
CustomEncodedPathTypes.path.json_encoding(["/", "Users",
"Shared"]),
),
(
pathlib.PurePosixPath("/Users/Shared/test"),
CustomEncodedPathTypes.pure_posix_path.json_encoding(
["/", "Users", "Shared", "test"]),
),
(
pathlib.PurePosixPath("/Users/guest/Documents/test"),
CustomEncodedPathTypes.pure_posix_path.json_encoding(
["/", "Users", "guest", "Documents", "test"]),
),
(
pathlib.PureWindowsPath("/Users/guest/Documents/test"),
CustomEncodedPathTypes.pure_windows_path.json_encoding(
["\\", "Users", "guest", "Documents", "test"]),
),
(
pathlib.PureWindowsPath(
"/Library/directory_name/sub_dir_name/test"),
CustomEncodedPathTypes.pure_windows_path.json_encoding(
["\\", "Library", "directory_name", "sub_dir_name", "test"]),
),
]
PATH_ENCODING_TEST_CASES = PATH_DECODING_TEST_CASES
PATH_ENCODING_UNSUPPORTED_CASES = [
pathlib.PurePath, pathlib.WindowsPath, pathlib.Path
]
@staticmethod
@pytest.mark.parametrize("value", PATH_ENCODING_UNSUPPORTED_CASES)
def test_encode_path_error(value):
""" Test that the JsonSerializer.encode_path raises an error if no encoding is defined. """
# Arrange
value = b"foobar"
# Act & Assert
with pytest.raises(NotImplementedError):
JsonSerializer.encode_path(value)
@staticmethod
@pytest.mark.parametrize("value, expected", PATH_ENCODING_TEST_CASES)
def test_encode_path(value, expected):
""" Test that the JsonSerializer.encode_collection encodes values as expected. """
# Arrange
# Act
result = JsonSerializer.encode_path(value)
# Assert
assert result == expected
@staticmethod
@pytest.mark.parametrize("expected, value", PATH_DECODING_TEST_CASES)
def test_decode_path(value, expected):
""" Test that the JsonDeserializer.decode_collection decodes collections as expected. """
# Arrange
# Act
result = JsonDeserializer.decode_path(value)
# Assert
assert result == expected
@staticmethod
def test_decode_path_error():
""" Test that the JsonDeserializer.decode_collection raises an error if no decoding is defined. """
# Arrange
value = {"foo": "bar"}
# Act & Assert
with pytest.raises(NotImplementedError):
JsonDeserializer.decode_path(value)
def url(self):
"""Return url to file."""
import pathlib
return pathlib.PurePosixPath(self.file).as_uri()
def build(self):
"""Building Docker image from dockerfile"""
log.info(logger.LINE_DOUBLE)
log.info('Preparing to build Docker image...')
tmp_folder, self.args.old_package_url = '', ''
if self.args.source == 'local' and self.args.package_url.startswith(
('http://', 'https://', 'ftp://')):
log.info('Downloading needed files...')
self.args.old_package_url = self.args.package_url
archive_name = self.args.old_package_url.split('/')[-1]
tmp_folder = self.location / 'tmp'
download_file(self.args.package_url,
tmp_folder / archive_name,
parents_=True)
self.args.package_url = (tmp_folder / archive_name).relative_to(
self.location)
self.args.package_url = str(
pathlib.PurePosixPath(self.args.package_url))
log.info('Downloading finished')
self.kwargs['package_url'] = self.args.package_url
log.info('Building Docker image...')
self.args.file = pathlib.Path(self.args.file)
if self.args.file.is_absolute():
self.args.file = pathlib.Path(self.args.file).relative_to(
self.location)
self.builder = DockerImageBuilder()
curr_time = timeit.default_timer()
log.info(f"Build log location: {self.logdir / 'image_build.log'}")
self.image = self.builder.build_docker_image(
dockerfile=self.args.file,
directory=str(self.location),
tag=self.image_name,
build_args=self.kwargs,
logfile=self.logdir / 'image_build.log')
log.info(
f'Build time: {format_timedelta(timeit.default_timer() - curr_time)}'
)
if not self.image:
raise FailedBuild(f'Error building Docker image {self.args.tags}')
log.info(f'Save image data in {self.args.image_json_path} file')
try:
if not self.args.image_json_path.parent.exists():
self.args.image_json_path.parent.mkdir()
with self.args.image_json_path.open(mode='w',
encoding='utf-8') as f:
json.dump(
{
'image_name': self.image_name,
'product_version': self.args.product_version,
'distribution': self.args.distribution,
'os': self.args.os
},
f,
ensure_ascii=False,
indent=4)
except Exception:
log.exception(
f'Failed to save image data in {self.args.image_json_path} file'
)
log.info(f'Docker image {self.args.tags} built successfully')
if self.args.old_package_url:
self.args.package_url, self.args.old_package_url = self.args.old_package_url, self.args.package_url
self.kwargs['package_url'] = self.args.package_url
if tmp_folder and tmp_folder.exists():
shutil.rmtree(tmp_folder, ignore_errors=True)
log.info('Build dependencies deleted')
def getFileExt(f):
ext = pathlib.PurePosixPath(f).suffix
return ext
import json
import os
import pathlib
from datetime import datetime, timedelta
from collections import namedtuple
from random import gauss
import numpy as np
import pandas as pd
from scipy.signal import sawtooth
# Import params from file and concatenate signals in order
# Script params
raw_path = os.path.dirname(os.path.realpath(__file__))
path = pathlib.PurePosixPath(raw_path)
config_path = path / 'signal_params.json'
export_path = path / 'signals' / 'signal_d.txt'
def import_config(path):
with open(path, 'r') as json_file:
config = json.load(json_file)
return config
def create_signal(config):
start_time = datetime.utcnow()
sampling_rate = config['samples_sec']
raw_segments = config['segments']
def install_img_path(self) -> pathlib.PurePosixPath:
"""Path to the install directory within the image/container"""
return pathlib.PurePosixPath("/tmp/install")
# To instantiate a new path, give a string as the first argument. The string representation of the path object is this name value. To create a new path referring to a value relative to an existing path, use the / operator to extend the path. The argument to the operator can either be a string or another path object.
# pathlib_operator.py
import pathlib
usr = pathlib.PurePosixPath('/usr')
print(usr)
usr_local = usr / 'local'
print(usr_local)
usr_share = usr / pathlib.PurePosixPath('share')
print(usr_share)
root = usr / '..'
print(root)
etc = root / '/etc/'
print(etc)
# As the value for root in the example output shows, the operator combines the path values as they are given, and does not normalize the result when it contains the parent directory reference "..". However, if a segment begins with the path separator it is interpreted as a new “root” reference in the same way as os.path.join(). Extra path separators are removed from the middle of the path value, as in the etc example here.
#
# $ python3 pathlib_operator.py
#
# /usr
# /usr/local
# /usr/share
# /usr/..
# /etc
def testfile_img_path(self) -> pathlib.PurePosixPath:
"""Path to the testfile within the image/container"""
return pathlib.PurePosixPath("/tmp").joinpath(self.testfile.name)
def execute_remote_ssh(self, *, connect_to={}, root, run_script=True):
"""
Execute build plan using the remote SSH strategy. Files from the build
plan are transferred via SFTP to the directory ``root`` on a remote
server. If ``run_script`` is ``True``, the ``paramiko`` SSH client will
then run ``{script}.sh``. ``root`` can either be an absolute or
relative (to the login directory) path.
``connect_to`` is a dictionary that holds all input arguments to
``paramiko``'s ``SSHClient.connect``
(`documentation <http://docs.paramiko.org/en/stable/api/client.html#paramiko.client.SSHClient.connect>`_).
At a minimum, the ``hostname`` input argument must be supplied in this
dictionary as the remote server.
Returns :class:`RemoteSSHBuildProducts`.
"""
from paramiko import SSHClient
with SSHClient() as client:
client.load_system_host_keys()
client.connect(**connect_to)
with client.open_sftp() as sftp:
def mkdir_exist_ok(path):
try:
sftp.mkdir(str(path))
except IOError as e:
# mkdir fails if directory exists. This is fine in nmigen.build.
# Reraise errors containing e.errno info.
if e.errno:
raise e
def mkdirs(path):
# Iteratively create parent directories of a file by iterating over all
# parents except for the root ("."). Slicing the parents results in
# TypeError, so skip over the root ("."); this also handles files
# already in the root directory.
for parent in reversed(path.parents):
if parent == pathlib.PurePosixPath("."):
continue
else:
mkdir_exist_ok(parent)
mkdir_exist_ok(root)
sftp.chdir(root)
for filename, content in self.files.items():
filename = pathlib.PurePosixPath(filename)
assert ".." not in filename.parts
mkdirs(filename)
mode = "wt" if isinstance(content, str) else "wb"
with sftp.file(str(filename), mode) as f:
# "b/t" modifier ignored in SFTP.
if mode == "wt":
f.write(content.encode("utf-8"))
else:
f.write(content)
if run_script:
transport = client.get_transport()
channel = transport.open_session()
channel.set_combine_stderr(True)
cmd = "if [ -f ~/.profile ]; then . ~/.profile; fi && cd {} && sh {}.sh".format(
root, self.script)
channel.exec_command(cmd)
# Show the output from the server while products are built.
buf = channel.recv(1024)
while buf:
print(buf.decode("utf-8"), end="")
buf = channel.recv(1024)
return RemoteSSHBuildProducts(connect_to, root)
def __init__(self, src, metadata):
self.kind = NodeKind.Document
self.metadata = metadata
self.src = src
self.path = pathlib.PurePosixPath(src)
#!/usr/bin/env python3
# encoding: utf-8
#
# Copyright (c) 2016 Doug Hellmann. All rights reserved.
# Written for https://pymotw.com
#
"""Parsing paths
"""
#end_pymotw_header
import pathlib
p = pathlib.PurePosixPath('./source/pathlib/pathlib_name.py')
print('path : {}'.format(p))
print('name : {}'.format(p.name))
print('suffix: {}'.format(p.suffix))
print('stem : {}'.format(p.stem))
def start_server(fname):
PORT = random_port()
LOCAL_IP = get_local_ip()
# Using .tmpqr since .tmp is very common
#create the .tmp_qr folder in /tmp file of unix system
TEMP_DIR_NAME = "/tmp/.tmp_qr"
# Variable to mark zip for deletion, if the user uses a folder as an argument
delete_zip = 0
# Checking if given fname is a path
if fname.startswith("/"):
os.chdir("/")
# Checking if given file name or path is a directory
if os.path.isdir(fname):
zip_name = pathlib.PurePosixPath(fname).name
try:
# Zips the directory
path_to_zip = make_archive(zip_name, "zip", fname)
fname = path_to_zip.replace(os.getcwd(), "")
# The above line replacement leaves a / infront of the file name
fname = fname.replace("/", "")
delete_zip = fname
except PermissionError:
print("Try with sudo")
sys.exit()
# Makes a directory name .tmp_qr and stores the file there
try:
os.makedirs(TEMP_DIR_NAME)
except:
print("Directory already exist"
) # preventing directory already exist crash..
try:
# Move the file to .tmpqr
copy2(fname, TEMP_DIR_NAME)
except FileNotFoundError:
print("File not found!")
rmtree(TEMP_DIR_NAME)
sys.exit()
# Change our directory to .tmpqr
os.chdir(TEMP_DIR_NAME)
handler = http.server.SimpleHTTPRequestHandler
httpd = socketserver.TCPServer(("", PORT), handler)
# tweaking fname to make a perfect url
fname = fname.split('/')
fname = fname[-1]
fname = fname.replace(" ", "%20")
# This is the url to be encoded into the QR code
address = "http://" + str(LOCAL_IP) + ":" + str(PORT) + "/" + fname
print(
"Scan the following QR to start downloading.\nMake sure that your smartphone is connected to the same WiFi network as this computer."
)
print_qr_code(address)
try:
httpd.serve_forever()
except KeyboardInterrupt:
os.chdir("..")
rmtree(TEMP_DIR_NAME)
# If the user sent a directory, a zip was created and then copied to the temporary directory, this deletes the first created zip
if delete_zip != 0:
os.remove(delete_zip)
print("\nExiting...")
sys.exit()
install_requires: List[str] = ["pyserial", "pyftdi", "pylibftdi"]
tests_require: List[str] = [
"pytest",
"pytest-cov",
"pytest-flake8",
"pytest-black",
"pytest-mypy",
]
dev_requires = install_requires + ["documenteer[pipelines]"]
scm_version_template = """# Generated by setuptools_scm
__all__ = ["__version__"]
__version__ = "{version}"
"""
tools_path = pathlib.PurePosixPath(setuptools.__path__[0])
base_prefix = pathlib.PurePosixPath(sys.base_prefix)
data_files_path = tools_path.relative_to(base_prefix).parents[1]
setuptools.setup(
name="ts_ess_controller",
description="Rubin Observatory Environment Sensors Support",
use_scm_version={
"write_to": "python/lsst/ts/ess/controller/version.py",
"write_to_template": scm_version_template,
},
setup_requires=["setuptools_scm"],
install_requires=install_requires,
package_dir={"": "python"},
packages=setuptools.find_namespace_packages(where="python"),
package_data={"": ["*.rst", "*.yaml", "*.xml"]},
# v1为Qt源码解压后的文件夹名,例如qt-everywhere-src-5.12.12
v1 = qt.replace('.zip', '')
# v为Qt的版本号,例如5.12.12
v = v1.replace('qt-everywhere-src-', '')
if not os.path.exists(v):
# 创建Qt版本号为名称的文件夹,将生成后的文件放置在该处
os.mkdir(v)
with zipfile.ZipFile(qt, 'r') as z:
z.extractall('.')
# 创建build文件夹
if not os.path.exists('build'):
os.mkdir('build')
# 将mingw32的路径声明为环境变量PATH
cwd = pathlib.PurePosixPath(pathlib.Path.cwd())
prefix = f'{cwd}/{v}'
os.environ['PATH'] = f'{cwd}/Tools/mingw32/bin;' + os.environ['PATH']
# 配置Qt
os.chdir('build')
run(f'{cwd}/{v1}/configure.bat -prefix {prefix} {args}')
# 编译
run('mingw32-make')
# 安装
run('mingw32-make install')
# mkspecs\common\目录下的gcc-base.conf文件中的QMAKE_LFLAGS参数值改为-static并保存
# mkspecs\common\目录下的g++-win32.conf文件中的QMAKE_LFLAGS_DLL参数值改为-static并保存
import pathlib
p = pathlib.PurePosixPath('/usr/local/lib')
print('parent: {}'.format(p.parent))
print('\nhierarchy:')
for up in p.parents:
print(up)
def has_target(self, target_file):
target = pathlib.PurePosixPath(target_file)
for x in self.decoders.values():
if target.match(x.as_posix()):
return True
return False
#import speech_recognition as sr
#recog = sr.Recognizer()
# redis
#redis_host = "localhost"
#redis_port = 6379
#redis_db = 0
# wav types
wav_files = ["context.wav","question.wav","choices.wav"]
# Serve Vue Application
index_view = never_cache(TemplateView.as_view(template_name='index.html'))
# Set asr decoder
decoder_dir = pathlib.PurePosixPath(asr_base_dir).joinpath('ailabs_asr/nnet3_adapt')
decoder = KgbAsrDecoder(decoder_dir=decoder_dir, KALDI_ROOT=KALDI_ROOT, base_dir=asr_base_dir)
# Set bert for qa task
bert_dir = os.path.join(asr_base_dir,'bert/bert_model')
bert_1 = BertClass(model_dir=os.path.join(bert_dir,'m33k_model'))
bert_2 = BertClass(model_dir=os.path.join(bert_dir,'m33k_model2'))
bert_3 = BertClass(model_dir=os.path.join(bert_dir,'all_model'))
bert_4 = BertClass(model_dir=os.path.join(bert_dir,'all_model2'))
# Set device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
def asr_passage_google(passage_path):
with sr.AudioFile(passage_path) as source:
speech = recog.record(source)
def extract_from_file(tool_name,
tool_args,
working_dir,
report_file,
file_path_list=None):
"""Extract properties from reports
:param tool_name: tool name
:param tool_args: tool args
:param working_dir: Working directory
:param report_file: Report file
:param file_path_list: Full file path for any manipulation
:return issues, metrics, skips information
"""
issues = []
metrics = None
skips = []
# If the tools did not produce any result do not crash
if not os.path.isfile(report_file):
return issues, metrics, skips
extn = pathlib.PurePosixPath(report_file).suffix
with io.open(report_file, "r") as rfile:
# Static check use jsonlines format, duh
if tool_name == "staticcheck":
contents = rfile.read()
try:
issues = [
json.loads(str(item))
for item in contents.strip().split("\n")
]
except json.decoder.JSONDecodeError:
LOG.warning(
"staticcheck produced no result since the project was not built before analysis!"
)
return issues, metrics, skips
if extn == ".json":
try:
report_data = json.loads(rfile.read())
except json.decoder.JSONDecodeError:
return issues, metrics, skips
# NG SAST (Formerly Inspect) uses vulnerabilities
if tool_name == "ng-sast":
for v in report_data.get("vulnerabilities"):
if not v:
continue
vuln = v["vulnerability"]
location_list = []
if vuln.get("dataFlow") and vuln.get("dataFlow",
{}).get("dataFlow"):
location_list = convert_dataflow(
working_dir, tool_args,
vuln["dataFlow"]["dataFlow"]["list"])
for location in location_list:
issues.append({
"rule_id":
vuln["category"],
"title":
vuln["title"],
"description":
vuln["description"],
"score":
vuln["score"],
"severity":
vuln["severity"],
"line_number":
location.get("line_number"),
"filename":
location.get("filename"),
"first_found":
vuln["firstVersionDetected"],
"issue_confidence":
"HIGH",
})
elif tool_name == "taint-php":
for entry in report_data:
taint_trace = entry.get("taint_trace")
labels = []
if taint_trace:
source, sink, labels = get_from_taints(taint_trace)
else:
source, _, _ = get_from_taints([entry])
issues.append({
"rule_id":
entry.get("shortcode"),
"test_name":
entry.get("type"),
"description":
"{}: {}".format(entry.get("message"),
"\\n".join(labels)),
"link":
entry.get("link"),
"severity":
entry.get("severity"),
"issue_confidence":
"HIGH",
"line_number":
source.get("line_number"),
"filename":
source.get("filename"),
})
elif tool_name == "taint-python":
taint_list = report_data.get("vulnerabilities")
for taint in taint_list:
source = taint.get("source")
sink = taint.get("sink")
issues.append({
"rule_id":
taint.get("rule_id"),
"test_name":
taint.get("rule_name"),
"short_description":
taint.get("short_description"),
"cwe_category":
taint.get("cwe_category"),
"owasp_category":
taint.get("owasp_category"),
"description":
taint.get("description"),
"severity":
taint.get("severity"),
"issue_confidence":
"HIGH",
"line_from":
source.get("line_number"),
"line_to":
sink.get("line_number"),
"filename":
source.get("path"),
})
elif tool_name == "phpstan" or tool_name == "source-php":
file_errors = report_data.get("files")
for filename, messageobj in file_errors.items():
messages = messageobj.get("messages")
for msg in messages:
# Create a rule id for phpstan
rule_word = msg.get("message", "").split(" ")[0]
rule_word = "phpstan-" + rule_word.lower()
issues.append({
"rule_id": rule_word,
"title": msg.get("message"),
"line_number": msg.get("line"),
"filename": filename,
"severity": "LOW",
"issue_confidence": "MEDIUM",
})
elif tool_name == "source-js":
njs_findings = report_data.get("nodejs", {})
njs_findings.update(report_data.get("templates", {}))
for k, v in njs_findings.items():
# Password detection by njsscan is full of false positives
if k == "node_password":
continue
files = v.get("files", [])
metadata = v.get("metadata", {})
if not files or not metadata:
continue
for afile in files:
line_number = 0
if afile.get("match_lines"):
line_number = afile.get("match_lines")[0]
issues.append({
"rule_id":
metadata.get("owasp").replace(":", "-").replace(
" ", "").lower(),
"title":
metadata.get("cwe"),
"description":
metadata.get("description"),
"severity":
metadata.get("severity"),
"line_number":
line_number,
"filename":
afile.get("file_path"),
"issue_confidence":
"HIGH",
})
elif tool_name == "checkov":
if isinstance(report_data, list):
for rd in report_data:
issues += rd.get("results", {}).get("failed_checks")
else:
issues = report_data.get("results",
{}).get("failed_checks")
elif isinstance(report_data, list):
issues = report_data
else:
if "sec_issues" in report_data:
# NodeJsScan uses sec_issues
sec_data = report_data["sec_issues"]
for key, value in sec_data.items():
if isinstance(value, list):
issues = issues + value
else:
issues.append(value)
elif "Issues" in report_data:
tmpL = report_data.get("Issues", [])
if tmpL:
issues += tmpL
else:
LOG.debug("%s produced no result" % tool_name)
elif "results" in report_data:
tmpL = report_data.get("results", [])
if tmpL:
issues += tmpL
else:
LOG.debug("%s produced no result" % tool_name)
if extn == ".csv":
headers, issues = csv_parser.get_report_data(rfile)
if extn == ".xml":
issues, metrics = xml_parser.get_report_data(
rfile, file_path_list=file_path_list, working_dir=working_dir)
return issues, metrics, skips
def test_escape_path_with_dotdot_components_raises(self) -> None:
path = pathlib.PurePosixPath("/path/to/../file.txt")
with self.assertRaises(ValueError):
systemd_escape_path(path)
