def run(self):
# Eveloghandler
self.eveloghandler_watchdog = eveloghandler.EveLogHandler(
self.watched_channels, self.ignore_patterns,
self.ignore_directories, self.case_sensitive, self.configuration,
self.logger)
self.eveloghandler_watchdog.message_ready.connect(
self.test_catch_connection)
# Observer
self.watchdog_observer = Observer()
self.watchdog_observer.schedule(
self.eveloghandler_watchdog,
self.configuration.value['eve_log_location'], False)
self.watchdog_observer.start()
while not self.event_stop.is_set():
# Windows workaround - for some reason Eve doesn't trigger the watchdog modify event on Windows.
if self.platform == "windows":
path = Path(self.configuration.value["eve_log_location"])
for file in path.iterdir():
Path(file).touch()
time.sleep(2)
else:
time.sleep(1)
# when stopping
self.eveloghandler_watchdog.pickle_dict()
self.watchdog_observer.stop()
def convert_to_csv_from_folder(self,
dat_folder: str,
csv_folder: Optional[str] = None) -> None:
"""Convert all dat in a folder to csv
Iterate over the given folder
Then normalize each .dat file found to .csv
:param dat_folder: folder containing .dat files
:param csv_folder: folder in which store CSV
:return: None
"""
folder = Path(dat_folder)
if not folder.exists() \
or not folder.is_dir():
raise BadFileFormatException
if not csv_folder:
csv_folder = self.DEFAULT_OUTPUT_FOLDER
for file in folder.iterdir():
if file.suffix != Dat.ext:
continue
self.convert_to_csv(
dat_path=str(file),
csv_path=f'{csv_folder}{file.name.replace(Dat.ext, Csv.ext)}')
def _get_system_wide_trusted_ca_certificates(self):
trusted_cas, errors = set([]), []
for p in self.system_wide_trusted_ca_search_paths:
cert_path = Path(p)
if not cert_path.is_dir():
continue
for entry in cert_path.iterdir():
cert_file_path = entry.absolute()
try:
if entry.suffix not in [".pem", ".crt"]:
continue
trusted_cas.update(self._get_certificates_from_file(cert_file_path))
except IOError:
logger.exception()
# This error is shown to the user as warning message during "activate changes".
# We keep this message for the moment because we think that it is a helpful
# trigger for further checking web.log when a really needed certificate can
# not be read.
#
# We know a permission problem with some files that are created by default on
# some distros. We simply ignore these files because we assume that they are
# not needed.
if cert_file_path == Path("/etc/ssl/certs/localhost.crt"):
continue
errors.append("Failed to add certificate '%s' to trusted CA certificates. "
"See web.log for details." % cert_file_path)
break
return list(trusted_cas), errors
def get_roi_from_gt(data_path, roi_file):
data_path = Path(data_path)
cases = sorted([d for d in data_path.iterdir() if d.is_dir()])
case_idx = 0
rois = {}
for case in tqdm(cases, ascii=True, dynamic_ncols=True):
img_file = case / 'imaging.nii.gz'
assert img_file.exists()
img = nib.load(str(img_file)).get_data()
total_z, total_y, total_x = img.shape
vol = {'total_x': total_x, 'total_y': total_y, 'total_z': total_z}
case_data = {'vol': vol}
seg_file = case / 'segmentation.nii.gz'
if seg_file.exists():
seg = nib.load(str(seg_file)).get_data()
kidney = calc(seg, idx=1)
tumor = calc(seg, idx=2)
case_data.update({'kidney': kidney, 'tumor': tumor})
rois[f'case_{case_idx:05d}'] = case_data
with open(roi_file, 'w') as f:
json.dump(rois, f, indent=4, separators=(',', ': '))
case_idx += 1
def main(src, dest):
"""links configfiles from one folder to another
if links exists it verifies content
if files exist at the target side it errors
Args:
src: source folder
dest: target folder
"""
src = Path(src)
if not src.exists():
print("WARNING:", src, "does not exist, skipping linking")
return
dest = Path(dest)
for element in filter(_is_yaml_file, src.iterdir()):
_warn_on_unknown_encryption(element)
target = dest.joinpath(element.name)
# the following is fragile
if target.is_symlink():
_warn_on_missmatching_symlink(src=element, target=target)
elif target.is_file():
_warn_on_existing_file(target)
else:
target.symlink_to(element.resolve())
def upload_img(request):
# 请求方法为POST时,进行处理;
if request.method == "POST":
# 获取上传的文件,如果没有文件,则默认为None;
print 'in post...'
File = request.FILES.get("myfile", None)
if File is None:
return HttpResponse("no files for upload!")
else:
LIB_PATH = ABS_PATH + 'heliwu'
# 读出所有 lib name 列表
lib_path = Path(ABS_PATH)
lib_names = []
for lib in lib_path.iterdir():
lib_names.append(lib.name)
# 打开特定的文件进行二进制的写操作;
image_file_path = LIB_PATH + "/%s" % File.name
with open(image_file_path, 'wb+') as f:
# 分块写入文件;
for chunk in File.chunks():
f.write(chunk)
return render(request,
'predict.html',
context={
"lib_names": lib_names,
'image_file_path': image_file_path
})
else:
print 'in get...'
return render(request, 'upload_img.html')
def scrape(self, path: pathlib2.Path, depth: int) -> Tuple[FileNode, bool]:
"""
Use recursion to scrape a given path and return a tree structure
:param path: target file path to scrape
:param depth: depth of node with respect to the root node
:return: the scraped file node tree and whether any target files set by filters were found
"""
children = []
found_any = False
paths = list(path.iterdir())
for filter_ in self.filters:
paths = filter_(paths)
if depth != self.depth_limit:
for filepath in paths:
node = FileNode(filepath, depth=depth + 1, root=self.root)
if (filepath.is_symlink() or filepath.is_dir()
) and node.get_id() not in self.history:
subtree, found_any_ = self.scrape(filepath, depth + 1)
if found_any_:
found_any = True
if self._keep_empty_dir or found_any_:
children.append(subtree)
elif filepath.is_file():
children.append(node)
found_any = True
else:
pass
self.history.add(node.get_id())
return FileNode(path, children=children, depth=depth,
root=self.root), found_any
def main(src, dest):
"""links configfiles from one folder to another
if links exists it verifies content
if files exist at the target side it errors
Args:
src: source folder
dest: target folder
"""
src = Path(src)
if not src.exists():
print("WARNING:", src, "does not exist, skipping linking")
return
dest = Path(dest)
for element in filter(_is_yaml_file, src.iterdir()):
_warn_on_unknown_encryption(element)
target = dest.joinpath(element.name)
# the following is fragile
if target.is_symlink():
_warn_on_missmatching_symlink(src=element, target=target)
elif target.is_file():
_warn_on_existing_file(target)
else:
target.symlink_to(element.resolve())
def download_images(self, services, registry, tag=None, nopull=False):
"""Download images from a remote registry and save to kard
Args:
* services: the name of the images to download
* registry: a DockerRegistry instance
* tag: the tag of the version to download
"""
tag = tag or self.kard.meta['tag']
save_path = Path(self.kard.path) / 'images'
write('Cleaning images destination {}'.format(save_path))
save_path.mkdir(exist_ok=True)
for child in save_path.iterdir():
child.unlink()
if not nopull:
self.pull_images(services, registry, tag=tag)
for service in services:
image_path = save_path / "{}.tar".format(service)
image_name = self.make_image_name(service, tag)
write('Saving {} to {}'.format(image_name, image_path))
sys.stdout.flush()
with open(image_path, 'wb') as f:
for chunk in self.docker.get_image(image_name):
f.write(chunk)
write(' Done !' + '\n')
write('All images has been saved successfully !' + '\n')
def move_flattened_files(src_dir, out_dir, filt):
p = Path(src_dir)
if p.is_dir():
for f in p.iterdir():
if f.is_file() and filt(f.absolute()):
copyfile(str(f.absolute()),str(out_dir + u'/' + f.name))
else:
move_flattened_files(str(f.absolute()), out_dir, filt)
def conversion_all(data, output):
data = Path(data)
output = Path(output)
cases = sorted([d for d in data.iterdir() if d.is_dir()])
pool = mp.Pool()
pool.map(conversion, zip(cases, [output] * len(cases)))
pool.close()
pool.join()
def _get_data(request, data_type):
data_dir = Path(DATA_PATH)
result = None
for file in data_dir.iterdir():
parts = file.stem.split('__')
test_name = request.node.name.split('test_')[-1]
test_name = test_name[:test_name.index('[')] if '[' in test_name else test_name
if parts[0] == data_type:
if parts[1] == test_name or (parts[1] == 'default' and result is None):
result = yaml.safe_load(file.read_text())
return result
def _get_cache_file(self, remote_url):
"""
:param remote_url: check if we have the remote url in our cache
:return: full path to file name, current file size or None
"""
folder = Path(get_cache_dir() /
CacheManager._storage_manager_folder / self._context)
folder.mkdir(parents=True, exist_ok=True)
local_filename = self._get_hashed_url_file(remote_url)
new_file = folder / local_filename
if new_file.exists():
new_file.touch(exist_ok=True)
# delete old files
def sort_max_access_time(x):
atime = x.stat().st_atime
# noinspection PyBroadException
try:
if x.is_dir():
dir_files = list(x.iterdir())
atime = (max(
atime, max(
s.stat().st_atime
for s in dir_files)) if dir_files else atime)
except Exception:
pass
return atime
files = sorted(folder.iterdir(),
reverse=True,
key=sort_max_access_time)
files = files[self._file_limit:]
for f in files:
if not f.is_dir():
f.unlink()
else:
try:
shutil.rmtree(f)
except Exception as e:
# failed deleting folder
LoggerRoot.get_base_logger().warning(
"Exception {}\nFailed deleting folder {}".format(
e, f))
# if file doesn't exist, return file size None
return (
new_file.as_posix(),
new_file.stat().st_size if new_file.exists() else None,
)
def test_all_files_cleaned_up(builder):
builder.assign('x', 1)
@builder
def x_plus_one(x):
return x + 1
flow = builder.build()
assert flow.get('x_plus_one') == 2
flow = builder.build()
assert flow.get('x_plus_one') == 2
tmp_dir_path = Path(flow.get('core__persistent_cache__flow_dir')) / 'tmp'
assert list(tmp_dir_path.iterdir()) == []
def test_update_and_build():
if quick_tests_only:
return
source = Path(__file__).parent / 'Examples'
destination = Path(tempfile.mkdtemp()) / 'Examples'
shutil.copytree(source, destination)
project_paths = [path for path in destination.iterdir() if path.is_dir()]
for project_path in project_paths:
with work_directory(project_path):
output = runner.check_run('punic update')
def test_update_and_build():
if quick_tests_only:
return
source = Path(__file__).parent / 'Examples'
destination = Path(tempfile.mkdtemp()) / 'Examples'
shutil.copytree(source, destination)
project_paths = [path for path in destination.iterdir() if path.is_dir()]
for project_path in project_paths:
with work_directory(project_path):
output = runner.check_run('punic update')
def py_git_stat():
# get the current working directory
cwd = os.getcwd()
# status messages are friends not food
print('Listing git statuses for ' + cwd)
# set up Path object in cwd to start workin'
p = Path(cwd)
# get subdirs of the cwd
subdirs = [item for item in p.iterdir() if item.is_dir()]
# then, see if each subdir contains ".git" and put it in git_repos list
git_repos = []
for subdir in subdirs:
q = subdir / '.git'
if q.exists():
git_repos.append(subdir)
# now that we have only top-level git repos, let's execute git status on all of them, then store the
# command output
status_results = {}
for git_repo in git_repos:
os.chdir(str(git_repo.resolve()))
result = subprocess.check_output(['git', 'status'])
status_results.update({git_repo.name: result.split('\n')})
# let's print out some headers for the table. first, get width of the terminal
if get_terminal_size()[0] != 0:
term_width = get_terminal_size()[0]
else:
term_width = 80
print_row_separators(term_width)
# let's get the longest repo name and make that the width of the first column in the table.
longest_repo_name = (get_longest_element(status_results)) + 1
longest_repo_branch = (get_longest_element(status_results.values()[1])) + 1
print('{0: <{longest_repo_name}}|{1}'.format(
'git repository', 'repo branch', longest_repo_name=longest_repo_name))
print_row_separators(term_width)
for repo_name, repo_info in status_results.iteritems():
print('{0: <{longest_repo_name}}|{1: <{longest_repo_branch}}'.format(
repo_name,
repo_info[0],
longest_repo_name=longest_repo_name,
longest_repo_branch=longest_repo_branch))
def update_journal(rrdfile, rrdfilenew):
journaldir = Path(cmk.utils.paths.omd_root, 'var/rrdcached/')
for filepath in journaldir.iterdir():
logger.info('- Updating journal file %s', filepath)
new_file = filepath.with_suffix(filepath.suffix + '.new')
try:
with filepath.open('r') as old_jou, new_file.open('w') as new_jou:
for line in old_jou:
if rrdfile in line:
line = line.replace(rrdfile, rrdfilenew)
new_jou.write(line)
except Exception:
new_file.unlink()
raise
finally:
if new_file.exists():
filepath.unlink()
new_file.rename(filepath)
def main(src, dest, force):
"""links configfiles from one folder to another
if links exists it verifies content
if files exist at the target side it errors
Args:
src: source folder
dest: target folder
force: override existing symlinks
"""
src = Path(src)
if not src.exists():
print("WARNING:", src, "does not exist, skipping linking")
return
dest = Path(dest)
for element in filter(_is_yaml_file, src.iterdir()):
_warn_on_unknown_encryption(element)
target = dest.joinpath(element.name)
if force:
try:
target.symlink_to(element.resolve())
except OSError as e:
if e.errno == errno.EEXIST:
backup_target = Path(dest.joinpath(element.name + "_bak"))
print("Replacing", target.name, "and saving backup as", backup_target.name)
# Would use 'backup_target.replace()' here but that's only supported in py3
if backup_target.exists():
os.remove(str(backup_target))
target.rename(backup_target)
target.symlink_to(element.resolve())
else:
raise
else:
if target.is_symlink():
# If symlink already exists and points to same src, do nothing.
_check_missmatching_symlink(src=element, target=target)
elif _check_existing_file(target):
target.symlink_to(element.resolve())
print("Symlink created for", target.name)
def update(self, path):
if (type(path) != str):
index = path[1]
path = path[0]
config = Path(path)
tknzr = TweetTokenizer()
lex, doc = dict(), dict()
if (config.is_file()): # If just a file
with config.open() as f:
sentences = sent_tokenize(f.read())
document = path.split('/')[-1]
doc[document] = list(sentences)
for i, sentence in enumerate(sentences):
token_list = tknzr.tokenize(sentence)
for word in token_list:
if (word not in self.stopwords
and self.is_not_number(word)
and len(word) > 1):
if (pos_tag([word])[0][1] != 'NNP'):
word = word.lower()
if (word not in lex):
lex[word] = Word(word, document, i)
else:
lex[word].update(document, i)
return (lex, doc)
elif (config.is_dir()
): # If directory, recursively read each directory and file
if (path[-1] != '/'): path = path + '/'
iterdir = list()
for d in config.iterdir():
iterdir.append(path + d.name)
self.outputs = [0] * len(list(iterdir))
p = Pool(self.maxPoolSize)
outputs = p.map(self.update, zip(iterdir,
list(range(len(iterdir)))))
for (lex, doc) in outputs:
for k, v in lex.items():
if (k not in self.lexicon):
self.lexicon[k] = Word(k, v.documents)
self.lexicon[k].count += v.count
self.lexicon[k].documents.update(v.documents)
self.documents.update(doc)
else:
print('{} doesn\'t exist.'.format(path))
def create_tables(self,
sql_folder: str = DatabaseUtils.sql_tables_path) -> None:
"""Create all tables from their schemas
:see DatabaseUtils.sql_tables:
"""
# reach the sql sources folder
folder = Path(sql_folder)
if not folder.exists() \
or not folder.is_dir():
raise BadFileFormatException
# for each file in the folder
for file in folder.iterdir():
# if it is an sql file, execute it
if file.suffix != DatabaseUtils.sql_extension:
continue
self._cursor.execute(file.read_text())
# commit transactions changes
self._connection.commit()
def copy_dir(source, target):
"""Remove existing files/folders before copying."""
source = Path(source)
if not source.exists():
return 0
target = Path(target)
if not target.exists():
target.mkdir(parents=1)
for s in source.iterdir():
t = target.joinpath(s.name)
if t.exists():
if t.is_dir():
shutil.rmtree(str(t))
else:
os.remove(str(t))
if s.is_dir():
shutil.copytree(str(s), str(t))
else:
shutil.copyfile(str(s), str(t))
def import_images(self, services, tag=None):
"""Import images from kard to local docker
Args:
* services: the name of the images to load
* tag: the tag of the version to load
"""
tag = tag or self.kard.meta['tag']
save_path = Path(self.kard.path) / 'images'
for child in save_path.iterdir():
service = child.name[:-4]
if service not in services:
continue
write('Importing {} ...'.format(child))
with open(child, 'rb') as f:
rsp = self.docker.load_image(f.read())
for message in rsp:
write(message.get('stream', ''))
write('\n')
write('All images have been loaded successfully !' + '\n')
def _get_cache_file(self, remote_url):
"""
:param remote_url: check if we have the remote url in our cache
:return: full path to file name, current file size or None
"""
folder = Path(get_cache_dir() /
CacheManager._storage_manager_folder / self._context)
folder.mkdir(parents=True, exist_ok=True)
local_filename = self._get_hashed_url_file(remote_url)
new_file = folder / local_filename
if new_file.exists():
new_file.touch(exist_ok=True)
# delete old files
files = sorted(folder.iterdir(),
reverse=True,
key=lambda x: x.stat().st_atime)
files = files[self._file_limit:]
for f in files:
f.unlink()
# if file doesn't exist, return file size None
return new_file.as_posix(), new_file.stat(
).st_size if new_file.exists() else None
def get_cache_file(self, remote_url=None, local_filename=None):
# type: (Optional[str], Optional[str]) -> Tuple[str, Optional[int]]
"""
:param remote_url: check if we have the remote url in our cache
:param local_filename: if local_file is given, search for the local file/directory in the cache folder
:return: full path to file name, current file size or None
"""
def safe_time(x):
# noinspection PyBroadException
try:
return x.stat().st_mtime
except Exception:
return 0
def sort_max_access_time(x):
atime = safe_time(x)
# noinspection PyBroadException
try:
if x.is_dir():
dir_files = list(x.iterdir())
atime = (max(atime, max(
safe_time(s)
for s in dir_files)) if dir_files else atime)
except Exception:
pass
return atime
folder = Path(get_cache_dir() /
CacheManager._storage_manager_folder / self._context)
folder.mkdir(parents=True, exist_ok=True)
local_filename = local_filename or self.get_hashed_url_file(
remote_url)
new_file = folder / local_filename
new_file_exists = new_file.exists()
if new_file_exists:
# noinspection PyBroadException
try:
new_file.touch(exist_ok=True)
except Exception:
pass
# first exclude lock files
lock_files = dict()
files = []
for f in sorted(folder.iterdir(),
reverse=True,
key=sort_max_access_time):
if f.name.startswith(
CacheManager._lockfile_prefix) and f.name.endswith(
CacheManager._lockfile_suffix):
# parse the lock filename
name = f.name[len(CacheManager._lockfile_prefix
):-len(CacheManager._lockfile_suffix)]
num, _, name = name.partition(".")
lock_files[name] = lock_files.get(name,
[]) + [f.as_posix()]
else:
files.append(f)
# remove new lock files from the list (we will delete them when time comes)
for f in files[:self._file_limit]:
lock_files.pop(f.name, None)
# delete old files
files = files[self._file_limit:]
for i, f in enumerate(files):
if i < self._file_limit:
continue
# check if the file is in the lock folder list:
folder_lock = self._folder_locks.get(f.absolute().as_posix())
if folder_lock:
# pop from lock files
lock_files.pop(f.name, None)
continue
# check if someone else holds the lock file
locks = lock_files.get(f.name, [])
for lck in locks:
try:
a_lock = FileLock(filename=lck)
a_lock.acquire(timeout=0)
a_lock.release()
a_lock.delete_lock_file()
del a_lock
except LockException:
# someone have the lock skip the file
continue
# if we got here we need to pop from the lock_files, later we will delete the leftover entries
lock_files.pop(f.name, None)
# if we are here we can delete the file
if not f.is_dir():
# noinspection PyBroadException
try:
f.unlink()
except Exception:
pass
else:
try:
shutil.rmtree(f.as_posix(), ignore_errors=False)
except Exception as e:
# failed deleting folder
LoggerRoot.get_base_logger().debug(
"Exception {}\nFailed deleting folder {}".format(
e, f))
# cleanup old lock files
for lock_files in lock_files.values():
for f in lock_files:
# noinspection PyBroadException
try:
os.unlink(f)
except BaseException:
pass
# if file doesn't exist, return file size None
# noinspection PyBroadException
try:
size = new_file.stat().st_size if new_file_exists else None
except Exception:
size = None
return new_file.as_posix(), size
from lisbonpose.lisbonpose import Lisbon
import cv2
import numpy as np
from pathlib2 import Path
lisbon = Lisbon()
vidspath = Path('Data/Videos/')
vidnames = [e for e in vidspath.iterdir()]
for i in vidnames:
#print(i.suffix)
#print(i.stem)
info = i.stem.split('_')
info.pop()
seperator = '/'
new_path = 'Data/clean/' + seperator.join(info)
Path(new_path).mkdir(parents=True, exist_ok=True)
new_vidpath = Path(new_path) / i.name
if not new_vidpath.exists():
with new_vidpath.open(mode='xb') as fid:
fid.write(i.read_bytes())
def get_cache_file(self, remote_url=None, local_filename=None):
"""
:param remote_url: check if we have the remote url in our cache
:param local_filename: if local_file is given, search for the local file/directory in the cache folder
:return: full path to file name, current file size or None
"""
def safe_time(x):
# noinspection PyBroadException
try:
return x.stat().st_mtime
except Exception:
return 0
def sort_max_access_time(x):
atime = safe_time(x)
# noinspection PyBroadException
try:
if x.is_dir():
dir_files = list(x.iterdir())
atime = (max(atime, max(
safe_time(s)
for s in dir_files)) if dir_files else atime)
except Exception:
pass
return atime
folder = Path(get_cache_dir() /
CacheManager._storage_manager_folder / self._context)
folder.mkdir(parents=True, exist_ok=True)
local_filename = local_filename or self._get_hashed_url_file(
remote_url)
new_file = folder / local_filename
new_file_exists = new_file.exists()
if new_file_exists:
# noinspection PyBroadException
try:
new_file.touch(exist_ok=True)
except Exception:
pass
# delete old files
files = sorted(folder.iterdir(),
reverse=True,
key=sort_max_access_time)
files = files[self._file_limit:]
for f in files:
if not f.is_dir():
# noinspection PyBroadException
try:
f.unlink()
except Exception:
pass
else:
try:
shutil.rmtree(f)
except Exception as e:
# failed deleting folder
LoggerRoot.get_base_logger().debug(
"Exception {}\nFailed deleting folder {}".format(
e, f))
# if file doesn't exist, return file size None
# noinspection PyBroadException
try:
size = new_file.stat().st_size if new_file_exists else None
except Exception:
size = None
return new_file.as_posix(), size
#!/usr/bin/python2
import numpy as np
import cv2 as cv
from matplotlib import pyplot as plt
from pathlib2 import Path
MIN_MATCH_COUNT = 10
path = Path('./Sample Banana Plant/Images')
img_path_list = sorted([str(x) for x in path.iterdir()])
# Initiate ORB detector
orb = cv.ORB_create()
img_info = []
print("Obtaining keypoints and descriptors.")
for img_path in img_path_list:
print(img_path)
img = cv.imread(img_path, 1)
# find the keypoints and descriptors with ORB
kp, des = orb.detectAndCompute(cv.cvtColor(img, cv.COLOR_RGB2GRAY), None)
img_info.append({
"img": img,
from Bio.Blast import NCBIWWW
from pathlib2 import Path
import logging
from datetime import datetime
data_in_catalog = "DataInExample"
data_out_catalog = 'DataOutExample'
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
startTime = datetime.now()
data_in = Path('.') / data_in_catalog
bacteria_paths = [x for x in data_in.iterdir() if x.is_file()]
for microbe_path in bacteria_paths:
logging.info("Processing: {0}".format(microbe_path))
with microbe_path.open() as f:
rna = f.read()
if rna == "":
logging.warning("EMPTY FILE !!!")
continue
microbe = microbe_path.stem
p = Path('.')
path = p / data_out_catalog / microbe
path.mkdir(parents=True, exist_ok=True)
def handle(self, *filenames, **options):
"""
* All yang modules from published RFCs should be extracted and be
available in an rfc-yang repository.
* All valid yang modules from active, not replaced, internet drafts
should be extracted and be available in a draft-valid-yang repository.
* All, valid and invalid, yang modules from active, not replaced,
internet drafts should be available in a draft-all-yang repository.
(Actually, given precedence ordering, it would be enough to place
non-validating modules in a draft-invalid-yang repository instead).
* In all cases, example modules should be excluded.
* Precedence is established by the search order of the repository as
provided to pyang.
* As drafts expire, models should be removed in order to catch cases
where a module being worked on depends on one which has slipped out
of the work queue.
"""
verbosity = int(options.get('verbosity'))
def extract_from(file, dir, strict=True):
saved_stdout = sys.stdout
saved_stderr = sys.stderr
xymerr = StringIO()
xymout = StringIO()
sys.stderr = xymerr
sys.stdout = xymout
model_list = []
try:
model_list = xym.xym(str(file),
str(file.parent),
str(dir),
strict=strict,
debug_level=verbosity - 2)
for name in model_list:
modfile = moddir / name
mtime = file.stat().st_mtime
os.utime(str(modfile), (mtime, mtime))
if '"' in name:
name = name.replace('"', '')
modfile.rename(str(moddir / name))
model_list = [n.replace('"', '') for n in model_list]
except Exception as e:
print("** Error when extracting from %s: %s" % (file, str(e)))
sys.stdout = saved_stdout
sys.stderr = saved_stderr
#
if verbosity > 1:
outmsg = xymout.getvalue()
self.stdout.write(outmsg)
if verbosity > 2:
errmsg = xymerr.getvalue()
self.stderr.write(errmsg)
return model_list
# Extract from new RFCs
rfcdir = Path(settings.RFC_PATH)
moddir = Path(settings.SUBMIT_YANG_RFC_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
latest = 0
for item in moddir.iterdir():
if item.stat().st_mtime > latest:
latest = item.stat().st_mtime
print("Extracting to %s ..." % moddir)
for item in rfcdir.iterdir():
if item.is_file() and item.name.startswith(
'rfc') and item.name.endswith(
'.txt') and item.name[3:-4].isdigit():
if item.stat().st_mtime > latest:
model_list = extract_from(item, moddir)
for name in model_list:
if name.startswith('ietf') or name.startswith('iana'):
if verbosity > 1:
print(" Extracted from %s: %s" % (item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
modfile = moddir / name
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" %
(item, name))
print("")
# Extract valid modules from drafts
six_months_ago = time.time() - 6 * 31 * 24 * 60 * 60
def active(item):
return item.stat().st_mtime > six_months_ago
draftdir = Path(settings.INTERNET_DRAFT_PATH)
moddir = Path(settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
print("Emptying %s ..." % moddir)
for item in moddir.iterdir():
item.unlink()
print("Extracting to %s ..." % moddir)
for item in draftdir.iterdir():
try:
if item.is_file() and item.name.startswith(
'draft') and item.name.endswith('.txt') and active(
item):
model_list = extract_from(item, moddir)
for name in model_list:
if not name.startswith('example'):
if verbosity > 1:
print(" Extracted valid module from %s: %s" %
(item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
modfile = moddir / name
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" %
(item, name))
except UnicodeDecodeError as e:
sys.stderr.write('\nError: %s\n' % (e, ))
sys.stderr.write(item.name)
sys.stderr.write('\n')
print("")
# Extract invalid modules from drafts
valdir = moddir
moddir = Path(settings.SUBMIT_YANG_INVAL_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
print("Emptying %s ..." % moddir)
for item in moddir.iterdir():
item.unlink()
print("Extracting to %s ..." % moddir)
for item in draftdir.iterdir():
try:
if item.is_file() and item.name.startswith(
'draft') and item.name.endswith('.txt') and active(
item):
model_list = extract_from(item, moddir, strict=False)
for name in model_list:
modfile = moddir / name
if (valdir / name).exists():
modfile.unlink()
if verbosity > 1:
print(" Skipped valid module from %s: %s" %
(item, name))
elif not name.startswith('example'):
if verbosity > 1:
print(
" Extracted invalid module from %s: %s" %
(item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" %
(item, name))
except UnicodeDecodeError as e:
sys.stderr.write('\nError: %s\n' % (e, ))
sys.stderr.write(item.name)
sys.stderr.write('\n')
print("")
args['model_type'] = 'tfv2-MobileNetV2'
# mlflow.log_params(args)
print('Writing out params...', end='')
with open(str(params), 'w') as f:
json.dump(args, f)
print(' Saved to {}'.format(str(params)))
info('Log Traning Parameters')
parmeters = pd.read_json(str(params), typ='series')
metadata = {
'outputs': [{
'type': 'table',
'storage': 'inline',
'format': 'csv',
'header': ['Name', 'Value'],
'source': parmeters.to_csv()
}]
}
# Path(output_path).parent.mkdir(parents=True, exist_ok=True)
with open(str(ui_metadata_file), 'w') as f:
json.dump(metadata, f)
model_output_content = []
for filename in target_path.iterdir():
model_output_content.append(str(filename))
with open(str(output_model_file), 'w+') as f:
f.write('\n'.join(model_output_content))
from pyjoplin.models import Note, NoteIndex, database as db
from pyjoplin.configuration import config
from pyjoplin import notification
path_exportdir = Path('/run/media/jesusbriales/KINGSTON/joplin-ThinkPad/')
printc = lambda x: print(colored(x, 'cyan'))
notes = Note.select().order_by(Note.title)
print("Listing empty notes:")
for note in notes:
if not note.body:
printc('Empty: %s %s' % (note.id, note.title))
notefile = '%s.md' % note.id
if path_exportdir / notefile in path_exportdir.iterdir():
note.from_file(str(path_exportdir / notefile))
filtered_lines = list()
for line in note.body.split('\n'):
if line.startswith('id: '):
break
filtered_lines.append(line)
note.body = '\n'.join(filtered_lines)
if not note.body:
continue
printc('Found non-empty export file:')
print(note.title)
print('')
print(note.body)
yn = raw_input('Save loaded note content? (Y/n)')
if yn == 'y' or yn == '':
class InventoryHousekeeping(object):
def __init__(self):
super(InventoryHousekeeping, self).__init__()
self._inventory_path = Path(cmk.utils.paths.var_dir) / "inventory"
self._inventory_archive_path = Path(
cmk.utils.paths.var_dir) / "inventory_archive"
self._inventory_delta_cache_path = Path(
cmk.utils.paths.var_dir) / "inventory_delta_cache"
def run(self):
if not self._inventory_delta_cache_path.exists(
) or not self._inventory_archive_path.exists( # pylint: disable=no-member
):
return
last_cleanup = self._inventory_delta_cache_path / "last_cleanup"
# TODO: remove with pylint 2
if last_cleanup.exists(
) and time.time() - last_cleanup.stat().st_mtime < 3600 * 12: # pylint: disable=no-member
return
# TODO: remove with pylint 2
inventory_archive_hosts = {
x.name
for x in self._inventory_archive_path.iterdir() if x.is_dir() # pylint: disable=no-member
}
inventory_delta_cache_hosts = {
x.name
for x in self._inventory_delta_cache_path.iterdir() if x.is_dir() # pylint: disable=no-member
}
folders_to_delete = inventory_delta_cache_hosts - inventory_archive_hosts
for foldername in folders_to_delete:
shutil.rmtree(str(self._inventory_delta_cache_path / foldername))
inventory_delta_cache_hosts -= folders_to_delete
for hostname in inventory_delta_cache_hosts:
available_timestamps = self._get_timestamps_for_host(hostname)
for filename in [
x.name for x in (self._inventory_delta_cache_path /
hostname).iterdir() if not x.is_dir()
]:
delete = False
try:
first, second = filename.split("_")
if first not in available_timestamps or second not in available_timestamps:
delete = True
except ValueError:
delete = True
if delete:
(self._inventory_delta_cache_path / hostname /
filename).unlink()
# TODO: remove with pylint 2
last_cleanup.touch() # pylint: disable=no-member
def _get_timestamps_for_host(self, hostname):
timestamps = {"None"} # 'None' refers to the histories start
try:
timestamps.add("%d" %
(self._inventory_path / hostname).stat().st_mtime)
except OSError:
pass
for filename in [
x for x in (self._inventory_archive_path / hostname).iterdir()
if not x.is_dir()
]:
timestamps.add(filename.name)
return timestamps
import sys
from pathlib2 import Path
if __name__ == '__main__':
TESTCASE_DIR = Path(__file__).resolve().parent.joinpath('testcase')
sys.path.append(str(TESTCASE_DIR))
for p in TESTCASE_DIR.iterdir():
if p.name.startswith('test') and p.name.endswith('.py'):
if sys.version_info[0] == 3:
import importlib.util
spec = importlib.util.spec_from_file_location(p.name, str(p))
testcase = importlib.util.module_from_spec(spec)
spec.loader.exec_module(testcase)
testcase.run()
else:
import imp
testcase = imp.load_source(p.name, str(p))
testcase.run()
