def process_one_file(self, url, desired_filename, filename_suff,
raw_xml_str):
xml_etree = etree.fromstring(raw_xml_str)
xml_filename = check_for_filename_collision(url, desired_filename,
filename_suff,
self._assigned_filenames,
self._tei_logger)
out_filename = os_path_basename(xml_filename)
try:
self._validator.assert_(xml_etree)
valid = True
except AssertionError as err:
self._tei_logger.log('ERROR', 'TEI validation error:', url,
out_filename, err)
valid = False
if valid:
digests = self._hasher.hash_file(BytesIO(raw_xml_str))
self._zipfile.writestr(xml_filename, raw_xml_str)
print(out_filename,
url,
*digests,
sep='\t',
file=self._hashsums_fh)
else:
with open(os_path_join(self._bad_urls_dir, out_filename),
'wb') as fh:
fh.write(raw_xml_str)
return xml_filename
def process_single_scan_battery_keras(model_folder,
source_scan_battery_dirname):
# Make sure model_folder and source_scan_battery_dirname exist.
if not os_path_isdir(model_folder):
raise OSError('{}: model folder {} does not exist'.format(
SCRIPT_FNAME, model_folder))
if not os_path_isdir(source_scan_battery_dirname):
raise OSError(
'{}: source scan battery folder {} does not exist'.format(
SCRIPT_FNAME, source_scan_battery_dirname))
# model/scan_batteries folders.
model_scan_batteries_dirname = os_path_join(model_folder,
SCAN_BATTERIES_DIRNAME)
model_scan_battery_dirname = os_path_join(
model_scan_batteries_dirname,
os_path_basename(source_scan_battery_dirname))
# Copy source scan_batteries folder into model scan_batteries folder
# TODO: Could also just copy the entire scan_batteries folder (all 3 types) into model_folder
# logging_info('{}: copying {} to {}'.format(SCRIPT_FNAME, source_scan_battery_dirname, model_scan_battery_dirname))
# copy_anything(source_scan_battery_dirname, model_scan_battery_dirname)
# model_scan_battery_process_scripts_dirname = os_path_abspath(os_path_join(model_scan_battery_dirname, PROCESS_SCRIPTS_DIRNAME))
# Grab all targets with glob
mode_scan_battery_target_prefix = os_path_join(model_scan_battery_dirname,
TARGET_PREFIX + '*')
target_dirnames = glob_glob(mode_scan_battery_target_prefix)
if not target_dirnames:
raise ValueError('{}: no targets found with prefix {}'.format(
SCRIPT_FNAME, mode_scan_battery_target_prefix))
for target_dirname in target_dirnames:
process_single_target(target_dirname)
# for target_dirname in target_dirnames:
# # print('{}: processing target directory {}'.format(SCRIPT_FNAME, target_dirname))
# process_single_target(target_dirname)
# Remove scan battery-level folders
for folder in SCAN_BATTERY_FOLDERS_TO_REMOVE:
folder_path = os_path_join(model_scan_battery_dirname, folder)
if os_path_isdir(folder_path):
# print('{}: Trying to remove {}'.format(SCRIPT_FNAME, folder_path))
try:
shutil.rmtree(folder_path)
except:
raise OSError(
'Error: unable to remove file {}'.format(folder_path))
# Remove scan battery-level files
for file in SCAN_BATTERY_FILES_TO_REMOVE:
file_path = os_path_join(model_scan_battery_dirname, file)
if os_path_isfile(file_path):
# print('{}: Trying to remove {}'.format(SCRIPT_FNAME, file_path))
try:
os_remove(file_path)
except:
raise OSError(
'Error: unable to remove file {}'.format(file_path))
def guess_filename(obj):
"""Tries to guess the filename of the given object."""
name = getattr(obj, 'name', None)
if (name and isinstance(name, basestring) and name[0] != '<' and
name[-1] != '>'):
return os_path_basename(name)
def address_from_address_family(address_family, single_address):
"""Given an AddressFamily and a SingleAddress, return an Addresses object containing the Address.
Raises an exception if the SingleAddress does not match an existing Address.
"""
name = single_address.name
if name is None:
name = os_path_basename(single_address.directory)
if name not in address_family.objects_by_name:
_raise_did_you_mean(address_family, single_address.name)
return Addresses(tuple([Address(address_family.namespace, name)]))
def address_from_address_family(address_family, single_address):
"""Given an AddressFamily and a SingleAddress, return an Addresses object containing the Address.
Raises an exception if the SingleAddress does not match an existing Address.
"""
name = single_address.name
if name is None:
name = os_path_basename(single_address.directory)
if name not in address_family.objects_by_name:
_raise_did_you_mean(address_family, single_address.name)
return Addresses(tuple([Address(address_family.namespace, name)]))
def parse_with_depends(paths):
result = {}
for path in paths:
name = os_path_basename(path)
result[name] = {'path': path, 'depends': []}
manifest_filename = is_module(path)
manifest = eval(open(manifest_filename).read())
result[name].update({'manifest': manifest})
depends = manifest.get('depends', [])
result[name]['depends'] = depends
return result
def goBlue(self):
filename = self.SOURCELIST.getFilename()
if not filename:
return
sourceDir = self.SOURCELIST.getCurrentDirectory()
if os_path_isdir(filename):
text = _("Rename directory")
filename = os_path_basename(os_path_dirname(filename))
else:
text = _("Rename file")
self.session.openWithCallback(self.doRename, VirtualKeyBoard, title = text, text = filename)
def evaluate_one_model_keras(model_dirpath):
# rename _trained as _evaluating
new_folder_name = model_dirpath.replace('_trained', '_evaluating')
shutil_move(model_dirpath, new_folder_name)
model_name = os_path_basename(new_folder_name)
copied_scan_battery_dirname = os_path_join(
new_folder_name, os_path_basename(SCAN_BATTERIES_DIRNAME))
copy_anything(SCAN_BATTERIES_DIRNAME, copied_scan_battery_dirname)
time_start = time_time()
# with Pool() as pool:
# list(pool.imap_unordered(process_single_target, target_dirnames))
for scan_battery_dirname in glob_glob(
os_path_join(SCAN_BATTERIES_DIRNAME, '*')):
process_single_scan_battery_keras(new_folder_name,
scan_battery_dirname)
print('{}: it took {:.2f} to evaluate model {} for all scan batteries'.
format(SCRIPT_FNAME,
time_time() - time_start, model_name))
shutil_move(new_folder_name,
new_folder_name.replace('_evaluating', '_evaluated'))
def update_db(self):
DB_update_folder = DB_DIR+"/Updates";
if(not os_path_exists(DB_update_folder)): return;
# Get all files that match versioning
file_versions = [];
for file in os_listdir(DB_update_folder):
filepath = os_path_join(DB_update_folder, file);
# Try to get the file's name excluding extension (valid filename example: v0.0.0.sql)
version_string = Version.version_string(os_path_splitext(os_path_basename(filepath))[0]);
# Include only files with proper version names within update range
if(os_path_isfile(filepath) and version_string and self.version_is_usable(Version(version_string))):
file_versions.append({"path": filepath, "version": Version(version_string)});
file_versions.sort(key=lambda file_version : file_version["version"]);
for file in file_versions:
if(self.call_shell_command(["sudo", "mysql", "-u", "root", "<", file["path"]])):
raise Exception(f"Failed to update DB with file {file['path']}");
def filter_buildfile_paths(address_mapper, files):
build_files = tuple(f for f in files.dependencies
if address_mapper.build_pattern.match(os_path_basename(f.path)))
return BuildFiles(build_files)
def filter_buildfile_paths(address_mapper, directory_listing):
build_files = tuple(
f for f in directory_listing.files
if address_mapper.build_pattern.match(os_path_basename(f.path)))
return BuildFilePaths(build_files)
def read_video(self, video_path: str):
self.video_capture = VideoCapture(video_path)
self.video_name = os_path_basename(video_path)
return self.video_capture
def filter_buildfile_paths(address_mapper, directory_listing):
build_files = tuple(f for f in directory_listing.files
if address_mapper.build_pattern.match(os_path_basename(f.path)))
return BuildFilePaths(build_files)
def document(infolder, outfolder, extension, loader, external_css=None,
generate_toc=None, overwrite=False):
# Get previously generated TOC object
TOC = os_path_join(infolder, '.cdoc_toc')
try:
with open(TOC, 'rb') as file:
old_toc = pickle_load(file)
except (FileNotFoundError, EOFError):
old_toc = table_Dict2D(OrderedDict)
# Create new TOC object
new_toc = table_Dict2D(OrderedDict)
# TODO: do we really need a separate OrderedDict for pages ???
pages = OrderedDict()
anonym = iter_count()
# TODO: Create real dependency graphs
# Document object:
# parents = set() # other documents depending on this document
# children = set() # other documents this document depending on
#
# If document changed:
# set all parents of document => changed
#
# If any of its children changed:
# set all parents of child => changed
#
# -- The loop should check if a document's change flag has already
# been set. If not, hash file, and set flag, and notify all
# dependencies (parents)
# Load all pages
with check_Checker(infolder, file='.cdoc_cache', lazy_update=True) as checker:
# Go through all files
for file in os_listdir(infolder):
# If file has the proper extension
if file.endswith(extension):
# Create full file path
filepath = os_path_join(infolder, file)
# If file has been changed since last check
if checker.ischanged(filepath) and not overwrite:
# Regenerate file
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
# If file hasn't been changed
else:
# If file has been cached before
try:
# Get previous infos
filename, depends = old_toc[filepath]
pagename = old_toc.otherkey(filepath)
pages[pagename] = None
# If any of the dependencies has changed
for dependency in depends:
if checker.ischanged(dependency) and not overwrite:
# Regenerate file
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
break
# If file is new and hasn't been cached before
except KeyError:
# Generate it for the first time
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
# Store new values
new_toc[pagename:filepath] = filename, depends
# If order changing, renaming, inserting, deleting, etc. happened
if set(old_toc) - set(new_toc):
for pagename, filepath in new_toc.keys():
if pages[pagename] is None:
_process(os_path_basename(filepath), filepath, pages, loader, anonym)
# Write back TOC object
with open(TOC, 'wb') as file:
pickle_dump(new_toc, file, pickle_HIGHEST_PROTOCOL)
# Generate Table of Content?
if generate_toc is None:
generate_toc = len(new_toc) > 1
# Create documents
_build(pages, outfolder, generate_toc, new_toc, external_css)
def filter_buildfile_paths(address_mapper, files):
build_files = tuple(
f for f in files.dependencies
if address_mapper.build_pattern.match(os_path_basename(f.path)))
return BuildFiles(build_files)
from time import time as time_time
from logging import basicConfig as logging_basicConfig, \
DEBUG as logging_DEBUG, \
INFO as logging_INFO, \
getLogger as logging_getLogger
from lib.process_single_scan_battery_keras import process_single_scan_battery_keras
from lib.utils import copy_anything
# SCAN_BATTERIES_TARGETS_GLOB_STRING = 'data/BEAM_Reverb_20181004_L74_70mm/target_*_SCR_*_0dB'
# SCAN_BATTERIES_DIRNAME = 'data/BEAM_Reverb_20181004_L74_70mm_selected'
SCAN_BATTERIES_DIRNAME = 'scan_batteries'
MODEL_SAVE_FNAME = 'model.joblib'
MODELS_DIRNAME = 'DNNs'
SCRIPT_FNAME = os_path_basename(__file__)
PROJECT_DIRNAME = os_path_dirname(__file__)
LIB_DIRNAME = os_path_join(PROJECT_DIRNAME, 'lib')
def evaluate_one_model_keras(model_dirpath):
# rename _trained as _evaluating
new_folder_name = model_dirpath.replace('_trained', '_evaluating')
shutil_move(model_dirpath, new_folder_name)
model_name = os_path_basename(new_folder_name)
copied_scan_battery_dirname = os_path_join(
new_folder_name, os_path_basename(SCAN_BATTERIES_DIRNAME))
copy_anything(SCAN_BATTERIES_DIRNAME, copied_scan_battery_dirname)
time_start = time_time()
isfile as os_path_isfile
from multiprocessing import Pool
from glob import glob as glob_glob
import shutil
from logging import info as logging_info
from scipy.io import loadmat
from lib.r2_dnn_stft import r2_dnn_stft
from lib.r3_dnn_apply import r3_dnn_apply
from lib.r4_dnn_istft import r4_dnn_istft
from lib.r5_dnn_image import r5_dnn_image
from lib.r6_dnn_image_display import r6_dnn_image_display
from lib.utils import get_mat_obj_from_h5py
SCRIPT_FNAME = os_path_basename(
__file__) # for error messages. File name can change.
OLD_STFT_FNAME = 'old_stft.mat'
CHANDAT_FNAME = 'chandat.mat'
SCAN_BATTERIES_DIRNAME = 'scan_batteries'
TARGET_PREFIX = 'target_'
PROCESS_SCRIPTS_DIRNAME = 'process_scripts'
TARGET_FILES_TO_REMOVE = [
'old_stft.mat', 'new_stft.mat', 'chandat.mat', 'chandat_dnn.mat',
'dnn_image.mat', 'box_left_max.txt', 'box_left_min.txt',
'box_right_max.txt', 'box_right_min.txt', 'circle_out_radius.txt',
'circle_out_xc.txt', 'circle_out_zc.txt', 'region_in.txt',
'circle_radius.txt', 'circle_xc.txt', 'circle_zc.txt', 'chandat_clean.mat',
'das_image.mat', 'masks.mat', 'masks.npz', 'startOffsets.mat'
]
SCAN_BATTERY_FOLDERS_TO_REMOVE = [
def document(infolder,
outfolder,
extension,
loader,
external_css=None,
generate_toc=None,
overwrite=False):
# Get previously generated TOC object
TOC = os_path_join(infolder, '.cdoc_toc')
try:
with open(TOC, 'rb') as file:
old_toc = pickle_load(file)
except (FileNotFoundError, EOFError):
old_toc = table_Dict2D(OrderedDict)
# Create new TOC object
new_toc = table_Dict2D(OrderedDict)
# TODO: do we really need a separate OrderedDict for pages ???
pages = OrderedDict()
anonym = iter_count()
# TODO: Create real dependency graphs
# Document object:
# parents = set() # other documents depending on this document
# children = set() # other documents this document depending on
#
# If document changed:
# set all parents of document => changed
#
# If any of its children changed:
# set all parents of child => changed
#
# -- The loop should check if a document's change flag has already
# been set. If not, hash file, and set flag, and notify all
# dependencies (parents)
# Load all pages
with check_Checker(infolder, file='.cdoc_cache',
lazy_update=True) as checker:
# Go through all files
for file in os_listdir(infolder):
# If file has the proper extension
if file.endswith(extension):
# Create full file path
filepath = os_path_join(infolder, file)
# If file has been changed since last check
if checker.ischanged(filepath) and not overwrite:
# Regenerate file
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
# If file hasn't been changed
else:
# If file has been cached before
try:
# Get previous infos
filename, depends = old_toc[filepath]
pagename = old_toc.otherkey(filepath)
pages[pagename] = None
# If any of the dependencies has changed
for dependency in depends:
if checker.ischanged(dependency) and not overwrite:
# Regenerate file
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
break
# If file is new and hasn't been cached before
except KeyError:
# Generate it for the first time
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
# Store new values
new_toc[pagename:filepath] = filename, depends
# If order changing, renaming, inserting, deleting, etc. happened
if set(old_toc) - set(new_toc):
for pagename, filepath in new_toc.keys():
if pages[pagename] is None:
_process(os_path_basename(filepath), filepath, pages, loader,
anonym)
# Write back TOC object
with open(TOC, 'wb') as file:
pickle_dump(new_toc, file, pickle_HIGHEST_PROTOCOL)
# Generate Table of Content?
if generate_toc is None:
generate_toc = len(new_toc) > 1
# Create documents
_build(pages, outfolder, generate_toc, new_toc, external_css)
