def create_subdirectories(self):
"""Iterate items in required input files and check
if there are any directories to create. Create found
directories directly to ToolInstance base directory.
Returns:
Boolean variable depending on success
"""
for i in range(self.input_file_model.rowCount()):
req_file_path = self.input_file_model.item(i,
0).data(Qt.DisplayRole)
# Check if this a directory or a file
path, filename = os.path.split(req_file_path)
if not filename:
# It's a directory
# logging.debug("path {0} should be created to work folder".format(path))
path_to_create = os.path.join(self.instance.basedir, path)
try:
create_dir(path_to_create)
except OSError:
self._toolbox.msg_error.emit(
"[OSError] Creating directory {0} failed."
" Check permissions.".format(path_to_create))
return False
self._toolbox.msg.emit(
"\tDirectory <b>{0}{1}</b> created".format(
os.path.sep, path))
else:
# It's a file -> skip
pass
return True
def extract_app_files(url, timestamp):
"""
Extract the app zip file
into an install directory (specified in config)
"""
install_path = path.join(install_parent, timestamp)
# Unless install dir already exists, extract it
if not (path.exists(install_path) and listdir(install_path)):
tempfile_path = '/tmp/wsgi-app-package.tgz'
create_dir(install_path)
log(
"Extracting '{url}' to '{dir}'".format(
url=url, dir=install_path
)
)
run(sh.rm, tempfile_path, f=True)
urlretrieve(url, tempfile_path)
# Extract files into install dir
run(
sh.tar,
file=tempfile_path,
directory=install_path,
strip="1", z=True, x=True
)
return install_path
def __init__(self, toolbox, name, description, references, x, y):
"""Class constructor."""
super().__init__(name, description)
self._toolbox = toolbox
self._project = self._toolbox.project()
self.item_type = "Data Connection"
# self._widget = DataConnectionWidget(self, self.item_type)
self.reference_model = QStandardItemModel() # References to files
self.data_model = QStandardItemModel(
) # Paths of project internal files. These are found in DC data directory
self.datapackage_icon = QIcon(QPixmap(":/icons/datapkg.png"))
self.data_dir_watcher = QFileSystemWatcher(self)
# Make project directory for this Data Connection
self.data_dir = os.path.join(self._project.project_dir,
self.short_name)
try:
create_dir(self.data_dir)
self.data_dir_watcher.addPath(self.data_dir)
except OSError:
self._toolbox.msg_error.emit(
"[OSError] Creating directory {0} failed."
" Check permissions.".format(self.data_dir))
# Populate references model
self.references = references
self.populate_reference_list(self.references)
# Populate data (files) model
data_files = self.data_files()
self.populate_data_list(data_files)
self._graphics_item = DataConnectionImage(self._toolbox, x - 35,
y - 35, 70, 70, self.name)
self.spine_datapackage_form = None
# self.ui.toolButton_datapackage.setMenu(self.datapackage_popup_menu) # TODO: OBSOLETE?
self._sigs = self.make_signal_handler_dict()
def __init__(self, toolbox, name, description, reference, x, y):
"""Class constructor."""
super().__init__(name, description)
self._toolbox = toolbox
self._project = self._toolbox.project()
self.item_type = "Data Store"
# Instance variables for saving selections in shared widgets
self.selected_dialect = ""
self.selected_dsn = ""
self.selected_sqlite_file = ""
self.selected_host = ""
self.selected_port = ""
self.selected_db = ""
self.selected_username = ""
self.selected_password = ""
self.tree_view_form = None
self.graph_view_form = None
self.tabular_view_form = None
# Make project directory for this Data Store
self.data_dir = os.path.join(self._project.project_dir,
self.short_name)
try:
create_dir(self.data_dir)
except OSError:
self._toolbox.msg_error.emit(
"[OSError] Creating directory {0} failed."
" Check permissions.".format(self.data_dir))
self._graphics_item = DataStoreImage(self._toolbox, x - 35, y - 35, 70,
70, self.name)
self._reference = reference
self.load_reference_into_selections()
self._sigs = self.make_signal_handler_dict()
def save_output_files(self, ret):
"""Copy output files from work directory to Tool output directory."""
output_dir_timestamp = create_output_dir_timestamp(
) # Get timestamp when tool finished
# Create an output folder with timestamp and copy output directly there
if ret != 0:
result_path = os.path.abspath(
os.path.join(self.tool_output_dir, 'failed',
output_dir_timestamp))
else:
result_path = os.path.abspath(
os.path.join(self.tool_output_dir, output_dir_timestamp))
try:
create_dir(result_path)
except OSError:
self._toolbox.msg_error.emit(
"\tError creating timestamped output directory. "
"Tool template output files not copied. Please check directory permissions."
)
self.output_dir = None
self.instance_finished_signal.emit(ret)
return
self.output_dir = result_path
# Make link to output folder
result_anchor = "<a style='color:#BB99FF;' title='" + result_path + "' href='file:///" + result_path \
+ "'>results directory</a>"
self._toolbox.msg.emit(
"*** Archiving output files to {0} ***".format(result_anchor))
if not self.outputfiles:
tip_anchor = "<a style='color:#99CCFF;' title='When you add output files to the Tool template,\n " \
"they will be archived into results directory. Also, output files are passed to\n " \
"subsequent project items.' href='#'>Tip</a>"
self._toolbox.msg_warning.emit(
"\tNo output files defined for this Tool template. {0}".format(
tip_anchor))
else:
saved_files, failed_files = self.copy_output(result_path)
if len(saved_files) == 0:
# If no files were saved
self._toolbox.msg_error.emit("\tNo files saved")
if len(saved_files) > 0:
# If there are saved files
self._toolbox.msg.emit(
"\tThe following output files were saved to results directory"
)
for i in range(len(saved_files)):
# fname = os.path.split(saved_files[i])[1]
self._toolbox.msg.emit("\t\t<b>{0}</b>".format(
saved_files[i]))
if len(failed_files) > 0:
# If saving some or all files failed
self._toolbox.msg_warning.emit(
"\tThe following output files were not found")
for i in range(len(failed_files)):
failed_fname = os.path.split(failed_files[i])[1]
self._toolbox.msg_warning.emit(
"\t\t<b>{0}</b>".format(failed_fname))
self.instance_finished_signal.emit(ret)
def copy_output(self, target_dir):
"""Save output of a tool instance
Args:
target_dir (str): Copy destination
Returns:
ret (bool): Operation success
"""
failed_files = list()
saved_files = list()
# logging.debug("Saving result files to <{0}>".format(target_dir))
for pattern in self.tool_template.outputfiles:
# Create subdirectories if necessary
dst_subdir, fname_pattern = os.path.split(pattern)
# logging.debug("pattern:{0} dst_subdir:{1} fname_pattern:{2}".format(pattern,
# dst_subdir, fname_pattern))
if not dst_subdir:
# No subdirectories to create
# self._toolbox.msg.emit("\tCopying file <b>{0}</b>".format(fname))
target = target_dir
else:
# Create subdirectory structure to result directory
result_subdir_path = os.path.abspath(
os.path.join(target_dir, dst_subdir))
if not os.path.exists(result_subdir_path):
try:
create_dir(result_subdir_path)
except OSError:
self._toolbox.msg_error.emit(
"[OSError] Creating directory <b>{0}</b> failed.".
format(result_subdir_path))
continue
self._toolbox.msg.emit(
"\tCreated result subdirectory <b>{0}{1}</b>".format(
os.path.sep, dst_subdir))
target = result_subdir_path
# Check for wildcards in pattern
if ('*' in pattern) or ('?' in pattern):
for fname_path in glob.glob(os.path.join(
self.basedir, pattern)): # fname_path is a full path
fname = os.path.split(fname_path)[1] # File name (no path)
dst = os.path.join(target, fname)
shutil.copy(fname_path, dst)
saved_files.append(os.path.join(dst_subdir, fname))
else:
output_file = os.path.join(self.basedir, pattern)
# logging.debug("Looking for {0}".format(output_file))
if not os.path.isfile(output_file):
failed_files.append(pattern)
continue
# logging.debug("Saving file {0}".format(fname_pattern))
dst = os.path.join(target, fname_pattern)
# logging.debug("Copying to {0}".format(dst))
shutil.copy(output_file, dst)
saved_files.append(pattern)
return saved_files, failed_files
def tokenize(arg1, model_path=None):
# arg1: path to file that will be tokenized/tagged
# input file is expected to end with the respective language for example: abc.xy.en
input_path = os.path.abspath(arg1)
input_filename = ntpath.basename(input_path)
input_dir = os.path.dirname(input_path)
output_dir = os.path.join(input_dir, 'conll')
output_file = os.path.join(output_dir, f'{input_filename}.conll')
create_dir(output_dir)
lang = input_filename.split('.')[-1]
model_path = default_by_lang(lang)
"""if not model_path:
model_path = f"{MODELS}/ud_custom_models"
if not os.path.isdir(model_path):
print('Create directory:', model_path)
os.mkdir(model_path)
print('Output directory:', output_dir)
sys.path.append(SCRIPTS)
from default_code_mappings import default_mappings as d
#d = {'de':'de_gsd', 'en':'en_ewt', 'cs':'cs_pdt', 'fr':'fr_ftb', 'sv':'sv_talbanken'}
#cs_pdt.model de_gsd.model en_ewt.model fr_ftb.model sv_talbanken.model
model_path = f'{model_path}/{d[lang]}.model'"""
log_path = f"{LOGS}/UDPipe"
create_dir(log_path)
batch_string = f"""#!/bin/sh
#SBATCH -t 48:00:00
#SBATCH -n 1
#SBATCH -J "{lang}_tok"
#SBATCH --mem-per-cpu=16GB
#SBATCH --account=nn9447k
#SBATCH --output={log_path}/tokenize_{lang}-%j.out
source ~/.bashrc
module purge
module load gcc
srun -t 48:00:00 \
--mem-per-cpu=16GB \
--account=nn9447k \
/projects/nlpl/software/udpipe/latest/bin/udpipe --tokenize --tag {model_path} {input_path} > {output_file}
"""
batch_path = f'{BATCHFILES}/tokenize.sh'
with open(batch_path, 'w') as f:
f.write(batch_string)
os.system(f'sbatch {batch_path}')
def __init__(self, toolbox, name, description, tool_template, use_work, x,
y):
"""Class constructor."""
super().__init__(name, description)
self._toolbox = toolbox
self._project = self._toolbox.project()
self.item_type = "Tool"
self.source_file_model = QStandardItemModel()
self.populate_source_file_model(None)
self.input_file_model = QStandardItemModel()
self.populate_input_file_model(None)
self.opt_input_file_model = QStandardItemModel()
self.populate_opt_input_file_model(None)
self.output_file_model = QStandardItemModel()
self.populate_output_file_model(None)
self.template_model = QStandardItemModel()
self.populate_template_model(False)
self.source_files = list()
self._tool_template = None
self.set_tool_template(tool_template)
if not self._tool_template:
self._tool_template_name = ""
else:
self._tool_template_name = self.tool_template().name
self.tool_template_options_popup_menu = None
self.instance = None # Instance of this Tool that can be sent to a subprocess for processing
self.extra_cmdline_args = '' # This may be used for additional Tool specific command line arguments
self.execute_in_work = use_work # Enables overriding the template default setting
# Make project directory for this Tool
self.data_dir = os.path.join(self._project.project_dir,
self.short_name)
try:
create_dir(self.data_dir)
except OSError:
self._toolbox.msg_error.emit(
"[OSError] Creating directory {0} failed."
" Check permissions.".format(self.data_dir))
# Make directory for results
self.output_dir = os.path.join(self.data_dir, TOOL_OUTPUT_DIR)
self._graphics_item = ToolImage(self._toolbox,
x - 35,
y - 35,
w=70,
h=70,
name=self.name)
self._sigs = self.make_signal_handler_dict()
def copy_ssl_certificates(timestamp):
"""
Copy either the default self-signed certificate
or the provided custom ones
into /etc/ssl/certs/wsgi-app.*
Return the locations of the created files
"""
certs_dir = '/etc/ssl/certs'
keyfile_path = path.join(
certs_dir,
'wsgi-app.{0}.key'.format(timestamp)
)
certificate_path = path.join(
certs_dir,
'wsgi-app.{0}.crt'.format(timestamp)
)
custom_keyfile = config('ssl_keyfile')
custom_certificate = config('ssl_certificate')
create_dir(certs_dir)
log('Saving certificate files')
if custom_keyfile and custom_certificate:
keyfile_content = b64decode(custom_keyfile)
certificate_content = b64decode(custom_certificate)
with open(keyfile_path, 'w') as keyfile:
keyfile.write(keyfile_content)
with open(certificate_path, 'w') as certificate:
certificate.write(certificate_content)
else:
config_path = path.join(charm_dir, 'ssl/wsgi-app.conf')
run(
sh.openssl.req,
"-new", "-nodes", "-x509", "-newkey", "rsa:2048", "-days",
"365", "-keyout", keyfile_path, "-out", certificate_path,
"-config", config_path
)
return (keyfile_path, certificate_path)
def _checkout(self):
"""Copy Tool files to work directory."""
n_copied_files = 0
# Make work directory anchor with path as tooltip
work_anchor = "<a style='color:#99CCFF;' title='" + self.basedir + "' href='file:///" + self.basedir \
+ "'>work directory</a>"
self._toolbox.msg.emit(
"*** Copying Tool template <b>{0}</b> source files to {1} ***".
format(self.tool_template.name, work_anchor))
for filepath in self.tool_template.includes:
dirname, file_pattern = os.path.split(filepath)
src_dir = os.path.join(self.tool_template.path, dirname)
dst_dir = os.path.join(self.basedir, dirname)
# Create the destination directory
try:
create_dir(dst_dir)
except OSError:
self._toolbox.msg_error.emit(
"Creating directory <b>{0}</b> failed".format(dst_dir))
return False
# Copy file if necessary
if file_pattern:
for src_file in glob.glob(os.path.join(src_dir, file_pattern)):
dst_file = os.path.join(dst_dir,
os.path.basename(src_file))
# logging.debug("Copying file {} to {}".format(src_file, dst_file))
try:
shutil.copyfile(src_file, dst_file)
n_copied_files += 1
except OSError as e:
logging.error(e)
self._toolbox.msg_error.emit(
"\tCopying file <b>{0}</b> to <b>{1}</b> failed".
format(src_file, dst_file))
return False
if n_copied_files == 0:
self._toolbox.msg_warning.emit("Warning: No files copied")
else:
self._toolbox.msg.emit(
"\tCopied <b>{0}</b> file(s)".format(n_copied_files))
return True
def __init__(self, toolbox, name, description, x, y):
"""Class constructor."""
super().__init__(name, description)
self._toolbox = toolbox
self._project = self._toolbox.project()
self.item_type = "View"
self.graph_view_form_refs = {}
self._references = list()
self.reference_model = QStandardItemModel() # References to databases
self.spine_ref_icon = QIcon(QPixmap(":/icons/Spine_db_ref_icon.png"))
# Make project directory for this View
self.data_dir = os.path.join(self._project.project_dir, self.short_name)
try:
create_dir(self.data_dir)
except OSError:
self._toolbox.msg_error.emit("[OSError] Creating directory {0} failed."
" Check permissions.".format(self.data_dir))
self._graphics_item = ViewImage(self._toolbox, x - 35, y - 35, 70, 70, self.name)
# Note: view_refresh_signal is not shared with other project items so there is no need to disconnect it
self.view_refresh_signal.connect(self.refresh)
self._sigs = self.make_signal_handler_dict()
def make_work_output_dirs(self):
"""Make sure that work directory has the necessary output directories for Tool output files.
Checks only "outputfiles" list. Alternatively you can add directories to "inputfiles" list
in the tool definition file.
Returns:
Boolean value depending on operation success.
"""
# TODO: Remove duplicate directory names from the list of created directories.
for path in self.tool_template.outputfiles:
dirname, file_pattern = os.path.split(path)
if dirname == '':
continue
dst_dir = os.path.join(self.basedir, dirname)
try:
create_dir(dst_dir)
except OSError:
self._toolbox.msg_error.emit(
"Creating work output directory '{}' failed".format(
dst_dir))
return False
return True
def __init__(self, toolbox, file_path, defaults=None):
"""Initialize configuration parser.
Args:
toolbox (ToolboxUI): QMainWindow instance
file_path (str): Absolute path to the configuration file.
defaults (dict): A dictionary containing configuration default options.
"""
self._toolbox = toolbox
self.parser = configparser.ConfigParser()
self.file_path = file_path
# Create conf directory if it does not exist
d = os.path.split(self.file_path)[0]
try:
create_dir(d)
except OSError:
self._toolbox.msg_error.emit(
"[OSError] Creating directory {0} failed. Check permissions.".
format(d))
if defaults:
self.parser['settings'] = defaults
else:
self.parser['settings'] = SETTINGS
error_G = - tf.reduce_mean(fake_hat)
error_D = tf.reduce_mean(real_hat) - tf.reduce_mean(fake_hat)
# Specify that we will use RMSProp (one optimiser for each model)
optimiser_G = tf.train.RMSPropOptimizer(lr).minimize(error_G, var_list=weights_G.values())
optimiser_D = tf.train.RMSPropOptimizer(lr).minimize(-error_D, var_list=weights_D.values())
# Generate Op that initialises global variables in the graph
init = tf.global_variables_initializer()
with tf.Session() as sess:
# Initialise variables and start the session
sess.run(init)
if path_to_images:
helpers.create_dir(path_to_images)
# Run a set number of epochs (default `n_critic` from the WGAN paper)
n_critic = 5
errors = [[], []]
for epoch in range(nb_epochs):
for critic in range(n_critic):
# Retrieve a batch from MNIST
X_batch, critic = mnist.train.next_batch(batch_size)
# Clip weights and run one step of the optimiser for D
sess.run(clipped_D)
sess.run(optimiser_D, feed_dict={Z: z_sampler(batch_size, z_size), X: X_batch})
# Run one step of the optimiser for G
sess.run(optimiser_G, feed_dict={Z: z_sampler(batch_size, z_size)})
import os
from helpers import create_dir
create_dir('data')
create_dir('data/raw')
create_dir('data/working')
create_dir('docs')
create_dir('output')
create_dir('output/tables')
create_dir('output/plots')
create_dir('output/submissions')
def copy_optional_input_files(self, paths):
"""Copy optional input files from given paths to work or source directory, depending on
where the Tool template requires them to be.
Args:
paths (dict): Key is the optional file name pattern, value is a list of paths to source files.
Returns:
Boolean variable depending on operation success
"""
n_copied_files = 0
for dst, src_paths in paths.items():
if not isinstance(src_paths, list):
self._toolbox.msg_error.emit(
"Copying optional input files failed. src_paths should be a list."
)
return False
for src_path in src_paths:
if not os.path.exists(src_path):
self._toolbox.msg_error.emit(
"\tFile <b>{0}</b> does not exist".format(src_path))
continue
# Get file name that matched the search pattern
src_dir, dst_fname = os.path.split(src_path)
# Check if the search pattern included subdirectories (e.g. 'input/*.csv')
# This means that /input/ directory should be created to work (or source) directory
# before copying the files
dst_subdir, _search_pattern = os.path.split(dst)
if not dst_subdir:
# No subdirectories to create
self._toolbox.msg.emit(
"\tCopying optional file <b>{0}</b>".format(dst_fname))
dst_path = os.path.abspath(
os.path.join(self.instance.basedir, dst_fname))
else:
# Create subdirectory structure to work or source directory
work_subdir_path = os.path.abspath(
os.path.join(self.instance.basedir, dst_subdir))
if not os.path.exists(work_subdir_path):
try:
create_dir(work_subdir_path)
except OSError:
self._toolbox.msg_error.emit(
"[OSError] Creating directory <b>{0}</b> failed."
.format(work_subdir_path))
continue
self._toolbox.msg.emit(
"\tCopying optional file <b>{0}</b> into subdirectory <b>{2}{1}</b>"
.format(dst_fname, dst_subdir, os.path.sep))
dst_path = os.path.abspath(
os.path.join(work_subdir_path, dst_fname))
try:
shutil.copyfile(src_path, dst_path)
n_copied_files += 1
except OSError as e:
self._toolbox.msg_error.emit(
"Copying optional file <b>{0}</b> to <b>{1}</b> failed"
.format(src_path, dst_path))
self._toolbox.msg_error.emit("{0}".format(e))
if e.errno == 22:
msg = "The reason might be:\n" \
"[1] The destination file already exists and it cannot be " \
"overwritten because it is locked by Julia or some other application.\n" \
"[2] You don't have the necessary permissions to overwrite the file.\n" \
"To solve the problem, you can try the following:\n[1] Execute the Tool in work " \
"directory.\n[2] If you are executing a Julia Tool with Julia 0.6.x, upgrade to " \
"Julia 0.7 or newer.\n" \
"[3] Close any other background application(s) that may have locked the file.\n" \
"And try again.\n"
self._toolbox.msg_warning.emit(msg)
self._toolbox.msg.emit(
"\tCopied <b>{0}</b> optional input file(s)".format(
n_copied_files))
return True
