def create_subdirectories(self): """Iterate items in required input files and check if there are any directories to create. Create found directories directly to ToolInstance base directory. Returns: Boolean variable depending on success """ for i in range(self.input_file_model.rowCount()): req_file_path = self.input_file_model.item(i, 0).data(Qt.DisplayRole) # Check if this a directory or a file path, filename = os.path.split(req_file_path) if not filename: # It's a directory # logging.debug("path {0} should be created to work folder".format(path)) path_to_create = os.path.join(self.instance.basedir, path) try: create_dir(path_to_create) except OSError: self._toolbox.msg_error.emit( "[OSError] Creating directory {0} failed." " Check permissions.".format(path_to_create)) return False self._toolbox.msg.emit( "\tDirectory <b>{0}{1}</b> created".format( os.path.sep, path)) else: # It's a file -> skip pass return True
def extract_app_files(url, timestamp): """ Extract the app zip file into an install directory (specified in config) """ install_path = path.join(install_parent, timestamp) # Unless install dir already exists, extract it if not (path.exists(install_path) and listdir(install_path)): tempfile_path = '/tmp/wsgi-app-package.tgz' create_dir(install_path) log( "Extracting '{url}' to '{dir}'".format( url=url, dir=install_path ) ) run(sh.rm, tempfile_path, f=True) urlretrieve(url, tempfile_path) # Extract files into install dir run( sh.tar, file=tempfile_path, directory=install_path, strip="1", z=True, x=True ) return install_path
def __init__(self, toolbox, name, description, references, x, y): """Class constructor.""" super().__init__(name, description) self._toolbox = toolbox self._project = self._toolbox.project() self.item_type = "Data Connection" # self._widget = DataConnectionWidget(self, self.item_type) self.reference_model = QStandardItemModel() # References to files self.data_model = QStandardItemModel( ) # Paths of project internal files. These are found in DC data directory self.datapackage_icon = QIcon(QPixmap(":/icons/datapkg.png")) self.data_dir_watcher = QFileSystemWatcher(self) # Make project directory for this Data Connection self.data_dir = os.path.join(self._project.project_dir, self.short_name) try: create_dir(self.data_dir) self.data_dir_watcher.addPath(self.data_dir) except OSError: self._toolbox.msg_error.emit( "[OSError] Creating directory {0} failed." " Check permissions.".format(self.data_dir)) # Populate references model self.references = references self.populate_reference_list(self.references) # Populate data (files) model data_files = self.data_files() self.populate_data_list(data_files) self._graphics_item = DataConnectionImage(self._toolbox, x - 35, y - 35, 70, 70, self.name) self.spine_datapackage_form = None # self.ui.toolButton_datapackage.setMenu(self.datapackage_popup_menu) # TODO: OBSOLETE? self._sigs = self.make_signal_handler_dict()
def __init__(self, toolbox, name, description, reference, x, y): """Class constructor.""" super().__init__(name, description) self._toolbox = toolbox self._project = self._toolbox.project() self.item_type = "Data Store" # Instance variables for saving selections in shared widgets self.selected_dialect = "" self.selected_dsn = "" self.selected_sqlite_file = "" self.selected_host = "" self.selected_port = "" self.selected_db = "" self.selected_username = "" self.selected_password = "" self.tree_view_form = None self.graph_view_form = None self.tabular_view_form = None # Make project directory for this Data Store self.data_dir = os.path.join(self._project.project_dir, self.short_name) try: create_dir(self.data_dir) except OSError: self._toolbox.msg_error.emit( "[OSError] Creating directory {0} failed." " Check permissions.".format(self.data_dir)) self._graphics_item = DataStoreImage(self._toolbox, x - 35, y - 35, 70, 70, self.name) self._reference = reference self.load_reference_into_selections() self._sigs = self.make_signal_handler_dict()
def save_output_files(self, ret): """Copy output files from work directory to Tool output directory.""" output_dir_timestamp = create_output_dir_timestamp( ) # Get timestamp when tool finished # Create an output folder with timestamp and copy output directly there if ret != 0: result_path = os.path.abspath( os.path.join(self.tool_output_dir, 'failed', output_dir_timestamp)) else: result_path = os.path.abspath( os.path.join(self.tool_output_dir, output_dir_timestamp)) try: create_dir(result_path) except OSError: self._toolbox.msg_error.emit( "\tError creating timestamped output directory. " "Tool template output files not copied. Please check directory permissions." ) self.output_dir = None self.instance_finished_signal.emit(ret) return self.output_dir = result_path # Make link to output folder result_anchor = "<a style='color:#BB99FF;' title='" + result_path + "' href='file:///" + result_path \ + "'>results directory</a>" self._toolbox.msg.emit( "*** Archiving output files to {0} ***".format(result_anchor)) if not self.outputfiles: tip_anchor = "<a style='color:#99CCFF;' title='When you add output files to the Tool template,\n " \ "they will be archived into results directory. Also, output files are passed to\n " \ "subsequent project items.' href='#'>Tip</a>" self._toolbox.msg_warning.emit( "\tNo output files defined for this Tool template. {0}".format( tip_anchor)) else: saved_files, failed_files = self.copy_output(result_path) if len(saved_files) == 0: # If no files were saved self._toolbox.msg_error.emit("\tNo files saved") if len(saved_files) > 0: # If there are saved files self._toolbox.msg.emit( "\tThe following output files were saved to results directory" ) for i in range(len(saved_files)): # fname = os.path.split(saved_files[i])[1] self._toolbox.msg.emit("\t\t<b>{0}</b>".format( saved_files[i])) if len(failed_files) > 0: # If saving some or all files failed self._toolbox.msg_warning.emit( "\tThe following output files were not found") for i in range(len(failed_files)): failed_fname = os.path.split(failed_files[i])[1] self._toolbox.msg_warning.emit( "\t\t<b>{0}</b>".format(failed_fname)) self.instance_finished_signal.emit(ret)
def copy_output(self, target_dir): """Save output of a tool instance Args: target_dir (str): Copy destination Returns: ret (bool): Operation success """ failed_files = list() saved_files = list() # logging.debug("Saving result files to <{0}>".format(target_dir)) for pattern in self.tool_template.outputfiles: # Create subdirectories if necessary dst_subdir, fname_pattern = os.path.split(pattern) # logging.debug("pattern:{0} dst_subdir:{1} fname_pattern:{2}".format(pattern, # dst_subdir, fname_pattern)) if not dst_subdir: # No subdirectories to create # self._toolbox.msg.emit("\tCopying file <b>{0}</b>".format(fname)) target = target_dir else: # Create subdirectory structure to result directory result_subdir_path = os.path.abspath( os.path.join(target_dir, dst_subdir)) if not os.path.exists(result_subdir_path): try: create_dir(result_subdir_path) except OSError: self._toolbox.msg_error.emit( "[OSError] Creating directory <b>{0}</b> failed.". format(result_subdir_path)) continue self._toolbox.msg.emit( "\tCreated result subdirectory <b>{0}{1}</b>".format( os.path.sep, dst_subdir)) target = result_subdir_path # Check for wildcards in pattern if ('*' in pattern) or ('?' in pattern): for fname_path in glob.glob(os.path.join( self.basedir, pattern)): # fname_path is a full path fname = os.path.split(fname_path)[1] # File name (no path) dst = os.path.join(target, fname) shutil.copy(fname_path, dst) saved_files.append(os.path.join(dst_subdir, fname)) else: output_file = os.path.join(self.basedir, pattern) # logging.debug("Looking for {0}".format(output_file)) if not os.path.isfile(output_file): failed_files.append(pattern) continue # logging.debug("Saving file {0}".format(fname_pattern)) dst = os.path.join(target, fname_pattern) # logging.debug("Copying to {0}".format(dst)) shutil.copy(output_file, dst) saved_files.append(pattern) return saved_files, failed_files
def tokenize(arg1, model_path=None): # arg1: path to file that will be tokenized/tagged # input file is expected to end with the respective language for example: abc.xy.en input_path = os.path.abspath(arg1) input_filename = ntpath.basename(input_path) input_dir = os.path.dirname(input_path) output_dir = os.path.join(input_dir, 'conll') output_file = os.path.join(output_dir, f'{input_filename}.conll') create_dir(output_dir) lang = input_filename.split('.')[-1] model_path = default_by_lang(lang) """if not model_path: model_path = f"{MODELS}/ud_custom_models" if not os.path.isdir(model_path): print('Create directory:', model_path) os.mkdir(model_path) print('Output directory:', output_dir) sys.path.append(SCRIPTS) from default_code_mappings import default_mappings as d #d = {'de':'de_gsd', 'en':'en_ewt', 'cs':'cs_pdt', 'fr':'fr_ftb', 'sv':'sv_talbanken'} #cs_pdt.model de_gsd.model en_ewt.model fr_ftb.model sv_talbanken.model model_path = f'{model_path}/{d[lang]}.model'""" log_path = f"{LOGS}/UDPipe" create_dir(log_path) batch_string = f"""#!/bin/sh #SBATCH -t 48:00:00 #SBATCH -n 1 #SBATCH -J "{lang}_tok" #SBATCH --mem-per-cpu=16GB #SBATCH --account=nn9447k #SBATCH --output={log_path}/tokenize_{lang}-%j.out source ~/.bashrc module purge module load gcc srun -t 48:00:00 \ --mem-per-cpu=16GB \ --account=nn9447k \ /projects/nlpl/software/udpipe/latest/bin/udpipe --tokenize --tag {model_path} {input_path} > {output_file} """ batch_path = f'{BATCHFILES}/tokenize.sh' with open(batch_path, 'w') as f: f.write(batch_string) os.system(f'sbatch {batch_path}')
def __init__(self, toolbox, name, description, tool_template, use_work, x, y): """Class constructor.""" super().__init__(name, description) self._toolbox = toolbox self._project = self._toolbox.project() self.item_type = "Tool" self.source_file_model = QStandardItemModel() self.populate_source_file_model(None) self.input_file_model = QStandardItemModel() self.populate_input_file_model(None) self.opt_input_file_model = QStandardItemModel() self.populate_opt_input_file_model(None) self.output_file_model = QStandardItemModel() self.populate_output_file_model(None) self.template_model = QStandardItemModel() self.populate_template_model(False) self.source_files = list() self._tool_template = None self.set_tool_template(tool_template) if not self._tool_template: self._tool_template_name = "" else: self._tool_template_name = self.tool_template().name self.tool_template_options_popup_menu = None self.instance = None # Instance of this Tool that can be sent to a subprocess for processing self.extra_cmdline_args = '' # This may be used for additional Tool specific command line arguments self.execute_in_work = use_work # Enables overriding the template default setting # Make project directory for this Tool self.data_dir = os.path.join(self._project.project_dir, self.short_name) try: create_dir(self.data_dir) except OSError: self._toolbox.msg_error.emit( "[OSError] Creating directory {0} failed." " Check permissions.".format(self.data_dir)) # Make directory for results self.output_dir = os.path.join(self.data_dir, TOOL_OUTPUT_DIR) self._graphics_item = ToolImage(self._toolbox, x - 35, y - 35, w=70, h=70, name=self.name) self._sigs = self.make_signal_handler_dict()
def copy_ssl_certificates(timestamp): """ Copy either the default self-signed certificate or the provided custom ones into /etc/ssl/certs/wsgi-app.* Return the locations of the created files """ certs_dir = '/etc/ssl/certs' keyfile_path = path.join( certs_dir, 'wsgi-app.{0}.key'.format(timestamp) ) certificate_path = path.join( certs_dir, 'wsgi-app.{0}.crt'.format(timestamp) ) custom_keyfile = config('ssl_keyfile') custom_certificate = config('ssl_certificate') create_dir(certs_dir) log('Saving certificate files') if custom_keyfile and custom_certificate: keyfile_content = b64decode(custom_keyfile) certificate_content = b64decode(custom_certificate) with open(keyfile_path, 'w') as keyfile: keyfile.write(keyfile_content) with open(certificate_path, 'w') as certificate: certificate.write(certificate_content) else: config_path = path.join(charm_dir, 'ssl/wsgi-app.conf') run( sh.openssl.req, "-new", "-nodes", "-x509", "-newkey", "rsa:2048", "-days", "365", "-keyout", keyfile_path, "-out", certificate_path, "-config", config_path ) return (keyfile_path, certificate_path)
def _checkout(self): """Copy Tool files to work directory.""" n_copied_files = 0 # Make work directory anchor with path as tooltip work_anchor = "<a style='color:#99CCFF;' title='" + self.basedir + "' href='file:///" + self.basedir \ + "'>work directory</a>" self._toolbox.msg.emit( "*** Copying Tool template <b>{0}</b> source files to {1} ***". format(self.tool_template.name, work_anchor)) for filepath in self.tool_template.includes: dirname, file_pattern = os.path.split(filepath) src_dir = os.path.join(self.tool_template.path, dirname) dst_dir = os.path.join(self.basedir, dirname) # Create the destination directory try: create_dir(dst_dir) except OSError: self._toolbox.msg_error.emit( "Creating directory <b>{0}</b> failed".format(dst_dir)) return False # Copy file if necessary if file_pattern: for src_file in glob.glob(os.path.join(src_dir, file_pattern)): dst_file = os.path.join(dst_dir, os.path.basename(src_file)) # logging.debug("Copying file {} to {}".format(src_file, dst_file)) try: shutil.copyfile(src_file, dst_file) n_copied_files += 1 except OSError as e: logging.error(e) self._toolbox.msg_error.emit( "\tCopying file <b>{0}</b> to <b>{1}</b> failed". format(src_file, dst_file)) return False if n_copied_files == 0: self._toolbox.msg_warning.emit("Warning: No files copied") else: self._toolbox.msg.emit( "\tCopied <b>{0}</b> file(s)".format(n_copied_files)) return True
def __init__(self, toolbox, name, description, x, y): """Class constructor.""" super().__init__(name, description) self._toolbox = toolbox self._project = self._toolbox.project() self.item_type = "View" self.graph_view_form_refs = {} self._references = list() self.reference_model = QStandardItemModel() # References to databases self.spine_ref_icon = QIcon(QPixmap(":/icons/Spine_db_ref_icon.png")) # Make project directory for this View self.data_dir = os.path.join(self._project.project_dir, self.short_name) try: create_dir(self.data_dir) except OSError: self._toolbox.msg_error.emit("[OSError] Creating directory {0} failed." " Check permissions.".format(self.data_dir)) self._graphics_item = ViewImage(self._toolbox, x - 35, y - 35, 70, 70, self.name) # Note: view_refresh_signal is not shared with other project items so there is no need to disconnect it self.view_refresh_signal.connect(self.refresh) self._sigs = self.make_signal_handler_dict()
def make_work_output_dirs(self): """Make sure that work directory has the necessary output directories for Tool output files. Checks only "outputfiles" list. Alternatively you can add directories to "inputfiles" list in the tool definition file. Returns: Boolean value depending on operation success. """ # TODO: Remove duplicate directory names from the list of created directories. for path in self.tool_template.outputfiles: dirname, file_pattern = os.path.split(path) if dirname == '': continue dst_dir = os.path.join(self.basedir, dirname) try: create_dir(dst_dir) except OSError: self._toolbox.msg_error.emit( "Creating work output directory '{}' failed".format( dst_dir)) return False return True
def __init__(self, toolbox, file_path, defaults=None): """Initialize configuration parser. Args: toolbox (ToolboxUI): QMainWindow instance file_path (str): Absolute path to the configuration file. defaults (dict): A dictionary containing configuration default options. """ self._toolbox = toolbox self.parser = configparser.ConfigParser() self.file_path = file_path # Create conf directory if it does not exist d = os.path.split(self.file_path)[0] try: create_dir(d) except OSError: self._toolbox.msg_error.emit( "[OSError] Creating directory {0} failed. Check permissions.". format(d)) if defaults: self.parser['settings'] = defaults else: self.parser['settings'] = SETTINGS
error_G = - tf.reduce_mean(fake_hat) error_D = tf.reduce_mean(real_hat) - tf.reduce_mean(fake_hat) # Specify that we will use RMSProp (one optimiser for each model) optimiser_G = tf.train.RMSPropOptimizer(lr).minimize(error_G, var_list=weights_G.values()) optimiser_D = tf.train.RMSPropOptimizer(lr).minimize(-error_D, var_list=weights_D.values()) # Generate Op that initialises global variables in the graph init = tf.global_variables_initializer() with tf.Session() as sess: # Initialise variables and start the session sess.run(init) if path_to_images: helpers.create_dir(path_to_images) # Run a set number of epochs (default `n_critic` from the WGAN paper) n_critic = 5 errors = [[], []] for epoch in range(nb_epochs): for critic in range(n_critic): # Retrieve a batch from MNIST X_batch, critic = mnist.train.next_batch(batch_size) # Clip weights and run one step of the optimiser for D sess.run(clipped_D) sess.run(optimiser_D, feed_dict={Z: z_sampler(batch_size, z_size), X: X_batch}) # Run one step of the optimiser for G sess.run(optimiser_G, feed_dict={Z: z_sampler(batch_size, z_size)})
import os from helpers import create_dir create_dir('data') create_dir('data/raw') create_dir('data/working') create_dir('docs') create_dir('output') create_dir('output/tables') create_dir('output/plots') create_dir('output/submissions')
def copy_optional_input_files(self, paths): """Copy optional input files from given paths to work or source directory, depending on where the Tool template requires them to be. Args: paths (dict): Key is the optional file name pattern, value is a list of paths to source files. Returns: Boolean variable depending on operation success """ n_copied_files = 0 for dst, src_paths in paths.items(): if not isinstance(src_paths, list): self._toolbox.msg_error.emit( "Copying optional input files failed. src_paths should be a list." ) return False for src_path in src_paths: if not os.path.exists(src_path): self._toolbox.msg_error.emit( "\tFile <b>{0}</b> does not exist".format(src_path)) continue # Get file name that matched the search pattern src_dir, dst_fname = os.path.split(src_path) # Check if the search pattern included subdirectories (e.g. 'input/*.csv') # This means that /input/ directory should be created to work (or source) directory # before copying the files dst_subdir, _search_pattern = os.path.split(dst) if not dst_subdir: # No subdirectories to create self._toolbox.msg.emit( "\tCopying optional file <b>{0}</b>".format(dst_fname)) dst_path = os.path.abspath( os.path.join(self.instance.basedir, dst_fname)) else: # Create subdirectory structure to work or source directory work_subdir_path = os.path.abspath( os.path.join(self.instance.basedir, dst_subdir)) if not os.path.exists(work_subdir_path): try: create_dir(work_subdir_path) except OSError: self._toolbox.msg_error.emit( "[OSError] Creating directory <b>{0}</b> failed." .format(work_subdir_path)) continue self._toolbox.msg.emit( "\tCopying optional file <b>{0}</b> into subdirectory <b>{2}{1}</b>" .format(dst_fname, dst_subdir, os.path.sep)) dst_path = os.path.abspath( os.path.join(work_subdir_path, dst_fname)) try: shutil.copyfile(src_path, dst_path) n_copied_files += 1 except OSError as e: self._toolbox.msg_error.emit( "Copying optional file <b>{0}</b> to <b>{1}</b> failed" .format(src_path, dst_path)) self._toolbox.msg_error.emit("{0}".format(e)) if e.errno == 22: msg = "The reason might be:\n" \ "[1] The destination file already exists and it cannot be " \ "overwritten because it is locked by Julia or some other application.\n" \ "[2] You don't have the necessary permissions to overwrite the file.\n" \ "To solve the problem, you can try the following:\n[1] Execute the Tool in work " \ "directory.\n[2] If you are executing a Julia Tool with Julia 0.6.x, upgrade to " \ "Julia 0.7 or newer.\n" \ "[3] Close any other background application(s) that may have locked the file.\n" \ "And try again.\n" self._toolbox.msg_warning.emit(msg) self._toolbox.msg.emit( "\tCopied <b>{0}</b> optional input file(s)".format( n_copied_files)) return True