def find_resources(self): """This resolves the cluster directory. This tries to find the cluster directory and if successful, it will have done: * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for the application. * Sets ``self.cluster_dir`` attribute of the application and config objects. The algorithm used for this is as follows: 1. Try ``Global.cluster_dir``. 2. Try using ``Global.profile``. 3. If both of these fail and ``self.auto_create_cluster_dir`` is ``True``, then create the new cluster dir in the IPython directory. 4. If all fails, then raise :class:`ClusterDirError`. """ try: cluster_dir = self.command_line_config.Global.cluster_dir except AttributeError: cluster_dir = self.default_config.Global.cluster_dir cluster_dir = expand_path(cluster_dir) try: self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) except ClusterDirError: pass else: self.log.info('Using existing cluster dir: %s' % \ self.cluster_dir_obj.location ) self.finish_cluster_dir() return try: self.profile = self.command_line_config.Global.profile except AttributeError: self.profile = self.default_config.Global.profile try: self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( self.ipython_dir, self.profile) except ClusterDirError: pass else: self.log.info('Using existing cluster dir: %s' % \ self.cluster_dir_obj.location ) self.finish_cluster_dir() return if self.auto_create_cluster_dir: self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile( self.ipython_dir, self.profile ) self.log.info('Creating new cluster dir: %s' % \ self.cluster_dir_obj.location ) self.finish_cluster_dir() else: raise ClusterDirError('Could not find a valid cluster directory.')
def find_resources(self): """This resolves the cluster directory. This tries to find the cluster directory and if successful, it will have done: * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for the application. * Sets ``self.cluster_dir`` attribute of the application and config objects. The algorithm used for this is as follows: 1. Try ``Global.cluster_dir``. 2. Try using ``Global.profile``. 3. If both of these fail and ``self.auto_create_cluster_dir`` is ``True``, then create the new cluster dir in the IPython directory. 4. If all fails, then raise :class:`ClusterDirError`. """ try: cluster_dir = self.command_line_config.Global.cluster_dir except AttributeError: cluster_dir = self.default_config.Global.cluster_dir cluster_dir = expand_path(cluster_dir) try: self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) except ClusterDirError: pass else: self.log.info('Using existing cluster dir: %s' % \ self.cluster_dir_obj.location ) self.finish_cluster_dir() return try: self.profile = self.command_line_config.Global.profile except AttributeError: self.profile = self.default_config.Global.profile try: self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( self.ipython_dir, self.profile) except ClusterDirError: pass else: self.log.info('Using existing cluster dir: %s' % \ self.cluster_dir_obj.location ) self.finish_cluster_dir() return if self.auto_create_cluster_dir: self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile( self.ipython_dir, self.profile) self.log.info('Creating new cluster dir: %s' % \ self.cluster_dir_obj.location ) self.finish_cluster_dir() else: raise ClusterDirError('Could not find a valid cluster directory.')
def check_furl_file_security(furl_file, secure): """Remove the old furl_file if changing security modes.""" furl_file = expand_path(furl_file) if os.path.isfile(furl_file): with open(furl_file, 'r') as f: oldfurl = f.read().strip() if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure): os.remove(furl_file)
def find_profile_dir(cls, profile_dir, config=None): """Find/create a profile dir and return its ProfileDir. This will create the profile directory if it doesn't exist. Parameters ---------- profile_dir : unicode or str The path of the profile directory. """ profile_dir = expand_path(profile_dir) if not os.path.isdir(profile_dir): raise ProfileDirError('Profile directory not found: %s' % profile_dir) return cls(location=profile_dir, config=config)
def find_furl(furl_or_file): """Find, validate and return a FURL in a string or file. This calls :func:`IPython.utils.path.expand_path` on the argument to properly handle ``~`` and ``$`` variables in the path. """ if is_valid_furl(furl_or_file): return furl_or_file furl_or_file = expand_path(furl_or_file) if is_valid_furl_file(furl_or_file): with open(furl_or_file, 'r') as f: furl = f.read().strip() if is_valid_furl(furl): return furl raise FURLError("Not a valid FURL or FURL file: %r" % furl_or_file)
def pre_construct(self): # The log and security dirs were set earlier, but here we put them # into the config and log them. config = self.master_config sdir = self.cluster_dir_obj.security_dir self.security_dir = config.Global.security_dir = sdir ldir = self.cluster_dir_obj.log_dir self.log_dir = config.Global.log_dir = ldir pdir = self.cluster_dir_obj.pid_dir self.pid_dir = config.Global.pid_dir = pdir self.log.info("Cluster directory set to: %s" % self.cluster_dir) config.Global.work_dir = unicode(expand_path(config.Global.work_dir)) # Change to the working directory. We do this just before construct # is called so all the components there have the right working dir. self.to_work_dir()
def find_cluster_dir(cls, cluster_dir): """Find/create a cluster dir and return its ClusterDir. This will create the cluster directory if it doesn't exist. Parameters ---------- cluster_dir : unicode or str The path of the cluster directory. This is expanded using :func:`IPython.utils.genutils.expand_path`. """ cluster_dir = expand_path(cluster_dir) if not os.path.isdir(cluster_dir): raise ClusterDirError('Cluster directory not found: %s' % cluster_dir) return ClusterDir(cluster_dir)
def find_cluster_dir(cls, cluster_dir): """Find/create a cluster dir and return its ClusterDir. This will create the cluster directory if it doesn't exist. Parameters ---------- cluster_dir : unicode or str The path of the cluster directory. This is expanded using :func:`IPython.utils.genutils.expand_path`. """ cluster_dir = expand_path(cluster_dir) if not os.path.isdir(cluster_dir): raise ClusterDirError('Cluster directory not found: %s' % cluster_dir) return ClusterDir(location=cluster_dir)
import os from IPython.utils.path import expand_path # Configuration file for ipcluster. c = get_config() # delay between controller starting and engine starting c.IPClusterStart.delay = 1.0 # set working directory work_dir = expand_path('~/ipclusterworkdir') if not os.path.exists(work_dir): os.makedirs(work_dir) c.IPClusterStart.work_dir = work_dir # set the default number of engines to be 1 c.IPClusterEngines.n = 1 c.IPClusterStart.daemonize = False c.IPClusterStart.controller_launcher_class = 'LocalControllerLauncher' c.IPClusterEngines.engine_launcher_class = 'PBSEngineSetLauncher' c.LocalControllerLauncher.controller_cmd = ['ipcontroller'] c.PBSEngineSetLauncher.job_id_regexp = '\\d+' c.PBSEngineSetLauncher.submit_command = ['qsub'] c.PBSEngineSetLauncher.delete_command = ['qdel'] c.PBSEngineSetLauncher.queue = u'default'
def render(self, filename, output=None): """ Convert the filename to the given output format(s) """ # Export each documents conversion_success = 0 converted_docs = [] # save here to change back after the conversation. orig_cwd = os.getcwd() needs_chdir = False # expand $HOME and so on... filename = expand_path(filename) filename = os.path.abspath(filename) self.log.info("Converting %s..." % filename) basedir = os.path.dirname(filename) basename = os.path.splitext(os.path.basename(filename))[0] # It's easier if we just change wd to the dir of the file if unicode_type(basedir) != py3compat.getcwd(): os.chdir(basedir) needs_chdir = True self.log.info("Changing to working dir: %s" % basedir) filename = os.path.basename(filename) outputdir_name = os.path.splitext(basename)[0] + "_files" # parse the input document parsed, metadata = self.parse_document(filename) # get the output formats # order: kwarg overwrites default overwrites document output_formats = [self._outputs[self.default_export_format]] if output is None: self.log.debug("Converting to default output format [%s]!" % self.default_export_format) elif output == "all": outputs = metadata.get("output", None) # if nothing is specified, we keep the default if outputs is None: self.log.debug( "Did not find any specified output formats: using only default!" ) else: output_formats = [] for fmt_name, config in iteritems(outputs): fod = self.get_output_format(fmt_name, config) output_formats.append(fod) self.log.debug( "Converting to all specified output formats: %s" % [fmt.name for fmt in output_formats]) else: self._ensure_valid_output(output) output_formats = [self._outputs[output]] for final_format in output_formats: self.log.info("Converting document %s to %s", filename, final_format.name) # TODO: build a proper way to specify final output... md_temp = TemporaryOutputDocument(fileoutputs=outputdir_name, export_config=final_format, log=self.log, parent=self) # get the temporary md file self.convert(parsed, md_temp) if final_format.keep_md or self.keep_md: mdfilename = basename + "." + final_format.name + ".md" self.log.info("Saving the temporary markdown as '%s'." % mdfilename) # TODO: remove the first yaml metadata block and # put "#<title>\n<author>\n<date>" before the rest with codecs.open(mdfilename, 'w+b', 'UTF-8') as f: f.write(md_temp.content) # convert the md file to the final filetype input_format = "markdown" \ "+autolink_bare_uris" \ "+ascii_identifiers" \ "+tex_math_single_backslash-implicit_figures" \ "+fenced_code_attributes" extra = [ "--smart", # typographically correct output (curly quotes, etc) "--email-obfuscation", "none", #do not obfuscation email names with javascript "--self-contained", # include img/scripts as data urls "--standalone", # html with header + footer "--section-divs", ] outfilename = basename + "." + final_format.file_extension # exported is irrelevant, as we pass in a filename exported = pandoc(source=md_temp.content, to=final_format.pandoc_export_format, format=input_format, extra_args=extra, outputfile=outfilename) self.log.info("Written final output: %s" % outfilename) converted_docs.append(os.path.join(basedir, outfilename)) if needs_chdir: os.chdir(orig_cwd) return converted_docs
def _work_dir_changed(self, name, old, new): self.work_dir = unicode_type(expand_path(new))
def _work_dir_changed(self, name, old, new): self.work_dir = str(expand_path(new))
def render(self, filename, output=None): """ Convert the filename to the given output format(s) """ # Export each documents conversion_success = 0 converted_docs = [] # save here to change back after the conversation. orig_cwd = os.getcwd() needs_chdir = False # expand $HOME and so on... filename = expand_path(filename) filename = os.path.abspath(filename) self.log.info("Converting %s..." % filename) basedir = os.path.dirname(filename) basename = os.path.splitext(os.path.basename(filename))[0] # It's easier if we just change wd to the dir of the file if unicode_type(basedir) != py3compat.getcwd(): os.chdir(basedir) needs_chdir = True self.log.info("Changing to working dir: %s" % basedir) filename = os.path.basename(filename) outputdir_name = os.path.splitext(basename)[0] + "_files" # parse the input document parsed, metadata = self.parse_document(filename) # get the output formats # order: kwarg overwrites default overwrites document output_formats = [self._outputs[self.default_export_format]] if output is None: self.log.debug("Converting to default output format [%s]!" % self.default_export_format) elif output == "all": outputs = metadata.get("output", None) # if nothing is specified, we keep the default if outputs is None: self.log.debug("Did not find any specified output formats: using only default!") else: output_formats = [] for fmt_name, config in iteritems(outputs): fod = self.get_output_format(fmt_name, config) output_formats.append(fod) self.log.debug("Converting to all specified output formats: %s" % [fmt.name for fmt in output_formats]) else: self._ensure_valid_output(output) output_formats = [self._outputs[output]] for final_format in output_formats: self.log.info("Converting document %s to %s", filename, final_format.name) # TODO: build a proper way to specify final output... md_temp = TemporaryOutputDocument(fileoutputs=outputdir_name, export_config=final_format, log=self.log, parent=self) # get the temporary md file self.convert(parsed, md_temp) if final_format.keep_md or self.keep_md: mdfilename = basename+"."+final_format.name+".md" self.log.info("Saving the temporary markdown as '%s'." % mdfilename) # TODO: remove the first yaml metadata block and # put "#<title>\n<author>\n<date>" before the rest with codecs.open(mdfilename, 'w+b','UTF-8') as f: f.write(md_temp.content) # convert the md file to the final filetype input_format = "markdown" \ "+autolink_bare_uris" \ "+ascii_identifiers" \ "+tex_math_single_backslash-implicit_figures" \ "+fenced_code_attributes" extra = ["--smart", # typographically correct output (curly quotes, etc) "--email-obfuscation", "none", #do not obfuscation email names with javascript "--self-contained", # include img/scripts as data urls "--standalone", # html with header + footer "--section-divs", ] outfilename = basename+"." +final_format.file_extension # exported is irrelevant, as we pass in a filename exported = pandoc(source=md_temp.content, to=final_format.pandoc_export_format, format=input_format, extra_args=extra, outputfile=outfilename) self.log.info("Written final output: %s" % outfilename) converted_docs.append(os.path.join(basedir, outfilename)) if needs_chdir: os.chdir(orig_cwd) return converted_docs
def _work_dir_changed(self, change): self.work_dir = unicode_type(expand_path(change['new']))
def _work_dir_changed(self, name, old, new): self.work_dir = unicode(expand_path(new))
def _work_dir_changed(self, change): self.work_dir = str(expand_path(change['new']))