def prepare_for_run(self,
configuration_path=None,
config=None,
run_id=None):
"""Configuration is given either as an opus path (configuration_path) or as a Configuration object (config)."""
run_manager = self.get_run_manager()
if run_id is not None:
config = run_manager.get_resources_for_run_id_from_history(
run_id=run_id)
else:
if configuration_path is not None:
config = get_config_from_opus_path(configuration_path)
elif config is None:
raise StandardError, "Either configuration_path, config or run_id must be given."
insert_auto_generated_cache_directory_if_needed(config)
run_manager.setup_new_run(
cache_directory=config['cache_directory'],
configuration=config)
run_id = run_manager.run_id
config['cache_directory'] = pathname2url(
run_manager.get_current_cache_directory())
## pathname2url converts '\' or '\\' to '/'; it is necessary when this script is invoked from a nt os
run_manager.add_row_to_history(run_id, config, "started")
#verify run_id has been added to services db
results = run_manager.services_db.GetResultsFromQuery(
"SELECT * from run_activity WHERE run_id = %s " % run_id)
if not len(results) > 1:
raise StandardError, "run_id %s doesn't exist in run_activity table." % run_id
return run_id, config
def prepare_for_run(self, configuration_path=None, config=None, run_id=None):
"""Configuration is given either as an opus path (configuration_path) or as a Configuration object (config)."""
run_manager = self.get_run_manager()
if run_id is not None:
config = run_manager.get_resources_for_run_id_from_history(run_id=run_id)
else:
if configuration_path is not None:
config = get_config_from_opus_path(configuration_path)
elif config is None:
raise StandardError, "Either configuration_path, config or run_id must be given."
insert_auto_generated_cache_directory_if_needed(config)
run_manager.setup_new_run(cache_directory = config['cache_directory'],
configuration = config)
run_id = run_manager.run_id
config['cache_directory'] = pathname2url(run_manager.get_current_cache_directory())
## pathname2url converts '\' or '\\' to '/'; it is necessary when this script is invoked from a nt os
run_manager.add_row_to_history(run_id, config, "started")
#verify run_id has been added to services db
results = run_manager.services_db.GetResultsFromQuery(
"SELECT * from run_activity WHERE run_id = %s " % run_id)
if not len(results) > 1:
raise StandardError, "run_id %s doesn't exist in run_activity table." % run_id
return run_id, config
def ParseURL(url, current = None):
url = url.encode('ascii')
if current:
current = current.encode('ascii')
if current.find('local://') == 0 and url.find('://') == -1:
url = 'local://' + url
else:
url = urlparse.urljoin(current, url)
elif url.find(':/') == -1:
url = 'http://' + url
repl = None
if 'res:/' in url:
repl = ('res:/', blue.paths.ResolvePath(u'res:/'))
elif 'script:/' in url:
repl = ('script:/', blue.paths.ResolvePath(u'script:/'))
elif 'cache:/' in url:
repl = ('cache:/', blue.paths.ResolvePath(u'cache:/'))
if repl:
url = url.replace(repl[0], 'file:' + nturl2path.pathname2url(repl[1]))
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
return (urlparse.urlunsplit((scheme,
netloc,
path,
query,
'')), fragment)
def path_to_url(path):
if not path:
raise ValueError('Not allowed: %r' % path)
if path.startswith('file:/'):
return path
path = abspath(expanduser(path))
url = urljoin('file:', pathname2url(path))
return url
def test_import_examples(example):
with open(IMPORT_SCHEMA) as schema_data:
schema = json.load(schema_data)
resolver = jsonschema.RefResolver(
'file:' + pathname2url(IMPORT_SCHEMA), schema)
result = jsonschema.Draft4Validator(
schema, resolver=resolver).validate(example)
assert result is None
def path_to_url(path):
if not path:
raise ValueError('Not allowed: %r' % path)
if path.startswith('file:/'):
return path
path = abspath(expanduser(path))
url = urljoin('file:', pathname2url(path))
return url
def test_html_doc(self):
result, doc_loc = get_pydoc_html(pydoc_mod)
mod_file = inspect.getabsfile(pydoc_mod)
if sys.platform == 'win32':
import nturl2path
mod_url = nturl2path.pathname2url(mod_file)
else:
mod_url = mod_file
expected_html = expected_html_pattern % (mod_url, mod_file, doc_loc)
if result != expected_html:
print_diffs(expected_html, result)
self.fail("outputs are not equal, see diff above")
def test_html_doc(self):
result, doc_loc = get_pydoc_html(pydoc_mod)
mod_file = inspect.getabsfile(pydoc_mod)
if sys.platform == 'win32':
import nturl2path
mod_url = nturl2path.pathname2url(mod_file)
else:
mod_url = mod_file
expected_html = expected_html_pattern % (
(mod_url, mod_file, doc_loc) +
expected_html_data_docstrings)
self.assertEqual(result, expected_html)
def test_html_doc(self):
result, doc_loc = get_pydoc_html(pydoc_mod)
mod_file = inspect.getabsfile(pydoc_mod)
if sys.platform == 'win32':
import nturl2path
mod_url = nturl2path.pathname2url(mod_file)
else:
mod_url = mod_file
expected_html = expected_html_pattern % (mod_url, mod_file, doc_loc)
if result != expected_html:
print_diffs(expected_html, result)
self.fail("outputs are not equal, see diff above")
def setPath(self, path):
rawDirectory = path
line = re.sub('.all', '', os.path.split(path)[1])
project = re.split('/', os.path.dirname(path))[1]
invProcessPath = '/Processing/CARIS/HDCS_Data/'
invProductsPath = '/Products/Caris_Exports/'
processDirectory = 'D:' + project + invProcessPath + project + '/' + project + '.hips'
processRoot = 'file:' + pathname2url(processDirectory)
tt = datetime.datetime.strptime(
re.split('[_.]',
os.path.split(path)[1])[1],
CarisBatch.dateFormat).timetuple()
vessel = re.split('[_.]',
os.path.split(path)[1])[3] + '_' + re.split(
'[_.]',
os.path.split(path)[1])[4]
jDay = str(tt.tm_year) + '-' + "{0:0>3}".format(str(tt.tm_yday))
lineQuery = 'Vessel=' + vessel + ';Day=' + jDay + ';Line=' + line
dayQuery = 'Vessel=' + vessel + ';Day=' + jDay
dayURI = 'file:' + pathname2url(
processDirectory) + '?Vessel=' + vessel + ';Day=' + jDay
self.invPointExport = 'D:/' + project + invProductsPath + 'Point Cloud/' + project + '_' + vessel
return rawDirectory, dayURI, lineQuery, dayQuery, processRoot
def ParseURL(url, current=None):
url = url.encode('ascii')
if current:
current = current.encode('ascii')
if current.find('local://') == 0 and url.find('://') == -1:
url = 'local://' + url
else:
url = urlparse.urljoin(current, url)
elif url.find(':/') == -1:
url = 'http://' + url
repl = None
if 'res:/' in url:
repl = ('res:/', blue.paths.ResolvePath(u'res:/'))
elif 'script:/' in url:
repl = ('script:/', blue.paths.ResolvePath(u'script:/'))
elif 'cache:/' in url:
repl = ('cache:/', blue.paths.ResolvePath(u'cache:/'))
if repl:
url = url.replace(repl[0], 'file:' + nturl2path.pathname2url(repl[1]))
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
return (urlparse.urlunsplit((scheme, netloc, path, query, '')), fragment)
def docmodule(self,
object,
name=None,
mod=None,
packageContext=None,
*ignored):
"""Produce HTML documentation for a module object."""
name = object.__name__ # ignore the passed-in name
parts = split(name, '.')
links = []
for i in range(len(parts) - 1):
links.append(
'<a href="%s.html"><font color="#ffffff">%s</font></a>' %
(join(parts[:i + 1], '.'), parts[i]))
linkedname = join(links + parts[-1:], '.')
head = '<big><big><strong>%s</strong></big></big>' % linkedname
try:
path = inspect.getabsfile(object)
url = path
if sys.platform == 'win32':
import nturl2path
url = nturl2path.pathname2url(path)
filelink = '<a href="file:%s">%s</a>' % (url, path)
except TypeError:
filelink = '(built-in)'
info = []
if hasattr(object, '__version__'):
version = str(object.__version__)
if version[:11] == '$' + 'Revision: ' and version[-1:] == '$':
version = strip(version[11:-1])
info.append('version %s' % self.escape(version))
if hasattr(object, '__date__'):
info.append(self.escape(str(object.__date__)))
if info:
head = head + ' (%s)' % join(info, ', ')
result = self.heading(head, '#ffffff', '#7799ee',
'<a href=".">index</a><br>' + filelink)
modules = inspect.getmembers(object, inspect.ismodule)
classes, cdict = [], {}
for key, value in inspect.getmembers(object, inspect.isclass):
if (inspect.getmodule(value) or object) is object:
classes.append((key, value))
cdict[key] = cdict[value] = '#' + key
for key, value in classes:
for base in value.__bases__:
key, modname = base.__name__, base.__module__
module = sys.modules.get(modname)
if modname != name and module and hasattr(module, key):
if getattr(module, key) is base:
if not cdict.has_key(key):
cdict[key] = cdict[base] = modname + '.html#' + key
funcs, fdict = [], {}
for key, value in inspect.getmembers(object, inspect.isroutine):
if inspect.isbuiltin(value) or inspect.getmodule(value) is object:
funcs.append((key, value))
fdict[key] = '#-' + key
if inspect.isfunction(value): fdict[value] = fdict[key]
data = []
for key, value in inspect.getmembers(object, pydoc.isdata):
if key not in ['__builtins__', '__doc__']:
data.append((key, value))
doc = self.markup(pydoc.getdoc(object), self.preformat, fdict, cdict)
doc = doc and '<tt>%s</tt>' % doc
result = result + '<p>%s</p>\n' % doc
packageContext.clean(classes, object)
packageContext.clean(funcs, object)
packageContext.clean(data, object)
if hasattr(object, '__path__'):
modpkgs = []
modnames = []
for file in os.listdir(object.__path__[0]):
path = os.path.join(object.__path__[0], file)
modname = inspect.getmodulename(file)
if modname and modname not in modnames:
modpkgs.append((modname, name, 0, 0))
modnames.append(modname)
elif pydoc.ispackage(path):
modpkgs.append((file, name, 1, 0))
modpkgs.sort()
contents = self.multicolumn(modpkgs, self.modpkglink)
## result = result + self.bigsection(
## 'Package Contents', '#ffffff', '#aa55cc', contents)
result = result + self.moduleSection(object, packageContext)
elif modules:
contents = self.multicolumn(
modules, lambda (key, value), s=self: s.modulelink(value))
result = result + self.bigsection('Modules', '#fffff', '#aa55cc',
contents)
if classes:
classlist = map(lambda (key, value): value, classes)
contents = [
self.formattree(inspect.getclasstree(classlist, 1), name)
]
for key, value in classes:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection('Classes', '#ffffff', '#ee77aa',
join(contents))
if funcs:
contents = []
for key, value in funcs:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection('Functions', '#ffffff',
'#eeaa77', join(contents))
if data:
contents = []
for key, value in data:
try:
contents.append(self.document(value, key))
except Exception, err:
pass
result = result + self.bigsection('Data', '#ffffff', '#55aa55',
join(contents, '<br>\n'))
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from resource_management import *
import nturl2path
config = Script.get_config()
ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
metric_collector_host = ams_collector_hosts[0]
metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_port and metric_collector_port.find(":") != -1:
metric_collector_port = metric_collector_port.split(":")[1]
pass
sink_home = os.environ["SINK_HOME"]
timeline_plugin_url = "file:" + nturl2path.pathname2url(
os.path.join(sink_home, "hadoop-sink", "ambari-metrics-hadoop-sink.jar")
)
hadoop_conf_dir = os.environ["HADOOP_CONF_DIR"]
hbase_conf_dir = os.environ["HBASE_CONF_DIR"]
hadoop_user = "******"
import nturl2path file = r"c:\my\little\pony" print(nturl2path.pathname2url(file)) print(nturl2path.url2pathname(nturl2path.pathname2url(file))) ## ///C|/my/little/pony ## C:\my\little\pony
def docmodule(self, object, name=None, mod=None, *ignored):
"""Produce HTML documentation for a module object."""
name = object.__name__ # ignore the passed-in name
try:
all = object.__all__
except AttributeError:
all = None
parts = name.split('.')
links = []
for i in range(len(parts) - 1):
links.append(
'<a href="{}.html" class="docmodule_link">{}</a>'.format(
'.'.join(parts[:i + 1]), parts[i]))
head = '.'.join(links + parts[-1:])
try:
path = inspect.getabsfile(object)
url = path
if sys.platform == 'win32':
import nturl2path
url = nturl2path.pathname2url(path)
filelink = self.filelink(url, path)
except TypeError:
filelink = '(built-in)'
info = []
if hasattr(object, '__version__'):
version = str(object.__version__)
if version[:11] == '$' + 'Revision: ' and version[-1:] == '$':
version = version[11:-1].strip()
info.append('version %s' % self.escape(version))
if hasattr(object, '__date__'):
info.append(self.escape(str(object.__date__)))
if info:
head = head + ' (%s)' % ', '.join(info)
docloc = self.getdocloc(object)
if docloc is not None:
docloc = ('<br><a href="%(docloc)s">Module Reference</a>' %
locals())
else:
docloc = ''
extras = '<a href=".">index</a><br>' + filelink + docloc
result = self.heading(head, extras)
modules = inspect.getmembers(object, inspect.ismodule)
classes, cdict = [], {}
for key, value in inspect.getmembers(object, inspect.isclass):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None
or (inspect.getmodule(value) or object) is object):
if visiblename(key, all, object):
classes.append((key, value))
cdict[key] = cdict[value] = '#' + key
for key, value in classes:
for base in value.__bases__:
key, modname = base.__name__, base.__module__
module = sys.modules.get(modname)
if modname != name and module and hasattr(module, key):
if getattr(module, key) is base:
if key not in cdict:
cdict[key] = cdict[base] = (modname +
'.html#' + key)
funcs, fdict = [], {}
for key, value in inspect.getmembers(object, inspect.isroutine):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None or inspect.isbuiltin(value)
or inspect.getmodule(value) is object):
if visiblename(key, all, object):
funcs.append((key, value))
fdict[key] = '#-' + key
if inspect.isfunction(value):
fdict[value] = fdict[key]
data = []
for key, value in inspect.getmembers(object, isdata):
if visiblename(key, all, object):
data.append((key, value))
doc = self.markup(getdoc(object), self.preformat, fdict, cdict)
doc = doc and '<code>{}</code>'.format(doc)
result = result + '<p>%s</p>\n' % doc
if hasattr(object, '__path__'):
modpkgs = []
for importer, modname, ispkg in pkgutil.iter_modules(
object.__path__):
modpkgs.append((modname, name, ispkg, 0))
modpkgs.sort()
contents = self.multicolumn(modpkgs, self.modpkglink)
result = result + self.bigsection(
'Package Contents', contents, css_class="package")
elif modules:
contents = self.multicolumn(modules,
lambda t: self.modulelink(t[1]))
result = result + self.bigsection(
'Modules', contents, css_class="module")
if classes:
classlist = [value for (key, value) in classes]
contents = [
self.formattree(inspect.getclasstree(classlist, 1), name)
]
for key, value in classes:
contents.append(
self.document(value, key, name, fdict, cdict))
result = result + self.bigsection(
'Classes', ' '.join(contents), css_class="classes")
if funcs:
contents = []
for key, value in funcs:
contents.append(
self.document(value, key, name, fdict, cdict))
result = result + self.bigsection(
'Functions', ' '.join(contents), css_class="functions")
if data:
contents = []
for key, value in data:
contents.append(self.document(value, key))
result = result + self.bigsection(
'Data', '<br>\n'.join(contents), css_class="data")
if hasattr(object, '__author__'):
contents = self.markup(str(object.__author__), self.preformat)
result = result + self.bigsection(
'Author', contents, css_class="author")
if hasattr(object, '__credits__'):
contents = self.markup(str(object.__credits__), self.preformat)
result = result + self.bigsection(
'Credits', contents, css_class="credits")
return result
def docmodule(self, object, name=None, mod=None, *ignored):
"""Produce HTML documentation for a module object."""
name = object.__name__ # ignore the passed-in name
try:
all = object.__all__
except AttributeError:
all = None
parts = split(name, '.')
links = []
for i in range(len(parts) - 1):
links.append(
'<a href="%s.html"><font color="#ffffff">%s</font></a>' %
(join(parts[:i + 1], '.'), parts[i]))
linkedname = join(links + parts[-1:], '.')
head = '<big><big><strong>%s</strong></big></big>' % linkedname
try:
path = inspect.getabsfile(object)
url = path
if sys.platform == 'win32':
import nturl2path
url = nturl2path.pathname2url(path)
# modified
filelink = self.filelink(url, path)
# end modified
except TypeError:
filelink = '(built-in)'
info = []
if hasattr(object, '__version__'):
version = str(object.__version__)
if version[:11] == '$' + 'Revision: ' and version[-1:] == '$':
version = strip(version[11:-1])
info.append('version %s' % self.escape(version))
if hasattr(object, '__date__'):
info.append(self.escape(str(object.__date__)))
if info:
head = head + ' (%s)' % join(info, ', ')
docloc = self.getdocloc(object)
if docloc is not None:
docloc = '<br><a href="%(docloc)s">Module Docs</a>' % locals()
else:
docloc = ''
result = self.heading(head, '#ffffff', '#7799ee',
'<a href=".">index</a><br>' + filelink + docloc)
modules = inspect.getmembers(object, inspect.ismodule)
classes, cdict = [], {}
for key, value in inspect.getmembers(object, inspect.isclass):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None
or (inspect.getmodule(value) or object) is object):
if visiblename(key, all):
classes.append((key, value))
cdict[key] = cdict[value] = '#' + key
for key, value in classes:
for base in value.__bases__:
key, modname = base.__name__, base.__module__
module = sys.modules.get(modname)
if modname != name and module and hasattr(module, key):
if getattr(module, key) is base:
if not key in cdict:
cdict[key] = cdict[base] = modname + '.html#' + key
funcs, fdict = [], {}
for key, value in inspect.getmembers(object, inspect.isroutine):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None or inspect.isbuiltin(value)
or inspect.getmodule(value) is object):
if visiblename(key, all):
funcs.append((key, value))
fdict[key] = '#-' + key
if inspect.isfunction(value): fdict[value] = fdict[key]
data = []
for key, value in inspect.getmembers(object, isdata):
if visiblename(key, all):
data.append((key, value))
doc = self.markup(getdoc(object), self.preformat, fdict, cdict)
doc = doc and '<tt>%s</tt>' % doc
result = result + '<p>%s</p>\n' % doc
if hasattr(object, '__path__'):
modpkgs = []
for importer, modname, ispkg in pkgutil.iter_modules(
object.__path__):
modpkgs.append((modname, name, ispkg, 0))
modpkgs.sort()
contents = self.multicolumn(modpkgs, self.modpkglink)
result = result + self.bigsection('Package Contents', '#ffffff',
'#aa55cc', contents)
elif modules:
contents = self.multicolumn(
modules, lambda (key, value), s=self: s.modulelink(value))
result = result + self.bigsection('Modules', '#fffff', '#aa55cc',
contents)
if classes:
classlist = map(lambda (key, value): value, classes)
contents = [
self.formattree(inspect.getclasstree(classlist, 1), name)
]
for key, value in classes:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection('Classes', '#ffffff', '#ee77aa',
join(contents))
if funcs:
contents = []
for key, value in funcs:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection('Functions', '#ffffff',
'#eeaa77', join(contents))
if data:
contents = []
for key, value in data:
contents.append(self.document(value, key))
result = result + self.bigsection('Data', '#ffffff', '#55aa55',
join(contents, '<br>\n'))
if hasattr(object, '__author__'):
contents = self.markup(str(object.__author__), self.preformat)
result = result + self.bigsection('Author', '#ffffff', '#7799ee',
contents)
if hasattr(object, '__credits__'):
contents = self.markup(str(object.__credits__), self.preformat)
result = result + self.bigsection('Credits', '#ffffff', '#7799ee',
contents)
return result
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
if 'cluster-env' in config['configurations'] and \
'metrics_collector_vip_host' in config['configurations']['cluster-env']:
metric_collector_host = config['configurations']['cluster-env'][
'metrics_collector_vip_host']
else:
metric_collector_host = ams_collector_hosts[0]
if 'cluster-env' in config['configurations'] and \
'metrics_collector_vip_port' in config['configurations']['cluster-env']:
metric_collector_port = config['configurations']['cluster-env'][
'metrics_collector_vip_port']
else:
metric_collector_web_address = default(
"/configurations/ams-site/timeline.metrics.service.webapp.address",
"localhost:6188")
if metric_collector_web_address.find(':') != -1:
metric_collector_port = metric_collector_web_address.split(':')[1]
else:
metric_collector_port = '6188'
sink_home = os.environ["SINK_HOME"]
timeline_plugin_url = "file:" + nturl2path.pathname2url(
os.path.join(sink_home, "hadoop-sink",
"ambari-metrics-hadoop-sink.jar"))
pass
hadoop_conf_dir = os.environ["HADOOP_CONF_DIR"]
hbase_conf_dir = os.environ["HBASE_CONF_DIR"]
hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
#_urlopen = _opener.open
_urlopen = aurlopen
try:
fd = _urlopen(url,data,referer=referer)
except HTTPError,why:
raise HTTPError,why
except (ValueError,URLError,OSError),why:
try:
fd = _urlopen(urllib.unquote(url),data,referer=referer)
except HTTPError,why:
raise HTTPError,why
except (ValueError,URLError,OSError),why:
from nturl2path import pathname2url
try:
fd = _urlopen('file:'+pathname2url(url),data,referer=referer)
except HTTPError,why:
raise HTTPError,why
except (ValueError,URLError,OSError):
fd = _urlopen('file:'+pathname2url(urllib.unquote(url)),referer=referer)
return fd
class RateController:
MINIMUM_RATE = 3
def __init__(self,job,current_rate):
self.job = job
self.current_rate = current_rate
self.new_rate = current_rate
import nturl2path file = r"c:\my\little\pony" print nturl2path.pathname2url(file) print nturl2path.url2pathname(nturl2path.pathname2url(file)) ## ///C|/my/little/pony ## C:\my\little\pony
def docmodule(self, object, name=None, mod=None, *ignored):
"""Produce HTML5 documentation for a module object."""
level = 1 # FIXME: use passed level in the future
components = {} # where we're storing all components to be output
name = object.__name__ # ignore the passed-in name. not passed in anyways?
try:
all = object.__all__
except AttributeError:
all = None
parts = name.split('.')
links = []
for i in range(len(parts) - 1):
links.append('<a href="%s">%s</a>' %
(self.url('.'.join(parts[:i + 1])), parts[i]))
head_link = '.'.join(links + parts[-1:])
try:
path = inspect.getabsfile(object)
if self.local:
url = path
if sys.platform == 'win32': # in case i want to give this to the python project
import nturl2path
url = nturl2path.pathname2url(path)
components[
'fileref'] = '<a class="file-reference" href="file:%s">%s</a>' % (
url, path)
else:
components[
'fileref'] = '<span class="file-reference">%s</span>' % path
except TypeError:
components[
'fileref'] = '<span class="file-reference builtin">(built-in)</span>'
components['fileref'] = '' # TODO remove fileref
info = []
if hasattr(object, '__version__'):
version = pydoc._binstr(object.__version__)
if version[:11] == '$' + 'Revision: ' and version[-1:] == '$':
version = strip(version[11:-1])
info.append('version %s' % self.escape(version))
if hasattr(object, '__date__'):
info.append(self.escape(pydoc._binstr(object.__date__)))
# build the main heading
if info:
components['head'] = self.heading(
level + 1,
'%s (<span class="info">%s)' % (head_link, ', '.join(info)))
else:
# heading which is a linked representation of the module "address"
components['head'] = self.heading(level + 1, head_link)
# get the official url of object, if any
docloc = self.getdocloc(object)
if docloc is not None:
components[
'docloc'] = '<a class="official-docs" href="%s" target="_blank" rel="noreferrer noopener">Module Docs</a>' % docloc
else:
components['docloc'] = ''
# collect modules, classes, functions and data in `object`
modules = inspect.getmembers(object, inspect.ismodule)
classes, cdict = [], {}
for key, value in inspect.getmembers(object, inspect.isclass):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None
or (inspect.getmodule(value) or object) is object):
if pydoc.visiblename(key, all, object):
classes.append((key, value))
cdict[key] = cdict[value] = '#' + \
key # key used as URL fragment
for key, value in classes:
for base in value.__bases__:
key, modname = base.__name__, base.__module__
module = sys.modules.get(modname)
if modname != name and module and hasattr(module, key):
if getattr(module, key) is base:
if not key in cdict:
cdict[key] = cdict[base] = self.url(
modname
) + '#' + key # key used as URL fragment
funcs, fdict = [], {}
for key, value in inspect.getmembers(object, inspect.isroutine):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None or inspect.isbuiltin(value)
or inspect.getmodule(value) is object):
if pydoc.visiblename(key, all, object):
funcs.append((key, value))
fdict[key] = '#-' + key
if inspect.isfunction(value):
fdict[value] = fdict[key]
data = []
for key, value in inspect.getmembers(object, pydoc.isdata):
if pydoc.visiblename(key, all, object):
data.append((key, value))
# build documentation for the thing passed in
components['doc'] = self.getdoc(object)
if hasattr(object, '__path__'):
modpkgs = []
for importer, modname, ispkg in pkgutil.iter_modules(
object.__path__):
modpkgs.append((modname, name, ispkg, 0))
modpkgs.sort()
components['modules'] = self.heading(
level + 2, 'Package Contents') + self.listing(
modpkgs, formatter=self.modpkglink)
elif modules:
components['modules'] = self.heading(
level + 2, 'Modules') + self.listing(
[module for _, module in modules],
formatter=self.modulelink)
if classes:
classlist = [cls for _, cls in classes]
classtree = self.formattree(inspect.getclasstree(classlist, 1),
name)
classdocs = []
for key, value in classes:
classdocs.append(self.document(value, key, name, fdict, cdict))
components['classes'] = self.heading(level + 2, 'Classes')
components['classes'] += classtree
components['classes'] += '<dl class="classes">'
components['classes'] += '\n'.join(classdocs)
components['classes'] += '</dl>'
if funcs:
docs = []
for key, value in funcs:
docs.append(self.document(value, key, name, fdict, cdict))
components['funcs'] = self.heading(level + 2, 'Functions')
components['funcs'] += '<dl class="functions">'
components['funcs'] += '\n'.join(docs)
components['funcs'] += '</dl>'
if data:
docs = []
for key, value in data:
docs.append(self.document(value, key))
components['data'] = self.heading(level + 2, 'Data')
components['data'] += '<dl class="data">'
components['data'] += '\n'.join(docs)
components['data'] += '</dl>'
if hasattr(object, '__author__'):
components['author'] = self.heading(
level + 2, 'Author') + pydoc._binstr(object.__author__)
if hasattr(object, '__credits__'):
components['credits'] = self.geadubg(
level + 2, 'Credits') + pydoc._binstr(object.__credits__)
result = '%(head)s %(docloc)s' % components
# result = '%(head)s %(fileref)s %(docloc)s' % components # TODO fileref disabled
result += '<div class="module">' % components
result += ' <div class="docstring">%(doc)s</div>' % components
if 'modules' in components:
result += ' <div class="modules">%(modules)s</div>' % components
if 'classes' in components:
result += ' <div class="classes">%(classes)s</div>' % components
if 'funcs' in components:
result += ' <div class="functions">%(funcs)s</div>' % components
if 'author' in components:
result += '<div class="author">%(author)s</div>' % components
if 'credits' in components:
result += '<div class="credits">%(credits)s</div>' % components
result += '</div>'
return result
def update_event(self, inp=-1):
self.set_output_val(0, nturl2path.pathname2url(self.input(0)))
def docmodule(self, obj, name=None, mod=None, package_context=None, *ignored):
"""Produce HTML documentation for a module object."""
name = obj.__name__ # ignore the passed-in name
parts = name.split('.')
links = []
for i in range(len(parts)-1):
links.append(
'<a href="%s.html"><font color="#ffffff">%s</font></a>' %
('.'.join(parts[:i+1]), parts[i]))
linked_name = '.'.join(links + parts[-1:])
head = '<big><big><strong>{0}</strong></big></big>'.format(linked_name)
try:
path = inspect.getabsfile(obj)
url = path
if sys.platform == 'win32':
import nturl2path
url = nturl2path.pathname2url(path)
fake_link = '<a href="file:{0}">{1}</a>'.format(url, path)
except TypeError:
fake_link = '(built-in)'
info = []
if hasattr(obj, '__version__'):
version = str(obj.__version__)
if version[:11] == '$' + 'Revision: ' and version[-1:] == '$':
version = version[11:-1].strip()
info.append('version %s' % self.escape(version))
if hasattr(obj, '__date__'):
info.append(self.escape(str(obj.__date__)))
if info:
head += ' ({0})'.format(', '.join(info))
result = self.heading(
head, '#ffffff', '#7799ee', '<a href=".">index</a><br>' + fake_link)
modules = inspect.getmembers(obj, inspect.ismodule)
classes, cdict = [], {}
for key, value in inspect.getmembers(obj, inspect.isclass):
if (inspect.getmodule(value) or obj) is obj:
classes.append((key, value))
cdict[key] = cdict[value] = '#' + key
for key, value in classes:
for base in value.__bases__:
key, modname = base.__name__, base.__module__
module = sys.modules.get(modname)
if modname != name and module and hasattr(module, key):
if getattr(module, key) is base:
if key not in cdict:
cdict[key] = cdict[base] = modname + '.html#' + key
funcs, fdict = [], {}
for key, value in inspect.getmembers(obj, inspect.isroutine):
if inspect.isbuiltin(value) or inspect.getmodule(value) is obj:
funcs.append((key, value))
fdict[key] = '#-' + key
if inspect.isfunction(value):
fdict[value] = fdict[key]
data = []
for key, value in inspect.getmembers(obj, pydoc.isdata):
if key not in ['__builtins__', '__doc__']:
data.append((key, value))
doc = self.markup(pydoc.getdoc(obj), self.preformat, fdict, cdict)
doc = doc and '<tt>%s</tt>' % doc
result += '<p>{0}</p>\n'.format(doc)
package_context.clean(classes, obj)
package_context.clean(funcs, obj)
package_context.clean(data, obj)
if hasattr(obj, '__path__'):
modpkgs = []
modnames = []
for file in os.listdir(obj.__path__[0]):
path = os.path.join(obj.__path__[0], file)
modname = inspect.getmodulename(file)
if modname and modname not in modnames:
modpkgs.append((modname, name, 0, 0))
modnames.append(modname)
elif pydoc.ispackage(path):
modpkgs.append((file, name, 1, 0))
modpkgs.sort()
contents = self.multicolumn(modpkgs, self.modpkglink)
## result = result + self.bigsection(
## 'Package Contents', '#ffffff', '#aa55cc', contents)
result = result + self.module_section(obj, package_context)
elif modules:
contents = self.multicolumn(
modules,
lambda a: self.modulelink(a[1])
)
result = result + self.bigsection(
'Modules', '#fffff', '#aa55cc', contents)
if classes:
class_list = map(lambda a: a[1], classes)
contents = [
self.formattree(inspect.getclasstree(class_list, 1), name)]
for key, value in classes:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection(
'Classes', '#ffffff', '#ee77aa', ''.join(contents)
)
if funcs:
contents = []
for key, value in funcs:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection(
'Functions', '#ffffff', '#eeaa77', ''.join(contents)
)
if data:
contents = []
for key, value in data:
try:
contents.append(self.document(value, key))
except Exception:
pass
result = result + self.bigsection(
'Data', '#ffffff', '#55aa55', '<br>\n'.join(contents)
)
if hasattr(obj, '__author__'):
contents = self.markup(str(obj.__author__), self.preformat)
result = result + self.bigsection(
'Author', '#ffffff', '#7799ee', contents)
if hasattr(obj, '__credits__'):
contents = self.markup(str(obj.__credits__), self.preformat)
result = result + self.bigsection(
'Credits', '#ffffff', '#7799ee', contents)
return result
def docmodule(self, object, name=None, mod=None, packageContext=None, *ignored):
"""Produce HTML documentation for a module object."""
name = object.__name__ # ignore the passed-in name
parts = split(name, ".")
links = []
for i in range(len(parts) - 1):
links.append(
'<a href="%s.html"><font color="#ffffff">%s</font></a>' % (join(parts[: i + 1], "."), parts[i])
)
linkedname = join(links + parts[-1:], ".")
head = "<big><big><strong>%s</strong></big></big>" % linkedname
try:
path = inspect.getabsfile(object)
url = path
if sys.platform == "win32":
import nturl2path
url = nturl2path.pathname2url(path)
filelink = '<a href="file:%s">%s</a>' % (url, path)
except TypeError:
filelink = "(built-in)"
info = []
if hasattr(object, "__version__"):
version = str(object.__version__)
if version[:11] == "$" + "Revision: " and version[-1:] == "$":
version = strip(version[11:-1])
info.append("version %s" % self.escape(version))
if hasattr(object, "__date__"):
info.append(self.escape(str(object.__date__)))
if info:
head = head + " (%s)" % join(info, ", ")
result = self.heading(head, "#ffffff", "#7799ee", '<a href=".">index</a><br>' + filelink)
modules = inspect.getmembers(object, inspect.ismodule)
classes, cdict = [], {}
for key, value in inspect.getmembers(object, inspect.isclass):
if (inspect.getmodule(value) or object) is object:
classes.append((key, value))
cdict[key] = cdict[value] = "#" + key
for key, value in classes:
for base in value.__bases__:
key, modname = base.__name__, base.__module__
module = sys.modules.get(modname)
if modname != name and module and hasattr(module, key):
if getattr(module, key) is base:
if not cdict.has_key(key):
cdict[key] = cdict[base] = modname + ".html#" + key
funcs, fdict = [], {}
for key, value in inspect.getmembers(object, inspect.isroutine):
if inspect.isbuiltin(value) or inspect.getmodule(value) is object:
funcs.append((key, value))
fdict[key] = "#-" + key
if inspect.isfunction(value):
fdict[value] = fdict[key]
data = []
for key, value in inspect.getmembers(object, pydoc.isdata):
if key not in ["__builtins__", "__doc__"]:
data.append((key, value))
doc = self.markup(pydoc.getdoc(object), self.preformat, fdict, cdict)
doc = doc and "<tt>%s</tt>" % doc
result = result + "<p>%s</p>\n" % doc
packageContext.clean(classes, object)
packageContext.clean(funcs, object)
packageContext.clean(data, object)
if hasattr(object, "__path__"):
modpkgs = []
modnames = []
for file in os.listdir(object.__path__[0]):
path = os.path.join(object.__path__[0], file)
modname = inspect.getmodulename(file)
if modname and modname not in modnames:
modpkgs.append((modname, name, 0, 0))
modnames.append(modname)
elif pydoc.ispackage(path):
modpkgs.append((file, name, 1, 0))
modpkgs.sort()
contents = self.multicolumn(modpkgs, self.modpkglink)
## result = result + self.bigsection(
## 'Package Contents', '#ffffff', '#aa55cc', contents)
result = result + self.moduleSection(object, packageContext)
elif modules:
contents = self.multicolumn(modules, lambda (key, value), s=self: s.modulelink(value))
result = result + self.bigsection("Modules", "#fffff", "#aa55cc", contents)
if classes:
classlist = map(lambda (key, value): value, classes)
contents = [self.formattree(inspect.getclasstree(classlist, 1), name)]
for key, value in classes:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection("Classes", "#ffffff", "#ee77aa", join(contents))
if funcs:
contents = []
for key, value in funcs:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection("Functions", "#ffffff", "#eeaa77", join(contents))
if data:
contents = []
for key, value in data:
contents.append(self.document(value, key))
result = result + self.bigsection("Data", "#ffffff", "#55aa55", join(contents, "<br>\n"))
if hasattr(object, "__author__"):
contents = self.markup(str(object.__author__), self.preformat)
result = result + self.bigsection("Author", "#ffffff", "#7799ee", contents)
if hasattr(object, "__credits__"):
contents = self.markup(str(object.__credits__), self.preformat)
result = result + self.bigsection("Credits", "#ffffff", "#7799ee", contents)
return result
if 0:
def jyhref(path):
href = os.path.abspath(path)
href = href.replace('\\', '/')
return 'file:' + href
name = os.path.join(os.path.dirname(__file__), '..', 'sheets',
'import.css')
#href = 'file:' + urllib.pathname2url(name)
from nturl2path import pathname2url
href = pathname2url(os.path.abspath(name))
href = href.replace('|', ':')
href = href[3:]
href = 'file:' + href
href = jyhref(name)
href = None
print(name, href)
s = cssutils.parseFile(name, href=href, media='screen', title='from file')
print(0, s)
print(1, s.cssRules[0].styleSheet)
print(2, s.cssRules[0].styleSheet.cssText)
sys.exit(0)
def docmodule(self, object, name=None, mod=None, *ignored):
"""Produce HTML documentation for a module object."""
name = object.__name__ # ignore the passed-in name
try:
all = object.__all__
except AttributeError:
all = None
parts = split(name, '.')
links = []
for i in range(len(parts)-1):
links.append(
'<a href="%s.html"><font color="#ffffff">%s</font></a>' %
(join(parts[:i+1], '.'), parts[i]))
linkedname = join(links + parts[-1:], '.')
head = '<big><big><strong>%s</strong></big></big>' % linkedname
try:
path = inspect.getabsfile(object)
url = path
if sys.platform == 'win32':
import nturl2path
url = nturl2path.pathname2url(path)
# modified
filelink = self.filelink(url, path)
# end modified
except TypeError:
filelink = '(built-in)'
info = []
if hasattr(object, '__version__'):
version = str(object.__version__)
if version[:11] == '$' + 'Revision: ' and version[-1:] == '$':
version = strip(version[11:-1])
info.append('version %s' % self.escape(version))
if hasattr(object, '__date__'):
info.append(self.escape(str(object.__date__)))
if info:
head = head + ' (%s)' % join(info, ', ')
docloc = self.getdocloc(object)
if docloc is not None:
docloc = '<br><a href="%(docloc)s">Module Docs</a>' % locals()
else:
docloc = ''
result = self.heading(
head, '#ffffff', '#7799ee',
'<a href=".">index</a><br>' + filelink + docloc)
modules = inspect.getmembers(object, inspect.ismodule)
classes, cdict = [], {}
for key, value in inspect.getmembers(object, inspect.isclass):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None or
(inspect.getmodule(value) or object) is object):
if visiblename(key, all):
classes.append((key, value))
cdict[key] = cdict[value] = '#' + key
for key, value in classes:
for base in value.__bases__:
key, modname = base.__name__, base.__module__
module = sys.modules.get(modname)
if modname != name and module and hasattr(module, key):
if getattr(module, key) is base:
if not key in cdict:
cdict[key] = cdict[base] = modname + '.html#' + key
funcs, fdict = [], {}
for key, value in inspect.getmembers(object, inspect.isroutine):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None or
inspect.isbuiltin(value) or inspect.getmodule(value) is object):
if visiblename(key, all):
funcs.append((key, value))
fdict[key] = '#-' + key
if inspect.isfunction(value): fdict[value] = fdict[key]
data = []
for key, value in inspect.getmembers(object, isdata):
if visiblename(key, all):
data.append((key, value))
doc = self.markup(getdoc(object), self.preformat, fdict, cdict)
doc = doc and '<tt>%s</tt>' % doc
result = result + '<p>%s</p>\n' % doc
if hasattr(object, '__path__'):
modpkgs = []
for importer, modname, ispkg in pkgutil.iter_modules(object.__path__):
modpkgs.append((modname, name, ispkg, 0))
modpkgs.sort()
contents = self.multicolumn(modpkgs, self.modpkglink)
result = result + self.bigsection(
'Package Contents', '#ffffff', '#aa55cc', contents)
elif modules:
contents = self.multicolumn(
modules, lambda (key, value), s=self: s.modulelink(value))
result = result + self.bigsection(
'Modules', '#fffff', '#aa55cc', contents)
if classes:
classlist = map(lambda (key, value): value, classes)
contents = [
self.formattree(inspect.getclasstree(classlist, 1), name)]
for key, value in classes:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection(
'Classes', '#ffffff', '#ee77aa', join(contents))
if funcs:
contents = []
for key, value in funcs:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection(
'Functions', '#ffffff', '#eeaa77', join(contents))
if data:
contents = []
for key, value in data:
contents.append(self.document(value, key))
result = result + self.bigsection(
'Data', '#ffffff', '#55aa55', join(contents, '<br>\n'))
if hasattr(object, '__author__'):
contents = self.markup(str(object.__author__), self.preformat)
result = result + self.bigsection(
'Author', '#ffffff', '#7799ee', contents)
if hasattr(object, '__credits__'):
contents = self.markup(str(object.__credits__), self.preformat)
result = result + self.bigsection(
'Credits', '#ffffff', '#7799ee', contents)
return result
return path
if 0:
def jyhref(path):
href = os.path.abspath(path)
href = href.replace('\\', '/')
return 'file:'+href
name = os.path.join(os.path.dirname(__file__),
'..', 'sheets', 'import.css')
#href = 'file:' + urllib.pathname2url(name)
from nturl2path import pathname2url
href = pathname2url(os.path.abspath(name))
href = href.replace('|', ':')
href = href[3:]
href = u'file:' + href
href = jyhref(name)
href = None
print name, href
s = cssutils.parseFile(name, href=href, media='screen', title='from file')
print 0, s
print 1, s.cssRules[0].styleSheet
print 2, s.cssRules[0].styleSheet.cssText
sys.exit(0)
