def __init__(self,structure,*args):
if not common.is_dataset(structure):
raise TypeError,'argument structure must be StructuredPoints|StructuredGrid|UnstructuredGrid|RectilinearGrid|PolyData but got %s'%(type(structure))
self.structure = structure
for a in args:
if common.is_string(a):
if len(a)>255:
self.skipping('striping header string to length 256')
self.header = a[:256]
elif common.is_pointdata(a):
self.point_data = a
elif common.is_celldata(a):
self.cell_data = a
else:
self.skipping('unexpexted argument %s'%(type(a)))
if self.header is None:
self.header = 'Really cool data'
self.warning('Using header=%s'%(`self.header`))
if self.point_data is None and self.cell_data is None:
self.warning('No data defined')
if self.point_data is not None:
s = self.structure.get_size()
s1 = self.point_data.get_size()
if s1 != s:
raise ValueError,'DataSet (size=%s) and PointData (size=%s) have different sizes'%(s,s1)
if self.cell_data is not None:
s = self.structure.get_cell_size()
s1 = self.cell_data.get_size()
if s1 != s:
raise ValueError,'DataSet (cell_size=%s) and CellData (size=%s) have different sizes'%(s,s1)
def __init__(self, structure, *args):
if not common.is_dataset(structure):
raise TypeError, 'argument structure must be StructuredPoints|StructuredGrid|UnstructuredGrid|RectilinearGrid|PolyData but got %s' % (
type(structure))
self.structure = structure
for a in args:
if common.is_string(a):
if len(a) > 255:
self.skipping('striping header string to length 256')
self.header = a[:256]
elif common.is_pointdata(a):
self.point_data = a
elif common.is_celldata(a):
self.cell_data = a
else:
self.skipping('unexpexted argument %s' % (type(a)))
if self.header is None:
self.header = 'Really cool data'
self.warning('Using header=%s' % ( ` self.header `))
if self.point_data is None and self.cell_data is None:
self.warning('No data defined')
if self.point_data is not None:
s = self.structure.get_size()
s1 = self.point_data.get_size()
if s1 != s:
raise ValueError, 'DataSet (size=%s) and PointData (size=%s) have different sizes' % (
s, s1)
if self.cell_data is not None:
s = self.structure.get_cell_size()
s1 = self.cell_data.get_size()
if s1 != s:
raise ValueError, 'DataSet (cell_size=%s) and CellData (size=%s) have different sizes' % (
s, s1)
def _get_name(self, name):
if name is None:
name = self._get_default_name()
self.warning('Using name=%s' % ( ` name `))
return name
if common.is_string(name):
name = name.strip().replace(' ', '_')
if name:
return name
raise ValueError, 'name=%s must be non-empty string' % ( ` name `)
def _get_lookup_table(self,name):
if name is None:
name = 'default'
self.warning('Using lookup_table=%s'%(`name`))
return name
if common.is_string(name):
name = name.strip().replace(' ','_')
if name:
return name
raise ValueError,'lookup_table=%s must be nonempty string'%(`name`)
def _get_name(self,name):
if name is None:
name = self._get_default_name()
self.warning('Using name=%s'%(`name`))
return name
if common.is_string(name):
name = name.strip().replace(' ','_')
if name:
return name
raise ValueError,'name=%s must be non-empty string'%(`name`)
def _get_name(self, name):
if name is None:
name = self._get_default_name()
self.warning("Using name=%s" % (` name `))
return name
if common.is_string(name):
name = name.strip().replace(" ", "_")
if name:
return name
raise ValueError, "name=%s must be non-empty string" % (` name `)
def _get_lookup_table(self, name):
if name is None:
name = "default"
self.warning("Using lookup_table=%s" % (` name `))
return name
if common.is_string(name):
name = name.strip().replace(" ", "_")
if name:
return name
raise ValueError, "lookup_table=%s must be nonempty string" % (` name `)
def _render_value(self, val, context, delimiters=None):
"""
Render an arbitrary value.
"""
if not is_string(val):
# In case the template is an integer, for example.
val = self.to_str(val)
if type(val) is not unicode:
val = self.literal(val)
return self.render(val, context, delimiters)
def _get_lookup_table(self,name):
if name is None:
name = 'default'
self.warning('Using lookup_table=%s'%(`name`))
return name
if common.is_string(name):
name = string.replace(string.strip(name),' ','_')
#name = name.strip().replace(' ','_')
if name:
return name
raise ValueError,'lookup_table=%s must be nonempty string'%(`name`)
def fetch_string(self, context, name):
"""
Get a value from the given context as a basestring instance.
"""
val = self.resolve_context(context, name)
if callable(val):
# Return because _render_value() is already a string.
return self._render_value(val(), context)
if not is_string(val):
return self.to_str(val)
return val
def __init__(self, *args, **kws):
assert args, 'expected at least one argument'
if type(args[0]) is types.StringType:
if kws.has_key('only_structure') and kws['only_structure']:
self.fromfile(args[0], 1)
else:
self.fromfile(args[0])
return
else:
structure = args[0]
args = list(args)[1:]
if not common.is_dataset(structure):
raise TypeError, 'argument structure must be StructuredPoints|StructuredGrid|UnstructuredGrid|RectilinearGrid|PolyData but got %s' % (
type(structure))
self.structure = structure
for a in args:
if common.is_string(a):
if len(a) > 255:
self.skipping('striping header string to a length =255')
self.header = a[:255]
elif common.is_pointdata(a):
self.point_data = a
elif common.is_celldata(a):
self.cell_data = a
else:
self.skipping('unexpexted argument %s' % (type(a)))
if self.header is None:
self.header = 'Really cool data'
self.warning('Using header=%s' % ( ` self.header `))
if self.point_data is None and self.cell_data is None:
self.warning('No data defined')
if self.point_data is not None:
s = self.structure.get_size()
s1 = self.point_data.get_size()
if s1 != s:
raise ValueError, 'DataSet (size=%s) and PointData (size=%s) have different sizes' % (
s, s1)
else:
self.point_data = PointData()
if self.cell_data is not None:
s = self.structure.get_cell_size()
s1 = self.cell_data.get_size()
if s1 != s:
raise ValueError, 'DataSet (cell_size=%s) and CellData (size=%s) have different sizes' % (
s, s1)
else:
self.cell_data = CellData()
def tofile(self, filename, format = 'ascii'):
"""Save VTK data to file.
"""
if not common.is_string(filename):
raise TypeError,'argument filename must be string but got %s'%(type(filename))
if format not in ['ascii','binary']:
raise TypeError,'argument format must be ascii | binary'
filename = filename.strip()
if not filename:
raise ValueError,'filename must be non-empty string'
if filename[-4:]!='.vtk':
filename += '.vtk'
#print 'Creating file',`filename`
f = open(filename,'wb')
f.write(self.to_string(format))
f.close()
def tofile(self, filename, format = 'ascii'):
"""Save VTK data to file.
"""
if not common.is_string(filename):
raise TypeError,'argument filename must be string but got %s'%(type(filename))
if format not in ['ascii','binary']:
raise TypeError,'argument format must be ascii | binary'
filename = filename.strip()
if not filename:
raise ValueError,'filename must be non-empty string'
if filename[-4:]!='.vtk':
filename += '.vtk'
#print 'Creating file',`filename`
f = open(filename,'wb')
f.write(self.to_string(format))
f.close()
def _determine_ok_200(self, requests_verb, url, timeout):
if common.is_string(self.regular_file_url):
reg_url = url + self.regular_file_url
ok_resp = requests_verb(reg_url, timeout=timeout)
ok_200 = ok_resp.status_code == 200
else:
ok_200 = False
for path in self.regular_file_url:
reg_url = url + path
ok_resp = requests_verb(reg_url)
if ok_resp.status_code == 200:
ok_200 = True
break
len_content = len(ok_resp.content)
return ok_200, len_content
def _determine_ok_200(self, requests_verb, url, timeout):
if common.is_string(self.regular_file_url):
reg_url = url + self.regular_file_url
ok_resp = requests_verb(reg_url, timeout=timeout)
ok_200 = ok_resp.status_code == 200
else:
ok_200 = False
for path in self.regular_file_url:
reg_url = url + path
ok_resp = requests_verb(reg_url, timeout=timeout)
if ok_resp.status_code == 200:
ok_200 = True
break
len_content = len(ok_resp.content)
return ok_200, len_content
def __init__(self,*args,**kws):
assert args,'expected at least one argument'
if type(args[0]) is types.StringType:
if kws.has_key('only_structure') and kws['only_structure']:
self.fromfile(args[0],1)
else:
self.fromfile(args[0])
return
else:
structure = args[0]
args = list(args)[1:]
if not common.is_dataset(structure):
raise TypeError,'argument structure must be StructuredPoints|StructuredGrid|UnstructuredGrid|RectilinearGrid|PolyData but got %s'%(type(structure))
self.structure = structure
for a in args:
if common.is_string(a):
if len(a)>255:
self.skipping('striping header string to a length =255')
self.header = a[:255]
elif common.is_pointdata(a):
self.point_data = a
elif common.is_celldata(a):
self.cell_data = a
else:
self.skipping('unexpexted argument %s'%(type(a)))
if self.header is None:
self.header = 'Really cool data'
self.warning('Using header=%s'%(`self.header`))
if self.point_data is None and self.cell_data is None:
self.warning('No data defined')
if self.point_data is not None:
s = self.structure.get_size()
s1 = self.point_data.get_size()
if s1 != s:
raise ValueError,'DataSet (size=%s) and PointData (size=%s) have different sizes'%(s,s1)
else:
self.point_data = PointData()
if self.cell_data is not None:
s = self.structure.get_cell_size()
s1 = self.cell_data.get_size()
if s1 != s:
raise ValueError,'DataSet (cell_size=%s) and CellData (size=%s) have different sizes'%(s,s1)
else:
self.cell_data = CellData()
def tofile(self, filename, format = 'ascii'):
if not common.is_string(filename):
raise TypeError,'argument filename must be string but got %s'%(type(filename))
if format not in ['ascii','binary']:
raise TypeError,'argument format must be ascii | binary'
filename = string.strip(filename)
if not filename:
raise ValueError,'filename must be non-empty string'
if filename[-4:]!='.vtk':
filename = filename + '.vtk'
f = open(filename,'wb')
f.write('# vtk DataFile Version 2.0\n')
f.write(self.header+'\n')
f.write(string.upper(format)+'\n')
f.write(self.structure.to_string(format)+'\n')
if self.cell_data:
f.write(self.cell_data.to_string(format)+'\n')
if self.point_data:
f.write(self.point_data.to_string(format))
f.close()
def tofile(self, filename, format='ascii'):
if not common.is_string(filename):
raise TypeError, 'argument filename must be string but got %s' % (
type(filename))
if format not in ['ascii', 'binary']:
raise TypeError, 'argument format must be ascii | binary'
filename = string.strip(filename)
if not filename:
raise ValueError, 'filename must be non-empty string'
if filename[-4:] != '.vtk':
filename = filename + '.vtk'
f = open(filename, 'wb')
f.write('# vtk DataFile Version 2.0\n')
f.write(self.header + '\n')
f.write(string.upper(format) + '\n')
f.write(self.structure.to_string(format) + '\n')
if self.cell_data:
f.write(self.cell_data.to_string(format) + '\n')
if self.point_data:
f.write(self.point_data.to_string(format))
f.close()
def render(self, template, *context, **kwargs):
"""
Render the given template string, view template, or parsed template.
Returns a unicode string.
Prior to rendering, this method will convert a template that is a
byte string (type str in Python 2) to unicode using the string_encoding
and decode_errors attributes. See the constructor docstring for
more information.
Arguments:
template: a template string that is unicode or a byte string,
a ParsedTemplate instance, or another object instance. In the
final case, the function first looks for the template associated
to the object by calling this class's get_associated_template()
method. The rendering process also uses the passed object as
the first element of the context stack when rendering.
*context: zero or more dictionaries, ContextStack instances, or objects
with which to populate the initial context stack. None
arguments are skipped. Items in the *context list are added to
the context stack in order so that later items in the argument
list take precedence over earlier items.
**kwargs: additional key-value data to add to the context stack.
As these arguments appear after all items in the *context list,
in the case of key conflicts these values take precedence over
all items in the *context list.
"""
if is_string(template):
return self._render_string(template, *context, **kwargs)
if isinstance(template, ParsedTemplate):
render_func = lambda engine, stack: template.render(engine, stack)
return self._render_final(render_func, *context, **kwargs)
# Otherwise, we assume the template is an object.
return self._render_object(template, *context, **kwargs)
def fetch_section_data(self, context, name):
"""
Fetch the value of a section as a list.
"""
data = self.resolve_context(context, name)
# From the spec:
#
# If the data is not of a list type, it is coerced into a list
# as follows: if the data is truthy (e.g. `!!data == true`),
# use a single-element list containing the data, otherwise use
# an empty list.
#
if not data:
data = []
else:
# The least brittle way to determine whether something
# supports iteration is by trying to call iter() on it:
#
# http://docs.python.org/library/functions.html#iter
#
# It is not sufficient, for example, to check whether the item
# implements __iter__ () (the iteration protocol). There is
# also __getitem__() (the sequence protocol). In Python 2,
# strings do not implement __iter__(), but in Python 3 they do.
try:
iter(data)
except TypeError:
# Then the value does not support iteration.
data = [data]
else:
if is_string(data) or isinstance(data, dict):
# Do not treat strings and dicts (which are iterable) as lists.
data = [data]
# Otherwise, treat the value as a list.
return data
def enumerate(self, url, base_url_supplied, scanning_method,
iterator_returning_method, iterator_len, max_iterator=500, threads=10,
verb='head', timeout=15, hide_progressbar=False, imu=None):
'''
@param url base URL for the website.
@param base_url_supplied Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method see ScanningMethod
@param iterator_returning_method a function which returns an
element that, when iterated, will return a full list of plugins
@param iterator_len the number of items the above iterator can
return, regardless of user preference.
@param max_iterator integer that will be passed unto iterator_returning_method
@param threads number of threads
@param verb what HTTP verb. Valid options are 'get' and 'head'.
@param timeout the time, in seconds, that requests should wait
before throwing an exception.
@param hide_progressbar if true, the progressbar will not be
displayed.
@param imu Interesting module urls. A list containing tuples in the
following format [('readme.txt', 'default readme')].
'''
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_common_file
else:
url_template = base_url
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb, plugin_url,
timeout=timeout)
if plugin_url.endswith('/'):
final_url = plugin_url
else:
final_url = dirname(plugin_url) + "/"
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': final_url,
})
if not hide_progressbar:
p = ProgressBar(sys.stderr)
items_progressed = 0
max_possible = max_iterator if int(max_iterator) < int(iterator_len) else iterator_len
items_total = int(max_possible) * len(base_urls)
no_results = True
found = []
for future_array in futures:
if not hide_progressbar:
items_progressed += 1
p.set(items_progressed, items_total)
r = future_array['future'].result()
if r.status_code in [200, 403]:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({
'name': plugin_name,
'url': plugin_url
})
elif r.status_code >= 500:
self.out.warn('\rGot a 500 error. Is the server overloaded?')
if not hide_progressbar:
p.hide()
if imu != None and not no_results:
found = self._enumerate_plugin_if(found, verb, threads, imu)
return found, no_results
def enumerate(self,
url,
base_url_supplied,
scanning_method,
iterator_returning_method,
max_iterator=500,
threads=10,
verb='head',
timeout=15):
'''
@param url base URL for the website.
@param base_url_supplied Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method see ScanningMethod
@param iterator_returning_method a function which returns an
element that, when iterated, will return a full list of plugins
@param max_iterator integer that will be passed unto iterator_returning_method
@param threads number of threads
@param verb what HTTP verb. Valid options are 'get' and 'head'.
@param timeout the time, in seconds, that requests should wait
before throwing an exception.
'''
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_readme_file
expected_status = 200
else:
url_template = base_url
expected_status = common.scan_http_status(scanning_method)
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb,
plugin_url,
timeout=timeout)
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': plugin_url,
})
p = ProgressBar(sys.stderr)
items_progressed = 0
items_total = len(base_urls) * int(max_iterator)
no_results = True
found = []
for future_array in futures:
items_progressed += 1
p.set(items_progressed, items_total)
r = future_array['future'].result()
if r.status_code == expected_status:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({'name': plugin_name, 'url': plugin_url})
elif r.status_code >= 500:
self.out.warn('Got a 500 error. Is the server overloaded?')
p.hide()
return found, no_results
def enumerate(self, url, base_url_supplied, scanning_method, iterator_returning_method, max_iterator=500, threads=10, verb='head', timeout=15):
'''
@param url base URL for the website.
@param base_url_supplied Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method see ScanningMethod
@param iterator_returning_method a function which returns an
element that, when iterated, will return a full list of plugins
@param max_iterator integer that will be passed unto iterator_returning_method
@param threads number of threads
@param verb what HTTP verb. Valid options are 'get' and 'head'.
@param timeout the time, in seconds, that requests should wait
before throwing an exception.
'''
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_readme_file
expected_status = 200
else:
url_template = base_url
expected_status = common.scan_http_status(scanning_method)
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb, plugin_url,
timeout=timeout)
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': plugin_url,
})
p = ProgressBar(sys.stderr)
items_progressed = 0
items_total = len(base_urls) * int(max_iterator)
no_results = True
found = []
for future_array in futures:
items_progressed += 1
p.set(items_progressed, items_total)
r = future_array['future'].result()
if r.status_code == expected_status:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({
'name': plugin_name,
'url': plugin_url
})
elif r.status_code >= 500:
self.out.warn('Got a 500 error. Is the server overloaded?')
p.hide()
return found, no_results
def enumerate(self, url, base_url_supplied, scanning_method,
iterator_returning_method, iterator_len, max_iterator=500, threads=10,
verb='head', timeout=15, hide_progressbar=False, imu=None):
'''
@param url: base URL for the website.
@param base_url_supplied: Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method: see ScanningMethod
@param iterator_returning_method: a function which returns an
element that, when iterated, will return a full list of plugins
@param iterator_len: the number of items the above iterator can
return, regardless of user preference.
@param max_iterator: integer that will be passed unto iterator_returning_method
@param threads: number of threads
@param verb: what HTTP verb. Valid options are 'get' and 'head'.
@param timeout: the time, in seconds, that requests should wait
before throwing an exception.
@param hide_progressbar: if true, the progressbar will not be
displayed.
@param imu: Interesting module urls. A list containing tuples in the
following format [('readme.txt', 'default readme')].
'''
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_common_file
else:
url_template = base_url
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb, plugin_url,
timeout=timeout)
if plugin_url.endswith('/'):
final_url = plugin_url
else:
final_url = dirname(plugin_url) + "/"
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': final_url,
})
if not hide_progressbar:
max_possible = max_iterator if int(max_iterator) < int(iterator_len) else iterator_len
items_total = int(max_possible) * len(base_urls)
p = ProgressBar(sys.stderr, items_total, "modules")
no_results = True
found = []
for future_array in futures:
if shutdown:
future_array['future'].cancel()
continue
if not hide_progressbar:
p.increment_progress()
r = future_array['future'].result()
if r.status_code in [200, 403]:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({
'name': plugin_name,
'url': plugin_url
})
elif r.status_code >= 500:
self.out.warn('\rGot a 500 error. Is the server overloaded?')
if not hide_progressbar:
p.hide()
if not shutdown and (imu != None and not no_results):
found = self._enumerate_plugin_if(found, verb, threads, imu,
hide_progressbar)
return found, no_results
