def __init__(self,structure,*args): if not common.is_dataset(structure): raise TypeError,'argument structure must be StructuredPoints|StructuredGrid|UnstructuredGrid|RectilinearGrid|PolyData but got %s'%(type(structure)) self.structure = structure for a in args: if common.is_string(a): if len(a)>255: self.skipping('striping header string to length 256') self.header = a[:256] elif common.is_pointdata(a): self.point_data = a elif common.is_celldata(a): self.cell_data = a else: self.skipping('unexpexted argument %s'%(type(a))) if self.header is None: self.header = 'Really cool data' self.warning('Using header=%s'%(`self.header`)) if self.point_data is None and self.cell_data is None: self.warning('No data defined') if self.point_data is not None: s = self.structure.get_size() s1 = self.point_data.get_size() if s1 != s: raise ValueError,'DataSet (size=%s) and PointData (size=%s) have different sizes'%(s,s1) if self.cell_data is not None: s = self.structure.get_cell_size() s1 = self.cell_data.get_size() if s1 != s: raise ValueError,'DataSet (cell_size=%s) and CellData (size=%s) have different sizes'%(s,s1)
def __init__(self, structure, *args): if not common.is_dataset(structure): raise TypeError, 'argument structure must be StructuredPoints|StructuredGrid|UnstructuredGrid|RectilinearGrid|PolyData but got %s' % ( type(structure)) self.structure = structure for a in args: if common.is_string(a): if len(a) > 255: self.skipping('striping header string to length 256') self.header = a[:256] elif common.is_pointdata(a): self.point_data = a elif common.is_celldata(a): self.cell_data = a else: self.skipping('unexpexted argument %s' % (type(a))) if self.header is None: self.header = 'Really cool data' self.warning('Using header=%s' % ( ` self.header `)) if self.point_data is None and self.cell_data is None: self.warning('No data defined') if self.point_data is not None: s = self.structure.get_size() s1 = self.point_data.get_size() if s1 != s: raise ValueError, 'DataSet (size=%s) and PointData (size=%s) have different sizes' % ( s, s1) if self.cell_data is not None: s = self.structure.get_cell_size() s1 = self.cell_data.get_size() if s1 != s: raise ValueError, 'DataSet (cell_size=%s) and CellData (size=%s) have different sizes' % ( s, s1)
def _get_name(self, name): if name is None: name = self._get_default_name() self.warning('Using name=%s' % ( ` name `)) return name if common.is_string(name): name = name.strip().replace(' ', '_') if name: return name raise ValueError, 'name=%s must be non-empty string' % ( ` name `)
def _get_lookup_table(self,name): if name is None: name = 'default' self.warning('Using lookup_table=%s'%(`name`)) return name if common.is_string(name): name = name.strip().replace(' ','_') if name: return name raise ValueError,'lookup_table=%s must be nonempty string'%(`name`)
def _get_name(self,name): if name is None: name = self._get_default_name() self.warning('Using name=%s'%(`name`)) return name if common.is_string(name): name = name.strip().replace(' ','_') if name: return name raise ValueError,'name=%s must be non-empty string'%(`name`)
def _get_name(self, name): if name is None: name = self._get_default_name() self.warning("Using name=%s" % (` name `)) return name if common.is_string(name): name = name.strip().replace(" ", "_") if name: return name raise ValueError, "name=%s must be non-empty string" % (` name `)
def _get_lookup_table(self, name): if name is None: name = "default" self.warning("Using lookup_table=%s" % (` name `)) return name if common.is_string(name): name = name.strip().replace(" ", "_") if name: return name raise ValueError, "lookup_table=%s must be nonempty string" % (` name `)
def _render_value(self, val, context, delimiters=None): """ Render an arbitrary value. """ if not is_string(val): # In case the template is an integer, for example. val = self.to_str(val) if type(val) is not unicode: val = self.literal(val) return self.render(val, context, delimiters)
def _get_lookup_table(self,name): if name is None: name = 'default' self.warning('Using lookup_table=%s'%(`name`)) return name if common.is_string(name): name = string.replace(string.strip(name),' ','_') #name = name.strip().replace(' ','_') if name: return name raise ValueError,'lookup_table=%s must be nonempty string'%(`name`)
def fetch_string(self, context, name): """ Get a value from the given context as a basestring instance. """ val = self.resolve_context(context, name) if callable(val): # Return because _render_value() is already a string. return self._render_value(val(), context) if not is_string(val): return self.to_str(val) return val
def __init__(self, *args, **kws): assert args, 'expected at least one argument' if type(args[0]) is types.StringType: if kws.has_key('only_structure') and kws['only_structure']: self.fromfile(args[0], 1) else: self.fromfile(args[0]) return else: structure = args[0] args = list(args)[1:] if not common.is_dataset(structure): raise TypeError, 'argument structure must be StructuredPoints|StructuredGrid|UnstructuredGrid|RectilinearGrid|PolyData but got %s' % ( type(structure)) self.structure = structure for a in args: if common.is_string(a): if len(a) > 255: self.skipping('striping header string to a length =255') self.header = a[:255] elif common.is_pointdata(a): self.point_data = a elif common.is_celldata(a): self.cell_data = a else: self.skipping('unexpexted argument %s' % (type(a))) if self.header is None: self.header = 'Really cool data' self.warning('Using header=%s' % ( ` self.header `)) if self.point_data is None and self.cell_data is None: self.warning('No data defined') if self.point_data is not None: s = self.structure.get_size() s1 = self.point_data.get_size() if s1 != s: raise ValueError, 'DataSet (size=%s) and PointData (size=%s) have different sizes' % ( s, s1) else: self.point_data = PointData() if self.cell_data is not None: s = self.structure.get_cell_size() s1 = self.cell_data.get_size() if s1 != s: raise ValueError, 'DataSet (cell_size=%s) and CellData (size=%s) have different sizes' % ( s, s1) else: self.cell_data = CellData()
def tofile(self, filename, format = 'ascii'): """Save VTK data to file. """ if not common.is_string(filename): raise TypeError,'argument filename must be string but got %s'%(type(filename)) if format not in ['ascii','binary']: raise TypeError,'argument format must be ascii | binary' filename = filename.strip() if not filename: raise ValueError,'filename must be non-empty string' if filename[-4:]!='.vtk': filename += '.vtk' #print 'Creating file',`filename` f = open(filename,'wb') f.write(self.to_string(format)) f.close()
def _determine_ok_200(self, requests_verb, url, timeout): if common.is_string(self.regular_file_url): reg_url = url + self.regular_file_url ok_resp = requests_verb(reg_url, timeout=timeout) ok_200 = ok_resp.status_code == 200 else: ok_200 = False for path in self.regular_file_url: reg_url = url + path ok_resp = requests_verb(reg_url) if ok_resp.status_code == 200: ok_200 = True break len_content = len(ok_resp.content) return ok_200, len_content
def _determine_ok_200(self, requests_verb, url, timeout): if common.is_string(self.regular_file_url): reg_url = url + self.regular_file_url ok_resp = requests_verb(reg_url, timeout=timeout) ok_200 = ok_resp.status_code == 200 else: ok_200 = False for path in self.regular_file_url: reg_url = url + path ok_resp = requests_verb(reg_url, timeout=timeout) if ok_resp.status_code == 200: ok_200 = True break len_content = len(ok_resp.content) return ok_200, len_content
def __init__(self,*args,**kws): assert args,'expected at least one argument' if type(args[0]) is types.StringType: if kws.has_key('only_structure') and kws['only_structure']: self.fromfile(args[0],1) else: self.fromfile(args[0]) return else: structure = args[0] args = list(args)[1:] if not common.is_dataset(structure): raise TypeError,'argument structure must be StructuredPoints|StructuredGrid|UnstructuredGrid|RectilinearGrid|PolyData but got %s'%(type(structure)) self.structure = structure for a in args: if common.is_string(a): if len(a)>255: self.skipping('striping header string to a length =255') self.header = a[:255] elif common.is_pointdata(a): self.point_data = a elif common.is_celldata(a): self.cell_data = a else: self.skipping('unexpexted argument %s'%(type(a))) if self.header is None: self.header = 'Really cool data' self.warning('Using header=%s'%(`self.header`)) if self.point_data is None and self.cell_data is None: self.warning('No data defined') if self.point_data is not None: s = self.structure.get_size() s1 = self.point_data.get_size() if s1 != s: raise ValueError,'DataSet (size=%s) and PointData (size=%s) have different sizes'%(s,s1) else: self.point_data = PointData() if self.cell_data is not None: s = self.structure.get_cell_size() s1 = self.cell_data.get_size() if s1 != s: raise ValueError,'DataSet (cell_size=%s) and CellData (size=%s) have different sizes'%(s,s1) else: self.cell_data = CellData()
def tofile(self, filename, format = 'ascii'): if not common.is_string(filename): raise TypeError,'argument filename must be string but got %s'%(type(filename)) if format not in ['ascii','binary']: raise TypeError,'argument format must be ascii | binary' filename = string.strip(filename) if not filename: raise ValueError,'filename must be non-empty string' if filename[-4:]!='.vtk': filename = filename + '.vtk' f = open(filename,'wb') f.write('# vtk DataFile Version 2.0\n') f.write(self.header+'\n') f.write(string.upper(format)+'\n') f.write(self.structure.to_string(format)+'\n') if self.cell_data: f.write(self.cell_data.to_string(format)+'\n') if self.point_data: f.write(self.point_data.to_string(format)) f.close()
def tofile(self, filename, format='ascii'): if not common.is_string(filename): raise TypeError, 'argument filename must be string but got %s' % ( type(filename)) if format not in ['ascii', 'binary']: raise TypeError, 'argument format must be ascii | binary' filename = string.strip(filename) if not filename: raise ValueError, 'filename must be non-empty string' if filename[-4:] != '.vtk': filename = filename + '.vtk' f = open(filename, 'wb') f.write('# vtk DataFile Version 2.0\n') f.write(self.header + '\n') f.write(string.upper(format) + '\n') f.write(self.structure.to_string(format) + '\n') if self.cell_data: f.write(self.cell_data.to_string(format) + '\n') if self.point_data: f.write(self.point_data.to_string(format)) f.close()
def render(self, template, *context, **kwargs): """ Render the given template string, view template, or parsed template. Returns a unicode string. Prior to rendering, this method will convert a template that is a byte string (type str in Python 2) to unicode using the string_encoding and decode_errors attributes. See the constructor docstring for more information. Arguments: template: a template string that is unicode or a byte string, a ParsedTemplate instance, or another object instance. In the final case, the function first looks for the template associated to the object by calling this class's get_associated_template() method. The rendering process also uses the passed object as the first element of the context stack when rendering. *context: zero or more dictionaries, ContextStack instances, or objects with which to populate the initial context stack. None arguments are skipped. Items in the *context list are added to the context stack in order so that later items in the argument list take precedence over earlier items. **kwargs: additional key-value data to add to the context stack. As these arguments appear after all items in the *context list, in the case of key conflicts these values take precedence over all items in the *context list. """ if is_string(template): return self._render_string(template, *context, **kwargs) if isinstance(template, ParsedTemplate): render_func = lambda engine, stack: template.render(engine, stack) return self._render_final(render_func, *context, **kwargs) # Otherwise, we assume the template is an object. return self._render_object(template, *context, **kwargs)
def fetch_section_data(self, context, name): """ Fetch the value of a section as a list. """ data = self.resolve_context(context, name) # From the spec: # # If the data is not of a list type, it is coerced into a list # as follows: if the data is truthy (e.g. `!!data == true`), # use a single-element list containing the data, otherwise use # an empty list. # if not data: data = [] else: # The least brittle way to determine whether something # supports iteration is by trying to call iter() on it: # # http://docs.python.org/library/functions.html#iter # # It is not sufficient, for example, to check whether the item # implements __iter__ () (the iteration protocol). There is # also __getitem__() (the sequence protocol). In Python 2, # strings do not implement __iter__(), but in Python 3 they do. try: iter(data) except TypeError: # Then the value does not support iteration. data = [data] else: if is_string(data) or isinstance(data, dict): # Do not treat strings and dicts (which are iterable) as lists. data = [data] # Otherwise, treat the value as a list. return data
def enumerate(self, url, base_url_supplied, scanning_method, iterator_returning_method, iterator_len, max_iterator=500, threads=10, verb='head', timeout=15, hide_progressbar=False, imu=None): ''' @param url base URL for the website. @param base_url_supplied Base url for themes, plugins. E.g. '%ssites/all/modules/%s/' @param scanning_method see ScanningMethod @param iterator_returning_method a function which returns an element that, when iterated, will return a full list of plugins @param iterator_len the number of items the above iterator can return, regardless of user preference. @param max_iterator integer that will be passed unto iterator_returning_method @param threads number of threads @param verb what HTTP verb. Valid options are 'get' and 'head'. @param timeout the time, in seconds, that requests should wait before throwing an exception. @param hide_progressbar if true, the progressbar will not be displayed. @param imu Interesting module urls. A list containing tuples in the following format [('readme.txt', 'default readme')]. ''' if common.is_string(base_url_supplied): base_urls = [base_url_supplied] else: base_urls = base_url_supplied requests_verb = getattr(self.session, verb) futures = [] with ThreadPoolExecutor(max_workers=threads) as executor: for base_url in base_urls: plugins = iterator_returning_method(max_iterator) if scanning_method == ScanningMethod.not_found: url_template = base_url + self.module_common_file else: url_template = base_url for plugin_name in plugins: plugin_url = url_template % (url, plugin_name) future = executor.submit(requests_verb, plugin_url, timeout=timeout) if plugin_url.endswith('/'): final_url = plugin_url else: final_url = dirname(plugin_url) + "/" futures.append({ 'base_url': base_url, 'future': future, 'plugin_name': plugin_name, 'plugin_url': final_url, }) if not hide_progressbar: p = ProgressBar(sys.stderr) items_progressed = 0 max_possible = max_iterator if int(max_iterator) < int(iterator_len) else iterator_len items_total = int(max_possible) * len(base_urls) no_results = True found = [] for future_array in futures: if not hide_progressbar: items_progressed += 1 p.set(items_progressed, items_total) r = future_array['future'].result() if r.status_code in [200, 403]: plugin_url = future_array['plugin_url'] plugin_name = future_array['plugin_name'] no_results = False found.append({ 'name': plugin_name, 'url': plugin_url }) elif r.status_code >= 500: self.out.warn('\rGot a 500 error. Is the server overloaded?') if not hide_progressbar: p.hide() if imu != None and not no_results: found = self._enumerate_plugin_if(found, verb, threads, imu) return found, no_results
def enumerate(self, url, base_url_supplied, scanning_method, iterator_returning_method, max_iterator=500, threads=10, verb='head', timeout=15): ''' @param url base URL for the website. @param base_url_supplied Base url for themes, plugins. E.g. '%ssites/all/modules/%s/' @param scanning_method see ScanningMethod @param iterator_returning_method a function which returns an element that, when iterated, will return a full list of plugins @param max_iterator integer that will be passed unto iterator_returning_method @param threads number of threads @param verb what HTTP verb. Valid options are 'get' and 'head'. @param timeout the time, in seconds, that requests should wait before throwing an exception. ''' if common.is_string(base_url_supplied): base_urls = [base_url_supplied] else: base_urls = base_url_supplied requests_verb = getattr(self.session, verb) futures = [] with ThreadPoolExecutor(max_workers=threads) as executor: for base_url in base_urls: plugins = iterator_returning_method(max_iterator) if scanning_method == ScanningMethod.not_found: url_template = base_url + self.module_readme_file expected_status = 200 else: url_template = base_url expected_status = common.scan_http_status(scanning_method) for plugin_name in plugins: plugin_url = url_template % (url, plugin_name) future = executor.submit(requests_verb, plugin_url, timeout=timeout) futures.append({ 'base_url': base_url, 'future': future, 'plugin_name': plugin_name, 'plugin_url': plugin_url, }) p = ProgressBar(sys.stderr) items_progressed = 0 items_total = len(base_urls) * int(max_iterator) no_results = True found = [] for future_array in futures: items_progressed += 1 p.set(items_progressed, items_total) r = future_array['future'].result() if r.status_code == expected_status: plugin_url = future_array['plugin_url'] plugin_name = future_array['plugin_name'] no_results = False found.append({'name': plugin_name, 'url': plugin_url}) elif r.status_code >= 500: self.out.warn('Got a 500 error. Is the server overloaded?') p.hide() return found, no_results
def enumerate(self, url, base_url_supplied, scanning_method, iterator_returning_method, max_iterator=500, threads=10, verb='head', timeout=15): ''' @param url base URL for the website. @param base_url_supplied Base url for themes, plugins. E.g. '%ssites/all/modules/%s/' @param scanning_method see ScanningMethod @param iterator_returning_method a function which returns an element that, when iterated, will return a full list of plugins @param max_iterator integer that will be passed unto iterator_returning_method @param threads number of threads @param verb what HTTP verb. Valid options are 'get' and 'head'. @param timeout the time, in seconds, that requests should wait before throwing an exception. ''' if common.is_string(base_url_supplied): base_urls = [base_url_supplied] else: base_urls = base_url_supplied requests_verb = getattr(self.session, verb) futures = [] with ThreadPoolExecutor(max_workers=threads) as executor: for base_url in base_urls: plugins = iterator_returning_method(max_iterator) if scanning_method == ScanningMethod.not_found: url_template = base_url + self.module_readme_file expected_status = 200 else: url_template = base_url expected_status = common.scan_http_status(scanning_method) for plugin_name in plugins: plugin_url = url_template % (url, plugin_name) future = executor.submit(requests_verb, plugin_url, timeout=timeout) futures.append({ 'base_url': base_url, 'future': future, 'plugin_name': plugin_name, 'plugin_url': plugin_url, }) p = ProgressBar(sys.stderr) items_progressed = 0 items_total = len(base_urls) * int(max_iterator) no_results = True found = [] for future_array in futures: items_progressed += 1 p.set(items_progressed, items_total) r = future_array['future'].result() if r.status_code == expected_status: plugin_url = future_array['plugin_url'] plugin_name = future_array['plugin_name'] no_results = False found.append({ 'name': plugin_name, 'url': plugin_url }) elif r.status_code >= 500: self.out.warn('Got a 500 error. Is the server overloaded?') p.hide() return found, no_results
def enumerate(self, url, base_url_supplied, scanning_method, iterator_returning_method, iterator_len, max_iterator=500, threads=10, verb='head', timeout=15, hide_progressbar=False, imu=None): ''' @param url: base URL for the website. @param base_url_supplied: Base url for themes, plugins. E.g. '%ssites/all/modules/%s/' @param scanning_method: see ScanningMethod @param iterator_returning_method: a function which returns an element that, when iterated, will return a full list of plugins @param iterator_len: the number of items the above iterator can return, regardless of user preference. @param max_iterator: integer that will be passed unto iterator_returning_method @param threads: number of threads @param verb: what HTTP verb. Valid options are 'get' and 'head'. @param timeout: the time, in seconds, that requests should wait before throwing an exception. @param hide_progressbar: if true, the progressbar will not be displayed. @param imu: Interesting module urls. A list containing tuples in the following format [('readme.txt', 'default readme')]. ''' if common.is_string(base_url_supplied): base_urls = [base_url_supplied] else: base_urls = base_url_supplied requests_verb = getattr(self.session, verb) futures = [] with ThreadPoolExecutor(max_workers=threads) as executor: for base_url in base_urls: plugins = iterator_returning_method(max_iterator) if scanning_method == ScanningMethod.not_found: url_template = base_url + self.module_common_file else: url_template = base_url for plugin_name in plugins: plugin_url = url_template % (url, plugin_name) future = executor.submit(requests_verb, plugin_url, timeout=timeout) if plugin_url.endswith('/'): final_url = plugin_url else: final_url = dirname(plugin_url) + "/" futures.append({ 'base_url': base_url, 'future': future, 'plugin_name': plugin_name, 'plugin_url': final_url, }) if not hide_progressbar: max_possible = max_iterator if int(max_iterator) < int(iterator_len) else iterator_len items_total = int(max_possible) * len(base_urls) p = ProgressBar(sys.stderr, items_total, "modules") no_results = True found = [] for future_array in futures: if shutdown: future_array['future'].cancel() continue if not hide_progressbar: p.increment_progress() r = future_array['future'].result() if r.status_code in [200, 403]: plugin_url = future_array['plugin_url'] plugin_name = future_array['plugin_name'] no_results = False found.append({ 'name': plugin_name, 'url': plugin_url }) elif r.status_code >= 500: self.out.warn('\rGot a 500 error. Is the server overloaded?') if not hide_progressbar: p.hide() if not shutdown and (imu != None and not no_results): found = self._enumerate_plugin_if(found, verb, threads, imu, hide_progressbar) return found, no_results