def _render_tmpl_func(tmpl, func, **kwargs): # 先尝试寻找本template中的func func = str(func) if tmpl.has_def(func): _t = tmpl.get_def(func) else: # 在当前template的parent中寻找该func from mako import util from mako.template import DefTemplate from mako.runtime import Context, _populate_self_namespace # disable unicode 状态下生成 context buf = util.StringIO() context = Context(buf, **kwargs) context._with_template = tmpl # 将当前的namespace, 如'self', 'next'等放入context func_render_body, context = _populate_self_namespace(context, tmpl) # 找到当前template的inherits self_ns = context['self'] inherit_m = self_ns.inherits.module # 在inherits中寻找该func // mako 模板中为render_xxx func_name = "render_%s" % func if hasattr(inherit_m, func_name): _t = DefTemplate(tmpl, getattr(inherit_m, func_name)) else: _t = None if 'self' in kwargs: kwargs.pop('self') return _t and _t.render(**kwargs) or ''
def _render_tmpl_func(tmpl, func, **kwargs): func = str(func) if tmpl.has_def(func): _t = tmpl.get_def(func) else: from mako import util from mako.template import DefTemplate from mako.runtime import Context, _populate_self_namespace buf = util.StringIO() context = Context(buf, **kwargs) context._with_template = tmpl func_render_body, context = _populate_self_namespace(context, tmpl) self_ns = context['self'] inherit_m = self_ns.inherits.module func_name = "render_%s" % func if hasattr(inherit_m, func_name): _t = DefTemplate(tmpl, getattr(inherit_m, func_name)) else: _t = None if 'self' in kwargs: kwargs.pop('self') return _t and _t.render(**kwargs) or ''
{ 'src': './plotly_streaming.pre.cpp', 'destination': '../', 'folder_prefix': 'plotly_streaming_' }, { 'src': './plotly_streaming.pre.h', 'destination': '../', 'folder_prefix': 'plotly_streaming_' }, { 'src': './simple.pre.ino', 'destination': '../examples', 'folder_prefix': '' }, ] for m in meta: for lib in libs: src = m['src'] # Write file to a destination folder directory = m['destination'] + '/' + m['folder_prefix'] + lib.lower() if not os.path.exists(directory): os.makedirs(directory) fout = open(directory + '/' + src.replace('.pre', '_' + lib.lower()), 'w') mytemplate = Template(filename=src) ctx = Context(fout, lib=lib) mytemplate.render_context(ctx) fout.close()
"""from mako.template import Template print Template("hello ${data}!").render(data="world")""" """from mako.template import Template mytemplate = Template("hello ,${name}") print mytemplate.render(name="Pavan")""" from mako.template import Template from mako.runtime import Context from StringIO import StringIO mytemplate = Template("hello, ${name}!") buf = StringIO() ctx = Context(buf, name="jack") mytemplate.render_context(ctx) print buf.getvalue()
def main(argv: List[str]) -> None: got_input = False module_directory = None preprocessed_output = None dictionary = {} json_dict = {} got_output = False output_name = None got_preprocessed_input = False output_merged = None try: opts, args = getopt.getopt(argv, 'hM:m:o:t:P:') except getopt.GetoptError: out('Unknown option') showhelp() sys.exit(2) for opt, arg in opts: if opt == '-h': out('Displaying showhelp') showhelp() sys.exit() elif opt == '-o': if got_output: out('Got more than one output') showhelp() sys.exit(3) got_output = True output_name = arg elif opt == '-m': if module_directory is not None: out('Got more than one cache directory') showhelp() sys.exit(4) module_directory = arg elif opt == '-M': if output_merged is not None: out('Got more than one output merged path') showhelp() sys.exit(5) output_merged = arg elif opt == '-P': assert not got_preprocessed_input assert json_dict == {} with open(arg, 'rb') as dict_file: dictionary = pickle.load(dict_file) got_preprocessed_input = True cleared_dir = False for arg in args: got_input = True with open(arg) as f: srcs = list(yaml.load_all(f.read(), Loader=yaml.FullLoader)) for src in srcs: if isinstance(src, str): assert len(srcs) == 1 template = Template(src, filename=arg, module_directory=module_directory, lookup=TemplateLookup(directories=['.'])) with open(output_name, 'w') as output_file: render_template(template, Context(output_file, **dictionary)) else: # we have optional control data: this template represents # a directory if not cleared_dir: if not os.path.exists(output_name): pass elif os.path.isfile(output_name): os.unlink(output_name) else: shutil.rmtree(output_name, ignore_errors=True) cleared_dir = True items = [] if 'foreach' in src: for el in dictionary[src['foreach']]: if 'cond' in src: args = dict(dictionary) args['selected'] = el if not eval(src['cond'], {}, args): continue items.append(el) assert items else: items = [None] for item in items: args = dict(dictionary) args['selected'] = item item_output_name = os.path.join( output_name, Template(src['output_name']).render(**args)) if not os.path.exists(os.path.dirname(item_output_name)): os.makedirs(os.path.dirname(item_output_name)) template = Template( src['template'], filename=arg, module_directory=module_directory, lookup=TemplateLookup(directories=['.'])) with open(item_output_name, 'w') as output_file: render_template(template, Context(output_file, **args)) if not got_input and not preprocessed_output: out('Got nothing to do') showhelp()
def serialize(self, resource): buf = StringIO() gresource = NepomukResource(resource) ctx = Context(buf, resource=gresource, mapper=self.mapper) self.tmpl.render_context(ctx) fileutil.appendStringToFile(buf.getvalue(), self.outputPath)
def context(self, new_context: dict): """ Replaces current context with new context and refreshes buffer """ self._buffer = StringIO() self._context = Context(self._buffer, **new_context)
#!/usr/bin/env python2 from mako.template import Template from mako.runtime import Context from StringIO import StringIO mytemplate = Template("Hello , ${name}") buf = StringIO() ctx = Context(buf, name="atupal") mytemplate.render_context(ctx) print buf.getvalue()
def main(argv): got_input = False module_directory = None preprocessed_output = None dictionary = {} json_dict = {} got_output = False plugins = [] output_name = None got_preprocessed_input = False output_merged = None try: opts, args = getopt.getopt(argv, 'hM:m:d:o:p:t:P:w:') except getopt.GetoptError: out('Unknown option') showhelp() sys.exit(2) for opt, arg in opts: if opt == '-h': out('Displaying showhelp') showhelp() sys.exit() elif opt == '-o': if got_output: out('Got more than one output') showhelp() sys.exit(3) got_output = True output_name = arg elif opt == '-m': if module_directory is not None: out('Got more than one cache directory') showhelp() sys.exit(4) module_directory = arg elif opt == '-M': if output_merged is not None: out('Got more than one output merged path') showhelp() sys.exit(5) output_merged = arg elif opt == '-P': assert not got_preprocessed_input assert json_dict == {} sys.path.insert( 0, os.path.abspath( os.path.join(os.path.dirname(sys.argv[0]), 'plugins'))) with open(arg, 'rb') as dict_file: dictionary = pickle.load(dict_file) got_preprocessed_input = True elif opt == '-d': assert not got_preprocessed_input with open(arg, 'r') as dict_file: bunch.merge_json( json_dict, yaml.load(dict_file.read(), Loader=yaml.SafeLoader)) elif opt == '-p': plugins.append(import_plugin(arg)) elif opt == '-w': preprocessed_output = arg if not got_preprocessed_input: for plugin in plugins: plugin.mako_plugin(json_dict) if output_merged: with open(output_merged, 'w') as yaml_file: yaml_file.write(yaml.dump(json_dict)) for k, v in json_dict.items(): dictionary[k] = bunch.to_bunch(v) if preprocessed_output: with open(preprocessed_output, 'wb') as dict_file: pickle.dump(dictionary, dict_file) cleared_dir = False for arg in args: got_input = True with open(arg) as f: srcs = list(yaml.load_all(f.read(), Loader=yaml.SafeLoader)) for src in srcs: if isinstance(src, str): assert len(srcs) == 1 template = Template(src, filename=arg, module_directory=module_directory, lookup=TemplateLookup(directories=['.'])) with open(output_name, 'w') as output_file: template.render_context(Context(output_file, **dictionary)) else: # we have optional control data: this template represents # a directory if not cleared_dir: if not os.path.exists(output_name): pass elif os.path.isfile(output_name): os.unlink(output_name) else: shutil.rmtree(output_name, ignore_errors=True) cleared_dir = True items = [] if 'foreach' in src: for el in dictionary[src['foreach']]: if 'cond' in src: args = dict(dictionary) args['selected'] = el if not eval(src['cond'], {}, args): continue items.append(el) assert items else: items = [None] for item in items: args = dict(dictionary) args['selected'] = item item_output_name = os.path.join( output_name, Template(src['output_name']).render(**args)) if not os.path.exists(os.path.dirname(item_output_name)): os.makedirs(os.path.dirname(item_output_name)) template = Template( src['template'], filename=arg, module_directory=module_directory, lookup=TemplateLookup(directories=['.'])) with open(item_output_name, 'w') as output_file: template.render_context(Context(output_file, **args)) if not got_input and not preprocessed_output: out('Got nothing to do') showhelp()
def main(): successful = False err = None global iniFilePath # obtain any command-line arguments # overriding any values set so far nextArg = "" for argv in sys.argv: if nextArg != "": if nextArg == "iniFilePath": iniFilePath = argv nextArg = "" else: if argv.lower() == "--inifilepath" or argv.lower() == "-inifilepath": nextArg = "iniFilePath" # expand any leading tilde # to the user's home path if iniFilePath.startswith("~"): iniFilePath = os.path.expanduser(iniFilePath) iniFilePath = os.path.abspath(iniFilePath) print ("Attempting to load INI file: %s" % iniFilePath) # if INI file path does not exist if not os.path.exists(iniFilePath): # output error message sys.stderr.write('iniFilePath does not exist: "%s"\n' % iniFilePath) # cease further processing sys.exit(0) # obtain the settings # from the INI file path config = configparser.ConfigParser(interpolation=ExtendedInterpolation(), delimiters=('=')) config.optionxform = str #this will preserve the case of the section names config.read(iniFilePath) # ----------------------------- # Logging settings # ----------------------------- logFilePathExpanded = 'PyDqsStatsGen.log' # if the log file # starts with a tilde (~) if logFilePathExpanded.startswith("~"): # derive the log file name's expanded path logFilePathExpanded = os.path.expanduser(logFilePathExpanded) # if the expanded log file name contains a folder prefix if os.path.dirname(logFilePathExpanded) != '': # if the expanded log file's parent folder does not yet exist if not os.path.exists(os.path.dirname(logFilePathExpanded)): try: # create the log file's parent folder os.makedirs(os.path.dirname(logFilePathExpanded)) except Exception as e: logging.error(str(e)) # if the specified log file exists if os.path.exists(logFilePathExpanded): # delete it os.remove(logFilePathExpanded) # maximum logging level that # will output to the STDOUT stream MAX_STDOUT_LEVEL = logging.INFO # obtain the [folders] section's settings from the INI file outFolder = config['folders'].get('outFolder', '~/temp') logSubFolder = config['folders'].get('logSubFolder', 'logFiles') tgtSubFolder = config['folders'].get('tgtSubFolder', 'tgtFiles') tgtFullPath = os.path.join(outFolder, tgtSubFolder) # obtain the [logging] section's settings from the INI file logFileName = config['logging'].get('logFileName', 'PyDqsStatsGen.log') maxStdOutLvl = config['logging'].get('MAX_STDOUT_LEVEL', 'info') # if log file name # was not specified if logFileName == "": # default the log file name logFileName = "PyDqsStatsGen.log" logFullPathExpanded = logFileName # if the log file # starts with a tilde (~) if logFullPathExpanded.startswith("~"): # derive the log file name's expanded path logFullPathExpanded = os.path.expanduser(logFullPathExpanded) # if the expanded log file name does NOT contain a folder prefix if os.path.dirname(logFullPathExpanded) == '': # expand the output folder joined to the default log subfolder joined to the log file name logFullPathExpanded = os.path.expanduser(os.path.join(outFolder, logSubFolder, logFullPathExpanded)) # if the expanded log file name contains a folder prefix if os.path.dirname(logFullPathExpanded) != '': # if the expanded log file's parent folder does not yet exist if not os.path.exists(os.path.dirname(logFullPathExpanded)): try: # create the log file's parent folder os.makedirs(os.path.dirname(logFullPathExpanded)) except Exception as e: logging.error(str(e)) return successful, err # if the specified log file exists if os.path.exists(logFullPathExpanded): # delete it os.remove(logFullPathExpanded) if maxStdOutLvl.lower() == 'info': MAX_STDOUT_LEVEL = logging.INFO elif maxStdOutLvl.lower() == 'debug': MAX_STDOUT_LEVEL = logging.DEBUG elif maxStdOutLvl.lower() == 'warning': MAX_STDOUT_LEVEL = logging.WARNING elif maxStdOutLvl.lower() == 'error': MAX_STDOUT_LEVEL = logging.ERROR elif maxStdOutLvl.lower() == 'critical': MAX_STDOUT_LEVEL = logging.CRITICAL else: MAX_STDOUT_LEVEL = logging.INFO # instantiate the logger object # logger = logging.getLogger(__name__) # remove any existing log handlers logging.getLogger('').handlers = [] # set the default logger's values logging.basicConfig(level=MAX_STDOUT_LEVEL, format='%(asctime)s\t%(levelname)s\t%(name)s\t%(message)s', datefmt='%Y-%m-%d %H:%M', filename=logFullPathExpanded, filemode='w') # attach stdout to the logger # so that outputing to the log also # outputs to the stdout console logStdOut = logging.StreamHandler(sys.stdout) logStdOut.addFilter(MaxLogLevelFilter(MAX_STDOUT_LEVEL)) logging.getLogger('').addHandler(logStdOut) # attach stderr to the logger # so that outputing to the log also # outputs to the stderr console logStdErr = logging.StreamHandler(sys.stderr) logStdErr.addFilter(MinLogLevelFilter(MAX_STDOUT_LEVEL)) logging.getLogger('').addHandler(logStdErr) # output a message to the log file # with the log file's location info logging.info('Log file: %s', logFullPathExpanded) # logging.debug('DEBUG: Log file located at %s', logFilePathExpanded) # logging.warning('WARNING: Log file located at %s', logFilePathExpanded) # logging.error('Log file located at %s', logFilePathExpanded) # ------------------------------------------------------------------------- # max data rows # to be processed # 0 means unlimited maxRows = int(config['DEFAULT'].get('maxRows', '0')) # show progress messages # every 'flushCount' number # of data rows flushCount = int(config['DEFAULT'].get('flushCount', '10000')) # maximum allowed column # count mismatches before # terminating the program maxColCountMisMatches = int(config['DEFAULT'].get('maxColCountMisMatches', '0')) # max number of HTML value # frequencies per column to report maxHtmlCount = int(config['DEFAULT'].get('maxHtmlCount', '5')) # max number of JDBC value # frequencies per column to report maxJdbcCount = int(config['DEFAULT'].get('maxJdbcCount', '10')) # the date of execution's output format string runDateFormatString = config['DEFAULT'].get('runDateFormatString', '%A %d %b %Y %I:%M %p %Z') # MAKO template path # for HTML output generation makoHtmlTemplateName = config['DEFAULT'].get('makoHtmlTemplateName', 'DqsStatsHtml.mako') # MAKO template path # for JDBC output generation makoJdbcTemplateName = config['DEFAULT'].get('makoJdbcTemplateName', 'DqsStatsJdbc.mako') srcFullPath = config['srcSpecs'].get('srcFullPath') srcDelim = config['srcSpecs'].get('srcDelim', ',') srcHeaderRows = int(config['srcSpecs'].get('srcHeaderRows', '1')) # handle edge-values # of source header rows if srcHeaderRows <= 0: srcHeaderRows = 1; # assume the source file # is minimally-quoted srcQuote = csv.QUOTE_MINIMAL # data provider's acronym dataProvider = config['srcSpecs'].get('dataProvider', 'unspecified') # comma-delimited list of columns upon # which statistics are to be calculated, # an empty ACCEPT list will signal the # processing of ALL of the row's columns acceptColNames = {} acceptColNamesStr = config['srcSpecs'].get('acceptColNames', '') if acceptColNamesStr != '': tempValues = acceptColNamesStr.split(',') for tempValue in tempValues: acceptColNames[tempValue] = tempValue # comma-delimited IGNORE list will suppress calculations # of the value frequency statistics for the specified columns # ignoreColNames = 'voter_reg_num','ncid' ignoreColNames = {} ignoreColNamesStr = config['srcSpecs'].get('ignoreColNames','') if ignoreColNamesStr != '': tempValues = ignoreColNamesStr.split(',') for tempValue in tempValues: ignoreColNames[tempValue] = tempValue # comma-delimited UNIQUE list will suppress calculations # of the value frequency statistics for the specified columns # uniqueColNames = 'voter_reg_num','ncid' uniqueColNames = {} uniqueColNamesStr = config['srcSpecs'].get('uniqueColNames','') if uniqueColNamesStr != '': tempValues = uniqueColNamesStr.split(',') for tempValue in tempValues: uniqueColNames[tempValue] = tempValue bypassColNames = {} for colName in uniqueColNames.keys(): if colName not in bypassColNames: bypassColNames[colName] = colName for colName in ignoreColNames.keys(): if colName not in bypassColNames: bypassColNames[colName] = colName # obtain the JDBC database connection parameters jdbcType = config['jdbcSpecs'].get('jdbcType', 'pgsql').lower() jdbcHost = config['jdbcSpecs'].get('jdbcHost', 'localhost') jdbcPort = int(config['jdbcSpecs'].get('jdbcPort', '5432')) # defaults to PostgreSQL's port jdbcDatabase = config['jdbcSpecs'].get('jdbcDatabase', 'dqsvalidator') jdbcUID = config['jdbcSpecs'].get('jdbcUID', 'dqsvalidator') jdbcPWD = config['jdbcSpecs'].get('jdbcPWD', '[redacted]') jdbcDropTableIfExistsCompliant = (config['jdbcSpecs'].get('jdbcDropTableIfExistsCompliant', 'True') == 'True') # tweak the connection parameters # depending upon the target database # is it MySQL if jdbcType == 'mysql': jdbcParms = { 'host':jdbcHost, 'port': jdbcPort, 'database':jdbcDatabase, 'user':jdbcUID, 'password':jdbcPWD } # is it SQL Server? elif jdbcType == 'mssql': jdbcParms = { 'server':jdbcHost + ":" + str(jdbcPort), 'port': jdbcPort, 'database':jdbcDatabase, 'user':jdbcUID, 'password':jdbcPWD } # otherwise else: # default to PostgreSQL jdbcParms = { 'host':jdbcHost, 'port': jdbcPort, 'dbname':jdbcDatabase, 'user':jdbcUID, 'password':jdbcPWD } executorName = getpass.getuser() runDate = time.strftime(runDateFormatString) srcPathExpanded = srcFullPath if srcFullPath.startswith('~'): srcPathExpanded = os.path.expanduser(srcFullPath) srcPathExpanded = os.path.abspath(srcPathExpanded) logging.info("SRC file: %s" % srcPathExpanded) if not os.path.exists(srcPathExpanded): logging.error("SRC file does NOT exist: %s" % srcPathExpanded) successful = False return successful, err tgtPathExpanded = tgtFullPath if tgtFullPath.startswith('~'): tgtPathExpanded = os.path.expanduser(tgtFullPath) tgtDqsStatsHtmlExpanded = os.path.join(tgtPathExpanded, os.path.splitext(os.path.basename(srcPathExpanded))[0] + ".html") tgtDqsStatsJdbcExpanded = os.path.join(tgtPathExpanded, os.path.splitext(os.path.basename(srcPathExpanded))[0] + ".sqlite") tgtDqsStatsHtmlExpanded = os.path.abspath(tgtDqsStatsHtmlExpanded) tgtDqsStatsJdbcExpanded = os.path.abspath(tgtDqsStatsJdbcExpanded) logging.info("TGT DQS Statistics HTML file: %s" % tgtDqsStatsHtmlExpanded) logging.info("TGT DQS Statistics JDBC file: %s" % tgtDqsStatsJdbcExpanded) if not os.path.exists(os.path.dirname(tgtDqsStatsHtmlExpanded)): os.makedirs(os.path.dirname(tgtDqsStatsHtmlExpanded)) if not os.path.exists(os.path.dirname(tgtDqsStatsJdbcExpanded)): os.makedirs(os.path.dirname(tgtDqsStatsJdbcExpanded)) # remove any pre-existing statistics files if os.path.exists(tgtDqsStatsHtmlExpanded): try: os.remove(tgtDqsStatsHtmlExpanded) except Exception as e: logging.error('Failed to remove pre-existing statistics file "%s"' % tgtDqsStatsHtmlExpanded) logging.error('Processing terminated immediately.') err = str(e) return successful,err # remove any pre-existing statistics files if os.path.exists(tgtDqsStatsJdbcExpanded): try: os.remove(tgtDqsStatsJdbcExpanded) except Exception as e: logging.error('Failed to remove pre-existing statistics file "%s"' % tgtDqsStatsJdbcExpanded) logging.error('Processing terminated immediately.') err = str(e) return successful,err makoHtmlPathExpanded = makoHtmlTemplateName if makoHtmlPathExpanded.startswith('~'): makoHtmlPathExpanded = os.path.expanduser(makoHtmlPathExpanded) makoHtmlPathExpanded = os.path.abspath(makoHtmlPathExpanded) logging.info("DQS Statistics HTML MAKO template file: %s" % makoHtmlPathExpanded) if not os.path.exists(makoHtmlPathExpanded): logging.error("MAKO template file for HTML output does NOT exist: %s" % makoHtmlPathExpanded) successful = False return successful, err makoJdbcPathExpanded = makoJdbcTemplateName if makoJdbcPathExpanded.startswith('~'): makoJdbcPathExpanded = os.path.expanduser(makoJdbcPathExpanded) makoJdbcPathExpanded = os.path.abspath(makoJdbcPathExpanded) logging.info("DQS Statistics JDBC MAKO template file: %s" % makoJdbcPathExpanded) if not os.path.exists(makoJdbcPathExpanded): logging.error("MAKO template file for JDBC output does NOT exist: %s" % makoJdbcPathExpanded) successful = False return successful, err colNames = [] logging.info('Accept columns: %s' % list(set(acceptColNames.keys()))) logging.info('Unique columns: %s' % list(set(uniqueColNames.keys()))) logging.info('Ignore columns: %s' % list(set(ignoreColNames.keys()))) # derive the columns for which NO value frequencies are to be calculated # bypassColNames = list(set(uniqueColNames)|set(ignoreColNames)) logging.info("Bypass value frequency processing for columns: %s" % list(set(bypassColNames))) # open the source file for reading srcFile = codecs.open(srcPathExpanded, 'r', 'cp1252') csvReader = csv.reader(srcFile, delimiter=srcDelim, quoting=srcQuote) bgnTime = time.time() fileRows = 0 dataRows = 0 for rowData in csvReader: fileRows += 1 # if this is the last # of the header rows if fileRows == srcHeaderRows: colNames, err = analyzeHead(rowData, colNames, acceptColNames, ignoreColNames, uniqueColNames) if err: # cease further processing logging.error("Processing terminated due to incorrect 'acceptColNames', 'ignoreColNames', or 'uniqueColNames' INI file settings.") break # otherwise, if this is a data row elif fileRows > srcHeaderRows: dataRows += 1 analyzeData(rowData, colNames, acceptColNames, bypassColNames, fileRows, dataRows) if maxRows > 0 and dataRows >= maxRows: break if dataRows > 0 and dataRows % flushCount == 0: endTime = time.time() seconds = endTime - bgnTime if seconds > 0: rcdsPerSec = dataRows / seconds else: rcdsPerSec = 0 logging.info("Read {:,} data rows in {:,.0f} seconds @ {:,.0f} records/second".format(dataRows, seconds, rcdsPerSec)) # if maximum column count mismatches value exceeded if maxColCountMisMatches > 0 and len(colCountMisMatches) >= maxColCountMisMatches: # cease further processing logging.error("Processing terminated due to the number of column count mismatches %d exceeding maximum allowed %d." % (colCountMisMatches, maxColCountMisMatches)) break del csvReader srcFile.close() endTime = time.time() seconds = endTime - bgnTime if seconds > 0: rcdsPerSec = dataRows / seconds else: rcdsPerSec = 0 logging.info('') logging.info("Read {:,} data rows in {:,.0f} seconds @ {:,.0f} records/second".format(dataRows, seconds, rcdsPerSec)) # if error found if err: # bypass further processing return successful, err # column-by-column for colName in colNames: # if there were # row of data found if dataRows > 0: # calculate the average width avgWidths[colName] = (totWidths[colName] * 1.0) / (dataRows * 1.0) # calculate the coverage percent cvgPrcnts[colName] = (nonBlanks[colName] * 1.0) / (dataRows * 1.0) else: avgWidths[colName] = 0.0 cvgPrcnts[colName] = 0.0 # column-by-column sort the value frequencies frqValueAscs = collections.OrderedDict() for colName in colNames: frqValueAscs[colName] = {} # bypass columns with unprocessed columns # since no value frequencies were tracked for them if not colName in bypassColNames: frqValueAscs[colName] = sorted(frqValues[colName].items(), key=lambda x:x[0]) # column-by-column, sort the width frequencies frqWidthAscs = collections.OrderedDict() for colName in colNames: frqWidthAscs[colName] = sorted(frqWidths[colName].items(), key=lambda x:x[0]) # ------------------------------------------------------------------------- # Output DQS statistics to HTML file # ------------------------------------------------------------------------- valueFreqs = collections.OrderedDict() for colName in colNames: valueFreqs[colName] = {} valueFreqs[colName]['frqValValAsc'] = {} valueFreqs[colName]['frqValFrqAsc'] = {} valueFreqs[colName]['frqValFrqDsc'] = {} if not colName in bypassColNames: if maxHtmlCount > 0: valueFreqs[colName]['frqValValAsc'] = heapq.nsmallest(maxHtmlCount, frqValues[colName].items(), key=lambda x:x[0]) valueFreqs[colName]['frqValFrqAsc'] = heapq.nsmallest(maxHtmlCount, frqValues[colName].items(), key=lambda x:x[1]) valueFreqs[colName]['frqValFrqDsc'] = heapq.nlargest(maxHtmlCount, frqValues[colName].items(), key=lambda x:x[1]) else: valueFreqs[colName]['frqValValAsc'] = sorted(frqValues[colName].items(), key=lambda x:x[0]) valueFreqs[colName]['frqValFrqAsc'] = sorted(frqValues[colName].items(), key=lambda x:x[1]) valueFreqs[colName]['frqValFrqDsc'] = sorted(frqValues[colName].items(), key=lambda x:x[1], reverse=True) htmlWriter = codecs.open(tgtDqsStatsHtmlExpanded, 'w', 'cp1252') makoHtmlTemplate = Template(filename=makoHtmlPathExpanded) buffer = StringIO() attrs = {} parms = { 'attrs':attrs, 'dataProvider': dataProvider, 'executorName': executorName, 'runDate': runDate, 'srcPathExpanded':srcPathExpanded, 'srcPathBaseName':os.path.basename(srcPathExpanded), 'srcDelim':srcDelim, 'srcHeaderRows':srcHeaderRows, 'maxRows':maxRows, 'maxHtmlCount':maxHtmlCount, 'maxJdbcCount':maxJdbcCount, 'tgtDqsStatsJdbcExpanded':tgtDqsStatsJdbcExpanded, 'inputRows':dataRows, 'inputCols':len(colNames), 'colNames':colNames, 'acceptColNames':acceptColNames, 'ignoreColNames':ignoreColNames, 'uniqueColNames':uniqueColNames, 'nonBlanks':nonBlanks, 'valueFreqs':valueFreqs, 'minWidths':minWidths, 'maxWidths':maxWidths, 'avgWidths':avgWidths, 'frqValueAscs':frqValueAscs, 'frqWidthAscs':frqWidthAscs, 'colCountMisMatches':colCountMisMatches } context = Context(buffer, **parms) makoHtmlTemplate.render_context(context) htmlWriter.write(buffer.getvalue()) htmlWriter.close() # ------------------------------------------------------------------------- # Output DQS statistics to JDBC file (SQLite) # ------------------------------------------------------------------------- valueFreqs.clear() for colName in colNames: valueFreqs[colName] = {} valueFreqs[colName]['frqValValAsc'] = {} valueFreqs[colName]['frqValFrqAsc'] = {} valueFreqs[colName]['frqValFrqDsc'] = {} # don't sort unprocessed columns as no # value frequencies were calculated for them if not colName in bypassColNames: if maxJdbcCount > 0: valueFreqs[colName]['frqValValAsc'] = heapq.nsmallest(maxJdbcCount, frqValues[colName].items(), key=lambda x:x[0]) valueFreqs[colName]['frqValFrqAsc'] = heapq.nsmallest(maxJdbcCount, frqValues[colName].items(), key=lambda x:x[1]) valueFreqs[colName]['frqValFrqDsc'] = heapq.nlargest(maxJdbcCount, frqValues[colName].items(), key=lambda x:x[1]) else: valueFreqs[colName]['frqValValAsc'] = sorted(frqValues[colName].items(), key=lambda x:x[0]) valueFreqs[colName]['frqValFrqAsc'] = sorted(frqValues[colName].items(), key=lambda x:x[1]) valueFreqs[colName]['frqValFrqDsc'] = sorted(frqValues[colName].items(), key=lambda x:x[1], reverse=True) # TODO: implement SQL as parameterized queries, necessitating NOT using MAKO template # push statistic records to SQLite database sqliteConn = sqlite3.connect(tgtDqsStatsJdbcExpanded) sqliteCursor = sqliteConn.cursor() makoJdbcTemplate = Template(filename=makoJdbcPathExpanded) buffer = StringIO() attrs = {} parms = { 'attrs':attrs, 'dataProvider': dataProvider, 'executorName': executorName, 'runDate': runDate, 'srcPathExpanded':srcPathExpanded, 'srcPathBaseName':os.path.basename(srcPathExpanded), 'srcDelim':srcDelim, 'srcHeaderRows':srcHeaderRows, 'maxRows':maxRows, 'maxHtmlCount':maxHtmlCount, 'maxJdbcCount':maxJdbcCount, 'tgtDqsStatsHtmlExpanded':tgtDqsStatsHtmlExpanded, 'tgtDqsStatsJdbcExpanded':tgtDqsStatsJdbcExpanded, 'inputRows':dataRows, 'inputCols':len(colNames), 'colNames':colNames, 'acceptColNames':acceptColNames, 'ignoreColNames':ignoreColNames, 'uniqueColNames':uniqueColNames, 'nonBlanks':nonBlanks, 'valueFreqs':valueFreqs, 'minWidths':minWidths, 'maxWidths':maxWidths, 'avgWidths':avgWidths, 'frqValueAscs':frqValueAscs, 'frqWidthAscs':frqWidthAscs, 'colCountMisMatches':colCountMisMatches, 'jdbcDropTableIfExistsCompliant': True # True for SQLite databases } context = Context(buffer, **parms) makoJdbcTemplate.render_context(context) sqlCmd = '' lines = buffer.getvalue().split(os.linesep) for line in lines: sqlCmd = sqlCmd + line.strip() if line.strip().endswith(';'): # print (sqlCmd) # print ('') try: sqliteCursor.execute(sqlCmd) except Exception as e: logging.error(sqlCmd) logging.error(str(e)) break; sqlCmd = '' sqliteConn.commit() sqliteConn.close() # push statistic records to traditional database # if jdbcType not found # default to PostgreSQL jdbcConn = None if jdbcType.lower() == 'mysql': try: jdbcConn = mysql.connector.connect(**jdbcParms) except Exception as e: logging.error('Failed to connect to MySQL database') logging.error(jdbcParms) logging.error(str(e)) elif jdbcType.lower() == 'mssql': try: jdbcConn = pymssql.connect(**jdbcParms) except Exception as e: logging.error('Failed to connect to MySQL database') logging.error(jdbcParms) logging.error(str(e)) else: try: jdbcConn = psycopg2.connect(**jdbcParms) except Exception as e: logging.error('Failed to connect to MySQL database') logging.error(jdbcParms) logging.error(str(e)) jdbcCursor = jdbcConn.cursor() makoJdbcTemplate = Template(filename=makoJdbcPathExpanded) buffer = StringIO() attrs = {} parms = { 'attrs':attrs, 'dataProvider': dataProvider, 'executorName': executorName, 'runDate': runDate, 'srcPathExpanded':srcPathExpanded, 'srcPathBaseName':os.path.basename(srcPathExpanded), 'srcDelim':srcDelim, 'srcHeaderRows':srcHeaderRows, 'maxRows':maxRows, 'maxHtmlCount':maxHtmlCount, 'maxJdbcCount':maxJdbcCount, 'tgtDqsStatsHtmlExpanded':tgtDqsStatsHtmlExpanded, 'tgtDqsStatsJdbcExpanded':tgtDqsStatsJdbcExpanded, 'inputRows':dataRows, 'inputCols':len(colNames), 'colNames':colNames, 'acceptColNames':acceptColNames, 'ignoreColNames':ignoreColNames, 'uniqueColNames':uniqueColNames, 'nonBlanks':nonBlanks, 'valueFreqs':valueFreqs, 'minWidths':minWidths, 'maxWidths':maxWidths, 'avgWidths':avgWidths, 'frqValueAscs':frqValueAscs, 'frqWidthAscs':frqWidthAscs, 'colCountMisMatches':colCountMisMatches, 'jdbcDropTableIfExistsCompliant': jdbcDropTableIfExistsCompliant # True for PostgreSQL and MySQL databases, False for Sql Server } context = Context(buffer, **parms) makoJdbcTemplate.render_context(context) sqlCmd = '' lines = buffer.getvalue().split(os.linesep) for line in lines: sqlCmd = sqlCmd + line.strip() if line.strip().endswith(';'): # print (sqlCmd) # print ('') try: jdbcCursor.execute(sqlCmd) except Exception as e: logging.error(sqlCmd) logging.error(str(e)) break; sqlCmd = '' jdbcConn.commit() jdbcConn.close() return
def main(argv): got_input = False module_directory = None dictionary = {} json_dict = {} got_output = False output_file = sys.stdout plugins = [] try: opts, args = getopt.getopt(argv, 'hm:d:o:p:') except getopt.GetoptError: out('Unknown option') showhelp() sys.exit(2) for opt, arg in opts: if opt == '-h': out('Displaying showhelp') showhelp() sys.exit() elif opt == '-o': if got_output: out('Got more than one output') showhelp() sys.exit(3) got_output = True output_file = open(arg, 'w') elif opt == '-m': if module_directory is not None: out('Got more than one cache directory') showhelp() sys.exit(4) module_directory = arg elif opt == '-d': dict_file = open(arg, 'r') bunch.merge_json(json_dict, simplejson.loads(dict_file.read())) dict_file.close() elif opt == '-p': plugins.append(import_plugin(arg)) for plugin in plugins: plugin.mako_plugin(json_dict) for k, v in json_dict.items(): dictionary[k] = bunch.to_bunch(v) ctx = Context(output_file, **dictionary) for arg in args: got_input = True template = Template(filename=arg, module_directory=module_directory, lookup=TemplateLookup(directories=['.'])) template.render_context(ctx) if not got_input: out('Got nothing to do') showhelp() output_file.close()
def _inject_stuff(self): try: with open(self._input_file, 'r') as infile: for thing in infile: key, val = self._prop_to_yaml(thing.strip()) if key and val: self._food += spacer + key + ': ' + val + '\n' buf = StringIO() t = Template(filename=self._template_file) if self._sns_topic_arn_found: sns_var_bits = sns_topic_arn sns_resource_bits = sns_subcription_resource else: sns_var_bits = '' sns_resource_bits = '' if self._trusted_service_found: trusted_service_var_bits = trusted_service trusted_service_resource_bits = trusted_service_resource else: trusted_service_var_bits = '' trusted_service_resource_bits = '' if self._schedule_found: schedule_var_bits = schedule_expression schedule_resource_bits = schedule_resource else: schedule_var_bits = '' schedule_resource_bits = '' if self._create_service: the_api_bits = get_the_api_chunk(region=self._region, stage_name=self._stage_name, short_name=self._short_name, account=self._account) else: the_api_bits = '' if self._import_role: current_role_parameter_section = '' role = self._find_imported_csv( self._stack_properties.get('role', None)) role_specification = imported_role_spec.format(role) else: current_role_parameter_section = role_parameter_section role_specification = parameter_role_spec subnet_specification = None if self._import_subnets: current_subnets_parameter_section = '' subnets = self._find_imported_csv( self._stack_properties.get('subnetIds', None)) for subnet in subnets.split(','): if subnet_specification: subnet_specification = subnet_specification + \ '\n' + spacer + \ imported_subnets_spec.format(subnet) else: subnet_specification = imported_subnets_spec.format( subnet) else: current_subnets_parameter_section = subnets_parameter_section subnet_specification = subnets_parameter_spec sg_specification = None if self._import_security_group: current_sg_parameter_section = '' sg_csv = self._find_imported_csv( self._stack_properties.get('securityGroupIds', None)) for sg in sg_csv.split(','): if sg_specification: sg_specification = sg_specification + \ '\n' + spacer + \ imported_sg_spec.format(sg) else: sg_specification = imported_sg_spec.format(sg) else: current_sg_parameter_section = sg_parameter_section sg_specification = sg_parameter_spec ctx = Context( buf, environment_section=self._food, snsTopicARN=sns_var_bits, snsSubscriptionResource=sns_resource_bits, trustedService=trusted_service_var_bits, trustedServiceResource=trusted_service_resource_bits, scheduleExpression=schedule_var_bits, scheduleResource=schedule_resource_bits, theAPI=the_api_bits, roleParameterSection=current_role_parameter_section, roleSpecification=role_specification, subnetsParameterSection=current_subnets_parameter_section, subnetIds=subnet_specification, sgParameterSection=current_sg_parameter_section, securityGroupIds=sg_specification) # securityGroupIds=sg_parameter_spec t.render_context(ctx) logging.info('writing template {}'.format(self._output_file)) with open(self._output_file, "w") as outfile: outfile.write(buf.getvalue()) except Exception as wtf: logging.error('Exception caught in inject_stuff(): {}'.format(wtf)) traceback.print_exc(file=sys.stdout) sys.exit(1)
def render(tmpl, dict1): buf = StringIO() ctx = Context(buf, **dict1) tmpl.render_context(ctx) return buf.getvalue()
def create_citation_command_from_footnote_list(footnote_list): print("Getting information for footnote with information " + str(footnote_list)) result = "EMPTY-CITATION-FIX-ME" if output_lyx is True: print("Outputting LyX citation command") mytemplate = Template( filename='lyx_template/lyx_citation_command_template.tmpl') buf = io.StringIO() # create context to impose on the template in the current iteration key_list = [] if footnote_list is not None: for entry in footnote_list: if 'uri' in entry.keys(): key_list.append( get_biblatex_cite_key_from_zotero_api(entry['uri'])) print(','.join(key_list)) pretext_list = [] if footnote_list is not None: for entry in footnote_list: if 'prefix' in entry.keys(): pretext_list.append( get_biblatex_cite_key_from_zotero_api(entry['uri'])) pretext_list.append(' ') pretext_list.append(entry['prefix'].replace("\"", "\\\"")) pretext_list.append('\t') posttext_list = [] if footnote_list is not None: for entry in footnote_list: if 'locator' in entry.keys(): posttext_list.append( get_biblatex_cite_key_from_zotero_api(entry['uri'])) posttext_list.append(' ') posttext_list.append(entry['locator'].replace( "\"", "\\\"")) posttext_list.append('\t') ctx = Context(buf, key_list=','.join(key_list), pretext_list=''.join(pretext_list), posttext_list=''.join(posttext_list)) # render template with context we've just generated mytemplate.render_context(ctx) result = buf.getvalue() print(result) else: result = "EMPTY-CITATION-FIX-ME" if footnote_list is not None: result = "\\autocites" for entry in footnote_list: if 'prefix' in entry.keys(): result += ("[" + entry['prefix'] + "]") else: result += "[]" if 'locator' in entry.keys(): result += ("[" + entry['locator'] + "]") else: result += "[]" if 'uri' in entry.keys(): result += ("{" + get_biblatex_cite_key_from_zotero_api( entry['uri']) + "}") return result
def main(): tree = etree.parse(os.path.join(document_temp_path, "word", "document.xml")) root = tree.getroot() document_content = [] for child in root[0]: new_list_object = List_object() new_list_object.text = "\n\n" document_content.append(new_list_object) for child in child: for child in child: if etree.QName(child.tag).localname is 't': if child.text is not None: new_list_object = List_object() new_list_object.footnote = False if child.get( '{http://www.w3.org/XML/1998/namespace}space' ) is not None: new_list_object.text = ' ' + child.text + ' ' else: new_list_object.text = ( child.text).lstrip(' ').rstrip(' ') document_content.append(new_list_object) else: if child.get( '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}id' ) is not None: new_list_object = List_object() new_list_object.footnote = True new_list_object.id = str( child.get( '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}id' )) new_list_object.footnote_information = get_footnote_information( new_list_object.id) document_content.append(new_list_object) new_list_object = List_object() new_list_object.text = '\n' document_content.append(new_list_object) if output_lyx is True: mytemplate = Template(filename='lyx_template/lyx_template.tmpl') buf = io.StringIO() content_list = [] content_list.append("\\begin_layout Standard\n") for entry in document_content: try: if entry.text == "\n\n": content_list.append("\\end_layout\n") content_list.append("\\begin_layout Standard\n") except AttributeError: pass if entry.footnote is True: if FOOTNOTE_WRITING is True: content_list.append( str( create_citation_command_from_footnote_list( entry.footnote_information))) else: content_list.append( "\n\\color red\nDEBUG: Writing footnotes disabled\n\\color inherit\n" ) else: current_entry = str(entry) current_entry = current_entry.replace( '“', """\\begin_inset Quotes eld \\end_inset """) current_entry = current_entry.replace( '”', """\\begin_inset Quotes erd \\end_inset """) content_list.append(current_entry) content_list.append("\\end_layout\n") ctx = Context(buf, document_content=''.join(content_list)) # render template with context we've just generated mytemplate.render_context(ctx) output_file = open("output.lyx", 'w') # write the string buffer to the file output_file.write(buf.getvalue()) # close file handle output_file.close() else: output_file = open("output.tex", 'w') for entry in document_content: if entry.footnote is True: if FOOTNOTE_WRITING is True: output_file.write( str( create_citation_command_from_footnote_list( entry.footnote_information))) else: output_file.write( "\n\\textcolor{red}{DEBUG: Writing footnotes disabled}\n" ) else: current_entry = str(entry) current_entry = current_entry.replace("&", "\\&") current_entry = current_entry.replace("%", "\\%") current_entry = current_entry.replace("$", "\\$") current_entry = current_entry.replace("“", "``") current_entry = current_entry.replace("„", "``") current_entry = current_entry.replace("”", "\'\'") output_file.write(current_entry) output_file.close()
else: ntop.printHTMLHeader('Host Map: Region View', 1, 0) if totalHosts == 0: ntop.printFlagedWarning('No hosts have been detected by ntop yet') elif len(dictionaryCountries) == 0: ntop.printFlagedWarning( 'No hosts have been successfully geo-located by ntop yet') else: try: basedir = os.getenv('DOCUMENT_ROOT', '.') + '/python/templates' mylookup = TemplateLookup(directories=[basedir], output_encoding='utf-8', input_encoding='latin1', encoding_errors='replace', default_filters=['decode.utf8']) myTemplate = mylookup.get_template('GeoPacketVisualizer.tmpl') buf = StringIO() ctx = Context(buf, countries=dictionaryCountries, totalHosts=totalHosts, unknownCountries=unknownCountries, unknownCities=unknownCities, filename=os.path.basename(__file__)) myTemplate.render_context(ctx) ntop.sendString(buf.getvalue()) except: ntop.sendString(exceptions.html_error_template().render()) ntop.printHTMLFooter()
def generator(self): self._buf = StringIO() ctx = Context(self._buf, java_class=self._java_class) self._template.render_context(ctx) pass
def render_interop_html_report(client_langs, server_langs, test_cases, auth_test_cases, http2_cases, http2_badserver_cases, client_langs_http2_badserver_cases, resultset, num_failures, cloud_to_prod, prod_servers, http2_interop): """Generate HTML report for interop tests.""" template_file = 'tools/run_tests/interop/interop_html_report.template' try: mytemplate = Template(filename=template_file, format_exceptions=True) except NameError: print( 'Mako template is not installed. Skipping HTML report generation.') return except IOError as e: print('Failed to find the template %s: %s' % (template_file, e)) return sorted_test_cases = sorted(test_cases) sorted_auth_test_cases = sorted(auth_test_cases) sorted_http2_cases = sorted(http2_cases) sorted_http2_badserver_cases = sorted(http2_badserver_cases) sorted_client_langs_http2_badserver_cases = sorted( client_langs_http2_badserver_cases) sorted_client_langs = sorted(client_langs) sorted_server_langs = sorted(server_langs) sorted_prod_servers = sorted(prod_servers) args = { 'client_langs': sorted_client_langs, 'server_langs': sorted_server_langs, 'test_cases': sorted_test_cases, 'auth_test_cases': sorted_auth_test_cases, 'http2_cases': sorted_http2_cases, 'http2_badserver_cases': sorted_http2_badserver_cases, 'client_langs_http2_badserver_cases': (sorted_client_langs_http2_badserver_cases), 'resultset': resultset, 'num_failures': num_failures, 'cloud_to_prod': cloud_to_prod, 'prod_servers': sorted_prod_servers, 'http2_interop': http2_interop } html_report_out_dir = 'reports' if not os.path.exists(html_report_out_dir): os.mkdir(html_report_out_dir) html_file_path = os.path.join(html_report_out_dir, 'index.html') try: with open(html_file_path, 'w') as output_file: mytemplate.render_context(Context(output_file, **args)) except: print(exceptions.text_error_template().render()) raise
#BlogPublished must execute before Published is reassigned BlogPublished = Published.strftime("%m%d") Published = Published.strftime("%B %d, %Y") #Parsing Artwork ArtHomePage = Art[:-3] + "html" #Rendering the HTML frame StoryHTMLFrame = Template(filename='0000-Mako-Template.html') buf = StringIO() ctx = Context(buf, Headline=Headline, Subject=Subject, Subtopics=Subtopics, Topic=Topic, Type=Type, Published=Published, ArtHomePage=ArtHomePage, Art=Art, Link=Link, Description=Description) StoryHTMLFrame.render_context(ctx) f = open("Delivery/" + FileName + ".html", "w") f.write(buf.getvalue()) f.close() #vestigial Function #Rendering the Intro text for the blog #First, rework the filename to include the day instead:
default_filters=['decode.utf8'], encoding_errors='replace') #buffer = codecs.open(fileName, 'w', encoding='utf-8') buffer = open(fileName, 'w') BITFIELDs = '' for i in reversed(range(0, data_width)): BITFIELDs += str(i) + '|' REG_OV = _('Register Overview').encode('utf8') ADDR = _('Address').encode('utf8') REG = _('Register').encode('utf8') RSTVAL = _('Reset value').encode('utf8') buffer.write('.' + REG_OV + '\n') buffer.write('[cols="^3e,^3s,'+ str(data_width) +'*^1,^5",options="header"]\n') buffer.write('|===================================================\n') buffer.write('| ' + ADDR + ' | ' + REG + ' |' + BITFIELDs + RSTVAL + '\n') for reg in addressBlock.register: template = lookup.get_template('adoc_regs_overview.mako') ctx = Context(buffer, reg = reg, cfg = cfg, addr_width = addr_width, data_width = data_width) template.render_context(ctx) buffer.write("\n|===================================================\n")
def context(buffer=None, **maps): if not buffer: buffer = StringIO() ctx = Context(buffer, **maps) return ctx
def reset_buffer(self): """ Refreshes the internal buffer and resets the context """ self._buffer = StringIO() self._context = Context(self._buffer, **self._context.kwargs)
def begin(): # Imports for mako try: from mako.template import Template from mako.runtime import Context from mako.lookup import TemplateLookup from mako import exceptions except: ntop.printHTMLHeader('ntop Python Configuration Error', 1, 1) ntop.sendString( "<b><center><font color=red>Please install <A HREF=http://www.makotemplates.org/>Mako</A> template engine</font><p></b><br>(1) 'sudo yum install python-setuptools' (on RedHat-like systems)<br>(2) 'sudo easy_install Mako'</font></center>" ) ntop.printHTMLFooter() return # Fix encoding #reload(sys) #sys.setdefaultencoding("latin1") templateFilename = 'ipPortQuery.tmpl' #fb_DB = '/tmp/' #ntop.getPreference ("fastbitDBPath"); #default location of the fastbit DB databasePath = ntop.getPreference("fastbit.DBPath") #default location of the fastbit DB if databasePath is None or databasePath == '': ntop.printHTMLHeader('ntop Fastbit Configuration Error', 1, 1) ntop.sendString( "<b><center><font color=red>Please set fastbit.DBPath ntop preferences from <i>Admin/Configure/Preferences</i> menu (es: fastbit.DBPath=/tmp/)</b></font></center>" ) ntop.printHTMLFooter() return #pathFastBit=os.path.join(databasePath,'fastbit'+os.path.sep) form = cgi.FieldStorage() #get from the url the parameter configfile that contains the #path+filename of the configfile to read fromAuto = form.getvalue('fromAuto') if fromAuto: #print>>sys.stderr, "AJAX REQUEST FOR PARTITION IN "+databasePath+" "+fromAuto jsonString = expandFrom(fromAuto, os.path.join(databasePath, "")) ntop.sendHTTPHeader(12) ntop.sendString(jsonString) return documentRoot = os.getenv('DOCUMENT_ROOT', '.') selectArg = 'PROTOCOL,IPV4_SRC_ADDR,L4_SRC_PORT,IPV4_DST_ADDR,L4_DST_PORT,IN_BYTES,IN_PKTS' fromArg = form.getvalue('partition') ipSrc = form.getvalue('ipSrc') ipDst = form.getvalue('ipDst') portSrc = form.getvalue('portSrc') portDst = form.getvalue('portDst') limit = int(form.getvalue('limit', 100)) if limit < 0: limit = 0 res = None #variable to store the results of the query ntop.printHTMLHeader('IP-Port Query', 1, 0) #regex to check passed parameters ipV4Type = re.compile( r'(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' ) portType = re.compile(r'\d{1,5}') #pprint.pprint(ipV4Type.match(str(ipSrc)), sys.stderr) formatErrorString = '' #without the partition do nothing if fromArg: whereArg = '1=1' #to avoid leaving the where clause empty if ipSrc: #print>>sys.stderr, 'ECCO '+str(ipV4Type.match(ipSrc)) if ipV4Type.match(ipSrc): whereArg = whereArg + ' AND IPV4_SRC_ADDR=' + str( ipToNumber(ipSrc)) else: formatErrorString = formatErrorString + 'Source ip format invalid! ipv4 format required. ' if portSrc: if portType.match(portSrc): whereArg = whereArg + ' AND L4_SRC_PORT=' + str(portSrc) else: formatErrorString = formatErrorString + 'Source Port format invalid! Number required. ' if ipDst: if ipV4Type.match(ipDst): whereArg = whereArg + ' AND IPV4_DST_ADDR=' + str( ipToNumber(ipDst)) else: formatErrorString = formatErrorString + 'Destination ip format invalid! ipv4 format required. ' if portDst: if portType.match(portDst): whereArg = whereArg + ' AND L4_DST_PORT=' + str(portDst) else: formatErrorString = formatErrorString + 'Destination Port format invalid! Number required. ' try: #pipe = subprocess.Popen (['ntop.getPreference ("fastbit.fbquery")', "-c", selectArg, "-d", fromArg, "-q", whereArg, "-P", "-L", limit], #print>>sys.stderr, "Query passed: SELECT %s FROM %s WHERE %s LIMIT %i" %(selectArg,os.path.join(databasePath, fromArg), whereArg, limit) if formatErrorString == '': res = fastbit.query(os.path.join(databasePath, fromArg), selectArg, whereArg, limit) else: print >> sys.stderr, 'ipPortQuery: ERROR ' + formatErrorString ntop.sendString('<center><font color=red>%s</font></center>' % formatErrorString) #print>>sys.stderr, 'Number of records: %i' % len(res['values']) except: print >> sys.stderr, 'ERROR Executing query: ' + ( "SELECT %s FROM %s WHERE %s LIMIT %i" % (selectArg, os.path.join(databasePath, fromArg), whereArg, limit)) res = {} if res is not None and 'columns' in res and 'values' in res: toFormat = getAddrCols( selectArg) #get a list of addr column numbers for x in res['values']: x[0] = getNameByProto(x[0]) #format protocol number to text for j in toFormat: #for every number in the list format as an IP ADDR ipStr = numberToIp(x[j]) x[j] = '<a href="/%s.html" class="tooltip">%s</a>' % ( ipStr, ipStr ) #format ip number to string ip and create a link to ntop host page '''else: print>>sys.stderr, 'ipPortQuery: ERROR partition required' ntop.sendString('<center><font color=red>Partition field required!</font></center>')''' #pprint.pprint(res, sys.stderr) #if res is not None: # res['columns']=['Protocol', 'IP Source Addr', 'IP Dest. Addr', 'Source Port', 'Dest. Port', 'Bytes Rcvd', 'Packets Rcvd'] try: basedir = os.path.join(documentRoot, 'python/templates') mylookup = TemplateLookup(directories=[basedir]) myTemplate = mylookup.get_template(templateFilename) buf = StringIO() ctx = Context(buf, results=res) myTemplate.render_context(ctx) ntop.sendString(buf.getvalue()) except: ntop.sendString(exceptions.html_error_template().render()) ntop.printHTMLFooter()
data[ 'hours'] = 'Half an hour past sunrise to half an hour past sunset.' else: data['hours'] = 'Sunrise to Sunset' if (not 'parking' in data): data['parking'] = '' if (not 'directions' in data): data['directions'] = '' if (not 'address' in data): data['address'] = '' data['directionsX'] = '' data['directionsY'] = '' if (not 'description' in data): data['description'] = '' data['baseFileName'] = baseFileName #print(data) buf = StringIO() ctx = Context(buf, **data) mytemplate.render_context(ctx) outputFile = codecs.open(outputFileName, encoding='utf-8', mode='w') outputFile.write(buf.getvalue()) outputFile.close()
def begin(): rrdFilesPath = os.path.join(ntop.getDBPath(), 'rrd') ''' Check the existence of the rrd database files that maintain history of this script''' def updateDBS(time, resultsValue, durationValue): #global rrdFilesPath rrdAlarmDbPath = os.path.join(rrdFilesPath, 'rrdAlarm/') if not os.path.exists(rrdAlarmDbPath) or not os.path.isdir( rrdAlarmDbPath): os.mkdir(rrdAlarmDbPath) nameDuration = "duration" nameResults = "results" #nameDuration="duration.rrd" #nameResults="results.rrd" #archives=['RRA:AVERAGE:0.5:1:60', # 'RRA:AVERAGE:0.5:30:6']#1 hour and 3 hours data try: #rrdtool.update(os.path.join(rrdAlarmDbPath,nameDuration), 'N:'+str(durationValue)) ntop.updateRRDGauge(rrdAlarmDbPath, nameDuration, durationValue, 0) #print>>sys.stderr, 'Updating'+str(durationValue) except: print >> sys.stderr, 'RRDAlarm: Error Updating rrdDB ' + nameDuration ''' dataSources=['DS:duration:GAUGE:120:0:U'] rrdtool.create(rrdAlarmDbPath+nameDuration, '--start', str(time), '--step', str(60), dataSources[0], archives[0], archives[1] ) rrdtool.update(rrdAlarmDbPath+nameDuration,'N:'+str(durationValue))''' try: #rrdtool.update(os.path.join(rrdAlarmDbPath,nameResults), 'N:'+str(resultsValue)) ntop.updateRRDGauge(rrdAlarmDbPath, nameResults, resultsValue, 0) except: print >> sys.stderr, 'RRDAlarm: Error Updating rrdDB ' + nameResults ''' dataSources=['DS:results:GAUGE:120:0:U'] rrdtool.create(rrdAlarmDbPath+nameResults, '--start', str(time), '--step', str(60), dataSources[0], archives[0], archives[1] ) rrdtool.update(rrdAlarmDbPath+nameResults, 'N:'+str(resultsValue))''' '''Function that must be called as a new thread so its execution time can be controlled and limited by the main thread. all the controls on the rrds are executed by this function ''' def controlT(): #global rrdFilesPath ntopSpoolPath = ntop.getSpoolPath() nameFileConfig = 'rrdAlarmConfig.txt' tempFileName = 'rrdAlarmStart.tmp' configuration = None TWODECIMAL = decimal.Decimal(10)**-2 timeStart = time.time() alarmsFired = 0 checkedFiles = 0 fmt = '%a, %d %b %Y %H:%M:%S' #format of the time showed form = cgi.FieldStorage() #get the parameter passed via the url noHTML = bool(form.getvalue('noHTML')) configFile = form.getvalue('configFile') if configFile and len(configFile) > 0: nameFileConfig = str(configFile) try: tempFile = open(os.path.join(ntopSpoolPath, tempFileName), 'r') configuration = pickle.load(tempFile) tempFile.close() if configuration and ( timeStart < float(configuration['timeStart']) + float(60)): ntop.sendHTTPHeader(1) ntop.printHTMLHeader('RRD Alarm Called too early!', 1, 0) ntop.sendString( "Wait at least a minute. Last Time started: %s" % time.strftime(fmt, time.localtime(configuration['timeStart']))) ntop.printHTMLFooter() return 0 #exit because the script was started less than one minute ago else: configuration['timeStart'] = timeStart except IOError: #the tempFile does not exist or some other problem print >> sys.stderr, 'RRDAlarm: IOError while accessing tempfile ' + tempFileName configuration = createNewConfiguration( rrdFilesPath, os.path.join(ntopSpoolPath, nameFileConfig), timeStart) except pickle.PickleError, pickle.UnpicklingError: print >> sys.stderr, "RRDAlarm: Problems during the UnPickling load, tempFile Delete..." os.remove(os.path.join(ntopSpoolPath, tempFileName)) return -1 if configuration['lastModified'] != os.path.getmtime( os.path.join(ntopSpoolPath, nameFileConfig)): #if the configuration file has been changed the temp file must be rebuild and so the configuration dictionary configuration = createNewConfiguration( rrdFilesPath, os.path.join(ntopSpoolPath, nameFileConfig), timeStart) listRows = [] parameterDic = { } #for each parameter inserted as a key a tupla will be assigned 'parameter':{actionToPerform, text} for threshold in configuration[ 'listThresholds']: #for all the thresholds listFiles = threshold.getListFilename() checkedFiles = checkedFiles + len(listFiles) for fileN in listFiles: #for all the filenames referred by the threshold #rrd_argv=[fileN,'AVERAGE', '--start', threshold.getStartTime(), '--end', threshold.getStartTime()] #Return :((start, end, step), (name1, name2, ...), [(data1, data2, ..), ...]) #print>>sys.stderr, '\nLOOK for the parameters '+str(threshold.getStartTime())+' '+str(threshold.getEndTime())+' '+str(fileN) rrdObj = ((0, 0, 0), (), []) #empty object try: #rrdObj=rrdtool.fetch(fileN, 'AVERAGE', '--start', threshold.getStartTime(), '--end', threshold.getEndTime()) rrdObj = ntop.rrd_fetch(fileN, 'AVERAGE', threshold.getStartTime(), threshold.getEndTime()) except Exception as e: print >> sys.stderr, 'start.py PyRRDTool exception ' + str( e) step = rrdObj[0][2] start = float(rrdObj[0][0]) end = float(rrdObj[0][1]) valueDataTuple = rrdObj[2] #for all the values returned check the threshold (from the end) if alarm has to be fired i = len(valueDataTuple) while i > 0: #for value in valueDataTuple: timeA = (step * i) + start i = i - 1 value = valueDataTuple[i] if threshold.checkIfFire( value[0]): #controls if the threshold was exceeded notFired = False alarmsFired = alarmsFired + 1 listRows.append( (threshold.getUniqueId(), fileN, value[0], threshold.getType(), threshold.getValue(), time.strftime(fmt, time.localtime(timeA)), timeA, threshold.getActionToPerform(), 'ALARM FIRED')) strAlarm = '<ALARM>\nID: %i FILENAME: %s\nVALUE: %s TYPE: %s THRESHOLD VALUE: %f\n LOCALTIME: %s START: %s END: %s\n</ALARM>\n' % ( threshold.getUniqueId(), fileN, value[0], threshold.getType(), threshold.getValue(), time.strftime(fmt, time.localtime(timeA)), threshold.getStartTime(), threshold.getEndTime()) if parameterDic.has_key( threshold.getActionParameter()): parameterDic[threshold.getActionParameter( )]['textAlarm'] = parameterDic[ threshold.getActionParameter( )]['textAlarm'] + strAlarm else: parameterDic[threshold.getActionParameter()] = { 'actionToPerform': threshold.getActionToPerform(), 'textAlarm': strAlarm } break #print>>sys.stderr, 'The type of the threshold was misconfigured!' else: #no alarm was fired listRows.append( (threshold.getUniqueId(), fileN, '-', threshold.getType(), threshold.getValue(), time.strftime(fmt, time.localtime(end)), end, 'None', 'OK') ) #at least one alarm was fired, adding the action to perform, text and parameter to the global dictionary saveTempFile( configuration, os.path.join(ntopSpoolPath, tempFileName) ) #save all the informations usefull for future calling of this script documentRoot = os.getenv('DOCUMENT_ROOT', '.') performActions( parameterDic, documentRoot ) #performs all the actions for the alarms fired (if any) duration = decimal.Decimal(str(time.time() - timeStart)).quantize( TWODECIMAL, decimal.ROUND_CEILING) updateDBS( int(timeStart), alarmsFired, duration ) #update rrds that trace the history of this script TODO check where to place this ntop.sendHTTPHeader(1) if not noHTML: #if this parameter was passed and if true send just the empty http response ntop.printHTMLHeader('RRD Alarm Report', 1, 0) try: basedir = os.path.join(documentRoot, 'python/templates') mylookup = TemplateLookup(directories=[basedir]) myTemplate = mylookup.get_template('rrdAlarmStart.tmpl') buf = StringIO() ctx = Context(buf, listRows=listRows, duration=duration, checkedFiles=checkedFiles, alarmsFired=alarmsFired) myTemplate.render_context(ctx) ntop.sendString(buf.getvalue()) except: ntop.sendString(exceptions.html_error_template().render()) return 1 #finally: #condition.notify() ntop.printHTMLFooter() print >> sys.stderr, '%s CET Exit rrdAlarm' % time.strftime( fmt, time.localtime(time.time())) return 0
def render_mako_str(template, context): t=Template(template) buf=StringIO() ctx=Context(buf, **context) t.render_context(ctx) return buf.getvalue()
def extract(self, async_extract=None, **kw): """ Will return a list of all extracted files """ if async_extract == True or (async_extract == None and ignore_fireworks()): return self.async_extract(**kw) parse_type = self.read_parse_type(self.filename) import os, os.path from mako.template import Template from mako.runtime import Context from StringIO import StringIO vol = urllib.quote(os.listdir('/Volumes')[0]) prefix = 'file:///' + vol temp_dir = tempfile.mkdtemp() eid = 'FILE' data = { 'in_file': prefix + os.path.abspath(self.filename), 'out_dir': prefix + temp_dir + '/', 'eid': eid } #script template here = os.path.dirname(os.path.dirname(__file__)) script = os.path.join(here, 'backend', 'fireworks_export.js') mytemplate = Template(filename=script) buf = StringIO() ctx = Context(buf, **data) mytemplate.render_context(ctx) val = buf.getvalue() script, scname = self._get_tmp_file(None, dest_format='js') script.write(val) script.close() c = adobe.Fireworks() c.connect() error = None special_script = prefix + scname try: x = c.call('fw', 'runScript', special_script) except adobe.AdobeException as e: error = e if e.code != adobe.ERROR_DIED: raise print 'Fireworks died. But we kind of dont care. ON WITH IT, BITCHES.' os.remove(scname) regex = re.compile('FILE([0-9]+)') def key(s): m = regex.search(s) if m: return int(m.group(1)) return None #now there should be files. files = os.listdir(temp_dir) files.sort(key=key) files = [ ExtractedFile(os.path.join(temp_dir, f), EXTRACT_TYPE_FULL) for f in files if key(f) != None ] if files: self.image = Image(files[0].filename) files += self.thumbnail() self.image.destroy() elif error: raise error #ok, maybe it legitimately died. return files, ExtractStatus(type=parse_type, file_type='FWPNG')
def render(template, **attrs): buf = StringIO() ctx = Context(buf, **attrs) template.render_context(ctx, **attrs) return buf.getvalue()
#failed ipv6 adn ipv4 conversion print >> sys.stderr, "fastbit.py: IMPOSSIBLE TO FORMAT value: " + str( x[j]) + " TO IP ADDR" #x[1]=socket.inet_ntop(socket.AF_INET,struct.pack('>L',x[1])) rows = res['values'] #pprint.pprint(res, sys.stderr) try: basedir = os.path.join(documentRoot, 'python/templates') mylookup = TemplateLookup(directories=[basedir]) myTemplate = mylookup.get_template(templateFilename) buf = StringIO() ctx = Context(buf, columns=cols, values=rows, queryPar=queryPar, queryHistory=history['history']) myTemplate.render_context(ctx) ntop.sendString(buf.getvalue()) except: ntop.sendString(exceptions.html_error_template().render()) ntop.printHTMLFooter() '''HERE STARTS THE SCRIPTS''' begin()
month = dater[2] + dater[3] Months = {"01": "January", "02" : "February", "03" : "March", "04" : "April", "05" : "May", "06" : "June", "07" : "July", "08" : "August", "09" : "September", "10" : "October", "11" : "November", "12" : "December", "00" : " " } PhotoDate = Months[month] + " " + year #Write the file #Put the file path and name back together PhotoPage = FilePath + PhotoPage PhotoFile = Photo buf = StringIO() ctx = Context(buf, PhotoTitle=PhotoTitle, PhotoDate=PhotoDate, PhotoFile=PhotoFile, PhotoDescription=PhotoDescription, PhotoLocation=PhotoLocation, PhotoKeywords=PhotoKeywords) PhotoFrame.render_context(ctx) f = open(PhotoPage,"w") f.write(buf.getvalue()) f.close()
cur = con.cursor(mdb.cursors.DictCursor) cur.execute(dbCommand) except: print "Something ain't right with the db, program exiting" sys.exit(0) print "database o.k. Onward!" #Only after db is shown to work do we rewrite the file f = open(StoryFile,"w") for i in range(cur.rowcount): row = cur.fetchone() Link = "[BASE URL]" + row["FileLocation"] + row["FileName"] + ".html" Description = row["Description"] Headline = row["Headline"] Art = row["Art"] Published = row["Published"] Subject = row["Subject"] #Parsing the date Published = Published.strftime("%a, %d %b %Y") buf = StringIO() ctx = Context(buf, Headline=Headline, Link=Link, Description=Description, Published=Published, Art=Art, Subject=Subject) StoryListing.render_context(ctx) f.write(buf.getvalue()) f.close() #DB connectivity code from http://zetcode.com/db/mysqlpython/ #Date format http://www.faqs.org/rfcs/rfc822.html
def gen_nav(url): global test_url test_url = url h = helper_struct(url_for=get_test_url) c = context_struct(subsubmenu={}) buf = StringIO() ctx = Context(buf, h=h, c=c) try: t.render_context(ctx) full_html = buf.getvalue() except: print exceptions.text_error_template().render() return full_html = buf.getvalue() sub_html_pt = full_html.find("<!-- Secondary navigation") pri_html = full_html[0:sub_html_pt] sec_html = full_html[sub_html_pt:-1] pri_text = [ y for y in (x.strip() for x in re.sub(r'<.*?>', '', pri_html).splitlines()) if y ] sec_text = [ y for y in (x.strip() for x in re.sub(r'<.*?>', '', sec_html).splitlines()) if y ] # put a flag "LINK" in each line with a url, strip it out later mangled_pri_links = re.sub(r'.*?href\s?=\s?[\'"](.*?)[\'"].*', r'LINK \1', pri_html) pri_links = [ x[5:].strip() for x in mangled_pri_links.splitlines() if x[0:5] == "LINK " ] mangled_sec_links = re.sub(r'.*?href\s?=\s?[\'"](.*?)[\'"].*', r'LINK \1', sec_html) sec_links = [ x[5:].strip() for x in mangled_sec_links.splitlines() if x[0:5] == "LINK " ] # Find the selected link, if there is one pri_sel = [ re.sub(r'.*?href\s?=\s?[\'"](.*?)[\'"].*', r'\1', x) for x in pri_html.splitlines() if "selected" in x ] sec_sel = [ re.sub(r'.*?href\s?=\s?[\'"](.*?)[\'"].*', r'\1', x) for x in sec_html.splitlines() if "selected" in x ] sec_sel_text = [ re.sub(r'<.*?>', '', x).strip() for x in sec_html.splitlines() if "selected" in x ] return { 'full_html': full_html, 'pri_text': pri_text, 'pri_links': pri_links, 'sec_text': sec_text, 'sec_links': sec_links, 'pri_sel': pri_sel[0] if pri_sel else None, 'sec_sel': sec_sel[0] if sec_sel else None, 'sec_sel_text': sec_sel_text[0] if sec_sel_text else None }
project = sys.argv[1] langs = get_available_lang('swissguesser/static/%s/data/locale' % project) for lang in langs: with open('swissguesser/static/%s/data/locale/%s/translation.json' % (project, lang)) as data_file: data = json.load(data_file) for key in data.keys(): data[key.replace('-','_')] = data.pop(key) data['lang'] = lang data['Preview_Url'] = u"images/preview.jpg" # relative url should work data['App_Url'] = u"http://storymaps.geo.admin.ch/storymaps/%s" % project buf = StringIO() ctx = Context(buf,**data) print ctx.keys() serve_template('index.html.mako', ctx) html = buf.getvalue().encode('utf-8') with open('swissguesser/static/%s/index.html.%s' % (project,lang), 'w') as f: f.write(html)