Father of the Bride Part II
'''
# poster filenames:
poster_file = {}
poster_file['toy_story'] = 'rhIRbceoE9lR4veEXuwCC2wARtG.jpg'
poster_file['jumanji'] = 'vzmL6fP7aPKNKPRTFnZmiUfciyV.jpg'
poster_file['grumpier_old_men'] = '6ksm1sjKMFLbO7UY2i6G1ju9SML.jpg'
poster_file['waiting_to_exhale'] = '16XOMpEaLWkrcPqSQqhTmeJuqQl.jpg'
poster_file['father_of_the_bride'] = 'e64sOI48hQXyru7naBFyssKFxVd.jpg'
app = Flask(__name__)
HTML_TEMPLATE = Template("""
<h1>Hello ${file_name}!</h1>
<img src="https://image.tmdb.org/t/p/w342/${file_name}" alt="poster for ${file_name}">
""")
@app.route('/')
def homepage():
return """<h1>Test of dynamic poster display here Feb 2</h1>"""
@app.route('/<some_file>')
def some_place_page(some_file):
return (HTML_TEMPLATE.substitute(file_name=some_file))
def test_call(test_text):
class THPPlugin(CWrapPlugin):
TYPE_UNPACK = {
'THFloatTensor*': Template('((THPFloatTensor*)$arg)->cdata'),
'THDoubleTensor*': Template('((THPDoubleTensor*)$arg)->cdata'),
'THLongTensor*': Template('((THPLongTensor*)$arg)->cdata'),
'THIntTensor*': Template('((THPIntTensor*)$arg)->cdata'),
'THTensor*': Template('((THPTensor*)$arg)->cdata'),
'THBoolTensor*': Template('((THPBoolTensor*)$arg)->cdata'),
'THIndexTensor*': Template('((THPIndexTensor*)$arg)->cdata'),
'THIntegerTensor*': Template('((THPIntegerTensor*)$arg)->cdata'),
'THCudaTensor*': Template('((THCPFloatTensor*)$arg)->cdata'),
'THCudaDoubleTensor*': Template('((THCPDoubleTensor*)$arg)->cdata'),
'THCudaIntTensor*': Template('((THCPIntTensor*)$arg)->cdata'),
'THCudaLongTensor*': Template('((THCPLongTensor*)$arg)->cdata'),
'THSFloatTensor*': Template('((THSPFloatTensor*)$arg)->cdata'),
'THSDoubleTensor*': Template('((THSPDoubleTensor*)$arg)->cdata'),
'THSLongTensor*': Template('((THSPLongTensor*)$arg)->cdata'),
'THSIntTensor*': Template('((THSPIntTensor*)$arg)->cdata'),
'THSTensor*': Template('((THSPTensor*)$arg)->cdata'),
'THSBoolTensor*': Template('((THSPBoolTensor*)$arg)->cdata'),
'THSIndexTensor*': Template('((THSPIndexTensor*)$arg)->cdata'),
'THLongStorage*': Template('((THPLongStorage*)$arg)->cdata'),
'THStorage*': Template('((THPStorage*)$arg)->cdata'),
'THGenerator*': Template('((THPGenerator*)$arg)->cdata'),
'THSize*': Template('__size.get()'),
'THStride*': Template('__stride.get()'),
'void*': Template('THPUtils_unpackLong($arg)'),
'long': Template('THPUtils_unpackLong($arg)'),
'int': Template('THPUtils_unpackLong($arg)'),
'bool': Template('($arg == Py_True ? true : false)'),
'float': Template('THPFloatUtils_unpackReal($arg)'),
'double': Template('THPDoubleUtils_unpackReal($arg)'),
'real': Template('THPUtils_(unpackReal)($arg)'),
'accreal': Template('THPUtils_(unpackAccreal)($arg)'),
}
TYPE_CHECK = {
'THDoubleTensor*':
Template('(PyObject*)Py_TYPE($arg) == THPDoubleTensorClass'),
'THFloatTensor*':
Template('(PyObject*)Py_TYPE($arg) == THPFloatTensorClass'),
'THLongTensor*':
Template('(PyObject*)Py_TYPE($arg) == THPLongTensorClass'),
'THIntTensor*':
Template('(PyObject*)Py_TYPE($arg) == THPIntTensorClass'),
'THTensor*':
Template('(PyObject*)Py_TYPE($arg) == THPTensorClass'),
'THBoolTensor*':
Template('(PyObject*)Py_TYPE($arg) == THPBoolTensorClass'),
'THIndexTensor*':
Template('(PyObject*)Py_TYPE($arg) == THPIndexTensorClass'),
'THIntegerTensor*':
Template('(PyObject*)Py_TYPE($arg) == THPIntegerTensorClass'),
'THCudaTensor*':
Template('(PyObject*)Py_TYPE($arg) == THCPFloatTensorClass'),
'THCudaDoubleTensor*':
Template('(PyObject*)Py_TYPE($arg) == THCPDoubleTensorClass'),
'THCudaIntTensor*':
Template('(PyObject*)Py_TYPE($arg) == THCPIntTensorClass'),
'THCudaLongTensor*':
Template('(PyObject*)Py_TYPE($arg) == THCPLongTensorClass'),
'THSDoubleTensor*':
Template('(PyObject*)Py_TYPE($arg) == THSPDoubleTensorClass'),
'THSFloatTensor*':
Template('(PyObject*)Py_TYPE($arg) == THSPFloatTensorClass'),
'THSLongTensor*':
Template('(PyObject*)Py_TYPE($arg) == THSPLongTensorClass'),
'THSIntTensor*':
Template('(PyObject*)Py_TYPE($arg) == THSPIntTensorClass'),
'THSTensor*':
Template('(PyObject*)Py_TYPE($arg) == THSPTensorClass'),
'THSBoolTensor*':
Template('(PyObject*)Py_TYPE($arg) == THSPBoolTensorClass'),
'THSIndexTensor*':
Template('(PyObject*)Py_TYPE($arg) == THSPIndexTensorClass'),
'THLongStorage*':
Template('(PyObject*)Py_TYPE($arg) == THPLongStorageClass'),
'THStorage*':
Template('(PyObject*)Py_TYPE($arg) == THPStorageClass'),
'THGenerator*':
Template('(PyObject*)Py_TYPE($arg) == THPGeneratorClass'),
'THSize*':
Template('THPUtils_tryUnpackLongs($arg, __size)'),
'THStride*':
Template('THPUtils_tryUnpackLongs($arg, __stride)'),
'void*':
Template('THPUtils_checkLong($arg)'),
'long':
Template('THPUtils_checkLong($arg)'),
'int':
Template('THPUtils_checkLong($arg)'),
'bool':
Template('PyBool_Check($arg)'),
'float':
Template('THPFloatUtils_checkReal($arg)'),
'double':
Template('THPDoubleUtils_checkReal($arg)'),
'real':
Template('THPUtils_(checkReal)($arg)'),
'accreal':
Template('THPUtils_(checkAccreal)($arg)'),
}
SIZE_VARARG_CHECK = Template(
'THPUtils_tryUnpackLongVarArgs(args, $idx, __size)')
RETURN_WRAPPER = {
'THTensor*': Template('return THPTensor_(New)($result);'),
'THSTensor*': Template('return THSPTensor_(New)($result);'),
'THIndexTensor*': Template('return THPIndexTensor_(New)($result);'),
'THLongTensor*': Template('return THPLongTensor_New($result);'),
'THLongStorage*': Template('return THPLongStorage_New($result);'),
'THCudaIntTensor*': Template('return THCPIntTensor_New($result);'),
'THCudaLongTensor*': Template('return THCPLongTensor_New($result);'),
# TODO: make it smarter - it should return python long if result doesn't fit into an int
'long': Template('return PyInt_FromLong($result);'),
'accreal': Template('return THPUtils_(newAccreal)($result);'),
'self': Template('Py_INCREF(self);\nreturn (PyObject*)self;'),
'real': Template('return THPUtils_(newReal)($result);'),
}
TENSOR_METHODS_DECLARATION = Template("""
static PyMethodDef TH${sparse}PTensor_$stateless(methods)[] = {
$methods
{NULL}
};
""")
WRAPPER_TEMPLATE = Template("""\
PyObject * $name(PyObject *self, PyObject *args, PyObject *kwargs)
{
HANDLE_TH_ERRORS
int __tuplecount = args ? PyTuple_Size(args) : 0;
int __dictcount = kwargs ? PyDict_Size(kwargs) : 0;
int __argcount = __tuplecount + __dictcount;
$variables
$init
$options
}
THPUtils_invalidArguments(args, kwargs, "$readable_name", $num_options, $expected_args);
return NULL;
END_HANDLE_TH_ERRORS
}
""")
ALLOCATE_TMPL = Template("""\
THP${type}TensorPtr _${name}_guard((THP${type}Tensor*) THP${type}Tensor_NewEmpty());
if (!_${name}_guard.get()) return NULL;
THP${type}Tensor* $name = _${name}_guard.get();
""")
ALLOCATE_CUDA = Template("""\
#if IS_CUDA
${cuda}
#else
${cpu}
#endif
""")
def _allocate(typename, tmpl, cuda_tmpl=None, sparse=False):
code = tmpl.safe_substitute(type=typename)
if typename == '':
code = code.replace('NewEmpty', '(NewEmpty)')
if cuda_tmpl:
cuda_code = code.replace('THP', 'THCP')
code = cuda_tmpl.substitute(cuda=cuda_code, cpu=code)
if sparse:
code = code.replace('THP', 'THSP')
code = code.replace('THCP', 'THCSP')
return Template(code)
ALLOCATE_TYPE = {
'THTensor*': _allocate('', ALLOCATE_TMPL),
'THLongTensor*': _allocate('Long', ALLOCATE_TMPL),
'THIntTensor*': _allocate('Int', ALLOCATE_TMPL),
'THBoolTensor*': _allocate('Byte', ALLOCATE_TMPL, ALLOCATE_CUDA),
'THIndexTensor*': _allocate('Long', ALLOCATE_TMPL, ALLOCATE_CUDA),
'THIntegerTensor*': _allocate('Int', ALLOCATE_TMPL, ALLOCATE_CUDA),
'THSTensor*': _allocate('', ALLOCATE_TMPL, sparse=True),
}
TYPE_NAMES = {
'THTensor*': '" THPTensorStr "',
'THSTensor*': '" THSPTensorStr "',
'THStorage*': '" THPStorageStr "',
'THGenerator*': 'torch.Generator',
'THLongStorage*': '" THPModuleStr "LongStorage',
'THLongTensor*': '" THPModuleStr "LongTensor',
'THIntTensor*': '" THPModuleStr "IntTensor',
'THBoolTensor*': '" THPModuleStr "ByteTensor',
'THIndexTensor*': '" THPModuleStr "LongTensor',
'THIntegerTensor*': '" THPModuleStr "IntTensor',
'THFloatTensor*': '" THPModuleStr "FloatTensor',
'THDoubleTensor*': '" THPModuleStr "DoubleTensor',
'THCudaTensor*': 'torch.cuda.FloatTensor',
'THCudaDoubleTensor*': 'torch.cuda.DoubleTensor',
'THCudaIntTensor*': 'torch.cuda.IntTensor',
'THCudaLongTensor*': 'torch.cuda.LongTensor',
'THSize*': 'torch.Size',
'THStride*': 'tuple',
'long': 'int',
'real': '" RealStr "',
'double': 'float',
'accreal': '" RealStr "',
'bool': 'bool',
'const char*': 'bool', # Can come only from bool option.
}
OUT_INIT = """
__out = kwargs ? PyDict_GetItemString(kwargs, "out") : NULL;
if (__out == Py_None) { __out = NULL; __dictcount--; __argcount--; }
"""
def __init__(self):
self.declarations = []
self.stateless_declarations = []
self.docstrings = []
def get_type_unpack(self, arg, option):
return self.TYPE_UNPACK.get(arg['type'], None)
def get_type_check(self, arg, option):
if arg['type'] == 'THSize*' and arg.get('long_args', False):
return self.SIZE_VARARG_CHECK
return self.TYPE_CHECK.get(arg['type'], None)
# TODO: argument descriptions shouldn't be part of THP, but rather a general cwrap thing
def get_wrapper_template(self, declaration):
arg_desc = OrderedDict()
def format_arg(arg, var_args=False):
if var_args and arg.get('long_args', False):
return 'int ... ' + arg['name']
else:
return self.TYPE_NAMES[arg['type']] + ' ' + arg['name']
def format_args(args, var_args=False):
option_desc = [
format_arg(arg, var_args) for arg in args
if not arg.get('ignore_check', False) and not arg.get('output')
]
output_args = list(filter(lambda a: a.get('output'), args))
if output_args:
if len(output_args) > 1:
out_type = 'tuple['
out_type += ', '.join(self.TYPE_NAMES[arg['type']]
for arg in output_args)
out_type += ']'
option_desc += ['#' + out_type + ' out']
else:
arg = output_args[0]
option_desc += [
'#' + self.TYPE_NAMES[arg['type']] + ' out'
]
if option_desc:
return '({})'.format(', '.join(option_desc))
else:
return 'no arguments'
for option in declaration['options']:
arg_desc[format_args(option['arguments'], False)] = True
arg_desc[format_args(option['arguments'], True)] = True
arg_desc = sorted(list(arg_desc.keys()), key=len)
arg_desc = ['"' + desc + '"' for desc in arg_desc]
arg_str = ', '.join(arg_desc)
variables_str = '\n'.join(declaration.get('variables', []))
init_str = '\n'.join(declaration.get('init', []))
if 'stateless' in declaration['name']:
readable_name = 'torch.' + declaration['python_name']
else:
readable_name = declaration['python_name']
return Template(
self.WRAPPER_TEMPLATE.safe_substitute(readable_name=readable_name,
num_options=len(arg_desc),
expected_args=arg_str,
variables=variables_str,
init=init_str))
def get_return_wrapper(self, option):
return self.RETURN_WRAPPER.get(option['return'], None)
def get_arg_accessor(self, arg, option):
if arg['name'] == 'self':
return 'self'
if arg.get('output'):
if not option['output_provided']:
return arg['name']
if option['output_count'] == 1:
return '__out'
else:
return 'PyTuple_GET_ITEM(__out, {})'.format(arg['output_idx'])
def process_docstrings(self):
for declaration in self.declarations:
docstr = declaration.get('docstring_method')
if docstr is None:
continue
declaration['docstring_content'] = docstr.replace('\n', '\\n')
declaration[
'docstring_var'] = 'docstr_' + declaration['python_name']
for declaration in self.stateless_declarations:
docstr = declaration.get('docstring_stateless')
if docstr is None:
continue
declaration['docstring_content'] = docstr.replace('\n', '\\n')
declaration['docstring_var'] = 'stateless_docstr_' + declaration[
'python_name']
def generate_out_options(self, declaration):
new_options = []
declaration.setdefault('init', [])
declaration['init'] += [self.OUT_INIT]
for option in declaration['options']:
out_idx = []
for i, arg in enumerate(option['arguments']):
if arg.get('output'):
out_idx.append(i)
if not out_idx:
option['has_output'] = True
option['output_provided'] = False
new_options.append(option)
continue
for output_provided in (True, False):
option_copy = deepcopy(option)
option_copy['has_output'] = True
option_copy['output_provided'] = output_provided
option_copy['output_count'] = len(out_idx)
for i, idx in enumerate(out_idx):
arg = option_copy['arguments'][idx]
arg['output_idx'] = i
if not output_provided:
arg['ignore_check'] = True
else:
option_copy['argcount_offset'] = -len(out_idx) + 1
arg['no_kwargs'] = True
arg['no_idx'] = True
new_options.append(option_copy)
declaration['options'] = new_options
def process_declarations(self, declarations):
new_declarations = []
def has_arg_type(declaration, type_name):
return any(arg['type'] == type_name
for option in declaration['options']
for arg in option['arguments'])
def has_long_args(declaration):
return any(
arg.get('long_args', False)
for option in declaration['options']
for arg in option['arguments'])
def has_output_args(declaration):
return any(
arg.get('output') for option in declaration['options']
for arg in option['arguments'])
for declaration in declarations:
# Disable all methods for THHalfTensor, unless cpu_half is True
if not declaration.get('cpu_half', False):
defined_if = '!defined(TH_REAL_IS_HALF)'
if 'defined_if' in declaration:
defined_if += ' && (' + declaration['defined_if'] + ')'
declaration['defined_if'] = defined_if
if declaration.get('only_register', False):
continue
declaration.setdefault('python_name', declaration['name'])
declaration.setdefault('variables', [])
if has_arg_type(declaration, 'THSize*'):
declaration['variables'] += ['THLongStoragePtr __size;']
if has_arg_type(declaration, 'THStride*'):
declaration['variables'] += ['THLongStoragePtr __stride;']
if has_output_args(declaration):
declaration['variables'] += ['PyObject *__out;']
self.generate_out_options(declaration)
if has_long_args(declaration):
declaration['no_kwargs'] = True
for option in declaration['options']:
option['cname'] = 'TH{}Tensor_({})'.format(
'S' if option.get('sparse', False) else '',
option['cname'])
if declaration.get('with_stateless', False) or declaration.get(
'only_stateless', False):
stateless_declaration = self.make_stateless(declaration)
new_declarations.append(stateless_declaration)
self.stateless_declarations.append(stateless_declaration)
if declaration.get('only_stateless', False):
continue
self.declarations.append(declaration)
declaration['name'] = 'TH{}PTensor_({})'.format(
'S' if declaration.get('sparse', False) else '',
declaration['name'])
for option in declaration['options']:
for arg in option['arguments']:
if arg['name'] == 'self':
arg['ignore_check'] = True
register_only = [
d for d in declarations if d.get('only_register', False)
]
declarations = [
d for d in declarations
if (not d.get('only_stateless', False)) and (
not d.get('only_register', False))
]
self.declarations.extend(
filter(lambda x: not x.get('only_stateless', False),
register_only))
self.stateless_declarations.extend(
filter(lambda x: x.get('only_stateless', False), register_only))
self.process_docstrings()
all_declarations = declarations + new_declarations
return all_declarations
def make_stateless(self, declaration):
declaration = deepcopy(declaration)
declaration['name'] = 'TH{}PTensor_stateless_({})'.format(
'S' if declaration.get('sparse', False) else '',
declaration['name'])
for option in declaration['options']:
for arg in option['arguments']:
if arg['name'] == 'self':
arg['assign_name'] = 'self'
arg['name'] = 'source'
return declaration
def declare_methods(self, stateless, sparse):
tensor_methods = ''
for declaration in (self.declarations
if not stateless else self.stateless_declarations):
if declaration.get('sparse', False) != sparse:
continue
flags = 'METH_VARARGS'
flags += ' | ' + declaration.get(
'method_flags') if 'method_flags' in declaration else ''
if not declaration.get('only_register'):
flags += ' | METH_KEYWORDS'
if declaration.get('override_method_flags'):
flags = declaration['override_method_flags']
entry = Template(
' {"$python_name", (PyCFunction)$name, $flags, $docstring},\n'
).substitute(python_name=declaration['python_name'],
name=declaration['name'],
flags=flags,
docstring=declaration.get('docstring_var', 'NULL'))
if 'defined_if' in declaration:
entry = self.preprocessor_guard(entry,
declaration['defined_if'])
tensor_methods += entry
generated = self.TENSOR_METHODS_DECLARATION.substitute(
methods=tensor_methods,
stateless=('' if not stateless else 'stateless_'),
sparse=('' if not sparse else 'S'),
)
if sparse:
generated = '#ifndef TH_REAL_IS_HALF\n' + generated + '\n#endif\n\n'
return generated
def process_full_file(self, code):
# We have to find a place before all undefs
idx = code.find('// PUT DEFINITIONS IN HERE PLEASE')
return (code[:idx] + self.declare_methods(False, False) +
self.declare_methods(True, False) +
self.declare_methods(False, True) +
self.declare_methods(True, True) + code[idx:])
def preprocessor_guard(self, code, condition):
return '#if ' + condition + '\n' + code + '#endif\n'
def process_wrapper(self, code, declaration):
if 'defined_if' in declaration:
return self.preprocessor_guard(code, declaration['defined_if'])
return code
def process_all_call_arg(self, code, option):
return 'LIBRARY_STATE ' + code
def process_all_checks(self, code, option):
if option.get('has_output'):
indent = " " * 10
if option['output_provided']:
checks = "__out != NULL &&\n" + indent
if option['output_count'] > 1:
checks += "PyTuple_Check(__out) &&\n" + indent
length_check = "PyTuple_GET_SIZE(__out) == {} &&\n".format(
option['output_count'])
checks += length_check + indent
code = checks + code
else:
code = "__out == NULL &&\n" + indent + code
if any(arg.get('long_args', False) for arg in option['arguments']):
code = code.replace('__argcount ==', '__argcount >=')
expected = str(int(option.get('output_provided', False)))
code = '__dictcount == ' + expected + ' &&\n ' + code
return code
def process_pre_arg_assign(self, template, option):
new_args = []
for arg in option['arguments']:
if not option.get('output_provided', True) and arg.get('output'):
new_args.append(self.ALLOCATE_TYPE[arg['type']].substitute(
name=arg['name']))
template = new_args + template
return template
def generate_docstrings_cpp(self):
template = Template('char* $name = "$content";')
return '\n\n'.join(
template.substitute(name=decl['docstring_var'],
content=decl['docstring_content'])
for decl in chain(self.declarations, self.stateless_declarations)
if 'docstring_var' in decl)
def generate_docstrings_h(self):
template = Template('extern char* $name;')
return '\n\n'.join(
template.substitute(name=decl['docstring_var'])
for decl in chain(self.declarations, self.stateless_declarations)
if 'docstring_var' in decl)
def generatecpp(self):
# Read the configuration manipulator template and insert line numbers and file name into template.
templatefile = os.path.join(self.dynconfpath, "templates", "ConfigType.h.template")
templatelines = []
templatefilesafe = templatefile.replace('\\', '\\\\') # line directive does backslash expansion.
curline = 1
f = open(templatefile)
for line in f:
curline = curline + 1
templatelines.append(Template(line).safe_substitute(linenum=curline, filename=templatefilesafe))
f.close()
template = ''.join(templatelines)
# Write the configuration manipulator.
self.mkdirabs(self.cpp_gen_dir)
f = open(os.path.join(self.cpp_gen_dir, self.name + "Config.h"), 'w')
paramdescr = []
groups = []
members = []
constants = []
for const in self.constants:
self.appendline(constants, "${cconsttype} ${configname}_${name} = $v;", const, "value")
def write_params(group):
if group.id == 0:
paramdescr.append(Template("${configname}Config::GroupDescription<${configname}Config::${class}, ${configname}Config> ${name}(\"${name}\", \"${type}\", ${parent}, ${id}, ${cstate}, &${configname}Config::${lower});").safe_substitute(group.to_dict(), configname=self.name))
else:
paramdescr.append(Template("${configname}Config::GroupDescription<${configname}Config::${class}, ${configname}Config::${parentclass}> ${name}(\"${name}\", \"${type}\", ${parent}, ${id}, ${cstate}, &${configname}Config::${field});").safe_substitute(group.to_dict(), configname=self.name))
for param in group.parameters:
self.appendline(members, "${ctype} ${name};", param)
self.appendline(paramdescr, "__min__.${name} = $v;", param, "min")
self.appendline(paramdescr, "__max__.${name} = $v;", param, "max")
self.appendline(paramdescr, "__default__.${name} = $v;", param, "default")
self.appendline(
paramdescr,
group.to_dict()['name'] +
".abstract_parameters.push_back(${configname}Config::AbstractParamDescriptionConstPtr(new ${configname}Config::ParamDescription<${ctype}>(\"${name}\", \"${type}\", ${level}, "
"\"${description}\", \"${edit_method}\", &${configname}Config::${name})));", param)
self.appendline(
paramdescr,
"__param_descriptions__.push_back(${configname}Config::AbstractParamDescriptionConstPtr(new ${configname}Config::ParamDescription<${ctype}>(\"${name}\", \"${type}\", ${level}, "
"\"${description}\", \"${edit_method}\", &${configname}Config::${name})));", param)
for g in group.groups:
write_params(g)
self.appendline(paramdescr, "${name}.convertParams();", group.to_dict())
if group.id == 0:
self.appendline(paramdescr, "__group_descriptions__.push_back(${configname}Config::AbstractGroupDescriptionConstPtr(new ${configname}Config::GroupDescription<${configname}Config::${class}, ${configname}Config>(${name})));", group.to_dict())
else:
self.appendline(paramdescr, "${parentname}.groups.push_back(${configname}Config::AbstractGroupDescriptionConstPtr(new ${configname}Config::GroupDescription<${configname}Config::${class}, ${configname}Config::${parentclass}>(${name})));", group.to_dict())
self.appendline(paramdescr, "__group_descriptions__.push_back(${configname}Config::AbstractGroupDescriptionConstPtr(new ${configname}Config::GroupDescription<${configname}Config::${class}, ${configname}Config::${parentclass}>(${name})));", group.to_dict())
write_params(self.group)
self.appendgroup(groups, self.group)
paramdescr = '\n'.join(paramdescr)
members = '\n'.join(members)
constants = '\n'.join(constants)
groups = '\n'.join(groups)
f.write(Template(template).substitute(
uname=self.name.upper(),
configname=self.name, pkgname=self.pkgname, paramdescr=paramdescr,
members=members, groups=groups, doline=LINEDEBUG, constants=constants))
f.close()
print("Wrote header file in " + os.path.join(self.cpp_gen_dir, self.name + "Config.h"))
def _generate_images(self, scene_list, video_name,
image_name_template='$VIDEO_NAME-Scene-$SCENE_NUMBER-$IMAGE_NUMBER',
output_dir=None):
# type: (List[Tuple[FrameTimecode, FrameTimecode]) -> None
if not scene_list:
return
if not self.options_processed:
return
if self.num_images <= 0:
raise ValueError()
self.check_input_open()
imwrite_param = []
if self.image_param is not None:
imwrite_param = [self.imwrite_params[self.image_extension], self.image_param]
# Reset video manager and downscale factor.
self.video_manager.release()
self.video_manager.reset()
self.video_manager.set_downscale_factor(1)
self.video_manager.start()
# Setup flags and init progress bar if available.
completed = True
logging.info('Generating output images (%d per scene)...', self.num_images)
progress_bar = None
if tqdm and not self.quiet_mode:
progress_bar = tqdm(
total=len(scene_list) * self.num_images, unit='images')
filename_template = Template(image_name_template)
scene_num_format = '%0'
scene_num_format += str(max(3, math.floor(math.log(len(scene_list), 10)) + 1)) + 'd'
image_num_format = '%0'
image_num_format += str(math.floor(math.log(self.num_images, 10)) + 2) + 'd'
timecode_list = dict()
fps = scene_list[0][0].framerate
timecode_list = [
[
FrameTimecode(int(f), fps=fps) for f in [
# middle frames
a[len(a)//2] if (0 < j < self.num_images-1) or self.num_images == 1
# first frame
else min(a[0] + self.image_frame_margin, a[-1]) if j == 0
# last frame
else max(a[-1] - self.image_frame_margin, a[0])
# for each evenly-split array of frames in the scene list
for j, a in enumerate(np.array_split(r, self.num_images))
]
]
for i, r in enumerate([
# pad ranges to number of images
r
if 1+r[-1]-r[0] >= self.num_images
else list(r) + [r[-1]] * (self.num_images - len(r))
# create range of frames in scene
for r in (
range(start.get_frames(), end.get_frames())
# for each scene in scene list
for start, end in scene_list
)
])
]
self.image_filenames = {i: [] for i in range(len(timecode_list))}
for i, scene_timecodes in enumerate(timecode_list):
for j, image_timecode in enumerate(scene_timecodes):
self.video_manager.seek(image_timecode)
self.video_manager.grab()
ret_val, frame_im = self.video_manager.retrieve()
if ret_val:
file_path = '%s.%s' % (filename_template.safe_substitute(
VIDEO_NAME=video_name,
SCENE_NUMBER=scene_num_format % (i + 1),
IMAGE_NUMBER=image_num_format % (j + 1),
FRAME_NUMBER=image_timecode.get_frames()),
self.image_extension)
self.image_filenames[i].append(file_path)
cv2.imwrite(
get_and_create_path(
file_path,
output_dir if output_dir is not None else self.output_directory),
frame_im, imwrite_param)
else:
completed = False
break
if progress_bar:
progress_bar.update(1)
if not completed:
logging.error('Could not generate all output images.')
import pathlib
from os import system, remove
from string import Template
UNIT_TEMPLATE = Template('''
[Unit]
Description=$packagename
After=syslog.target
[Service]
Type=simple
ExecStart=$entrypoint --command listen --filename $filename
SyslogIdentifier=$packagename
StandardOutput=syslog
StandardError=syslog
Restart=always
RestartSec=3
[Install]
WantedBy=multi-user.target
''')
def register(packagename: str, entrypoint: str, filename: str):
unit = UNIT_TEMPLATE.substitute(packagename=packagename, entrypoint=entrypoint, filename=filename)
service = packagename + ".service"
unit_file_fullname = str(pathlib.Path("/", "etc", "systemd", "system", service))
with open(unit_file_fullname, "w") as file:
file.write(unit)
system("sudo systemctl daemon-reload")
system("sudo systemctl enable " + service)
def current_view_html(self, body=None):
""" This func uses session's 'current_view' attribute to render the html
for that view.
A view's HTML does NOT include header/footer type elements. Rather, it
involes a container element that contains all the user controls, and
then, outside of/beneath the container element, all of the 'modal'
UI stuff that a user might need in a view.
In this method then, we use the current view to figure out what to
put in the container and what to append to the container.
The whole thing is decorated in a function that captures failures and
asks users to report them (and kills their session, logging them out, so
they don't get stuck in a state where they can't stop re-creating the
error, etc.
"""
# set the current view and settlement, if possible
self.get_current_view()
if not hasattr(self, "Settlement") or self.Settlement is None:
self.set_current_settlement()
include_ui_templates = True
# start the container
output = html.meta.start_container
# now get us some HTML
if self.current_view == "dashboard":
body = "dashboard"
include_ui_templates = False
output += html.get_template('dashboard')
elif self.current_view == "new_settlement":
body = 'create_new_settlement'
output += html.get_template('new_settlement.html')
elif self.current_view == "view_campaign":
body = 'view_campaign_summary'
output += html.get_template('campaign_summary')
elif self.current_view == "view_settlement":
body = 'view_settlement_sheet'
output += html.get_template('settlement_sheet')
elif self.current_view == "view_survivor":
body = 'view_survivor_sheet'
output += html.get_template('survivor_sheet')
else:
self.logger.error("[%s] requested unhandled view '%s'" %
(self.User, self.current_view))
raise Exception("Unknown View!")
# now close the container
output += html.meta.close_container
# add UI templates
if include_ui_templates:
for t in settings.get('application', 'ui_templates').split(','):
output += html.get_template(t.strip())
# finally, do a single, monster variable substitution pass:
output = Template(output).safe_substitute(
api_url=api.get_api_url(),
application_version=settings.get("application", "version"),
application_age_rough_years=(datetime.now() -
datetime(2015, 11, 10)).days / 365,
user_id=self.User.user['_id'],
user_login=self.User.user["login"],
settlement_id=self.session['current_settlement'],
survivor_id=self.session.get('current_asset', None),
blog_api_key=settings_private.get('api', 'blog_api_key'),
)
return output, body
DEFAULT_LOG_CONFIG = Template("""
version: 1
formatters:
precise:
format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - \
%(request)s - %(message)s'
filters:
context:
(): synapse.util.logcontext.LoggingContextFilter
request: ""
handlers:
file:
class: logging.handlers.RotatingFileHandler
formatter: precise
filename: ${log_file}
maxBytes: 104857600
backupCount: 10
filters: [context]
encoding: utf8
console:
class: logging.StreamHandler
formatter: precise
filters: [context]
loggers:
synapse:
level: INFO
synapse.storage.SQL:
# beware: increasing this to DEBUG will make synapse log sensitive
# information such as access tokens.
level: INFO
root:
level: INFO
handlers: [file, console]
""")
def gather_instance_info(self):
predicate_query = Template("""
prefix owl: <http://www.w3.org/2002/07/owl#>
prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
prefix xsd: <http://www.w3.org/2001/XMLSchema#>
prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
prefix gist: <https://ontologies.semanticarts.com/gist/>
prefix skos: <http://www.w3.org/2004/02/skos/core#>
select distinct ?predicate ?label ?type where {
{
select distinct ?predicate where {
?s ?predicate ?o
FILTER(?predicate NOT IN (rdf:type, rdfs:label, skos:prefLabel, skos:altLabel, skos:definition))
}
}
FILTER (!STRSTARTS(STR(?predicate), 'http://www.w3.org/2002/07/owl#'))
FILTER (!STRSTARTS(STR(?predicate), 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'))
FILTER (!STRSTARTS(STR(?predicate), 'http://www.w3.org/2000/01/rdf-schema#'))
OPTIONAL {
?predicate skos:prefLabel|rdfs:label ?label
FILTER(lang(?label) = '$language' || lang(?label) = '')
}
OPTIONAL {
values ?type { owl:DatatypeProperty owl:ObjectProperty }
?predicate a ?type
}
}
""").substitute(language=self.label_lang)
self.node_data = {}
if self.repo:
all_predicates = list(self.remote_select_query(predicate_query))
else:
self.data = Graph()
for file_path in self.files:
filename = os.path.basename(file_path)
logging.debug('Parsing %s for documentation', filename)
self.data.parse(file_path, format=guess_format(file_path))
all_predicates = list(self.local_select_query(predicate_query))
hidden_predicates = set(predicate['predicate']
for predicate in all_predicates
if self.hidden(predicate['predicate']))
logging.debug("Hiding predicates: %s", hidden_predicates)
all_predicates = [
predicate for predicate in all_predicates
if predicate['predicate'] not in hidden_predicates
]
if not all_predicates:
logging.warning('No interesting predicates found in %s', self.repo
or ' specified files')
return
self.build_class_hierarchy()
for count, predicate_row in enumerate(all_predicates):
predicate = predicate_row['predicate']
predicate_str = predicate_row['label'] if predicate_row.get('label') \
else self.strip_uri(predicate)
self.print_progress_bar(count,
len(all_predicates),
prefix='Processing predicates:',
suffix=predicate_str + ' ' * 20,
length=50)
pre_time = perf_counter()
query_text = self.create_predicate_query(predicate,
predicate_row.get('type'),
self.limit)
predicate_usage = list(self.select_query(query_text))
logging.debug("%s items returned for %s", len(predicate_usage),
predicate)
for usage in predicate_usage:
if 'src' not in usage or usage['src'] is None or int(
usage.get('num', 0)) < self.threshold:
continue
self.record_predicate_usage(predicate, predicate_str, usage)
logging.debug("Fetching %s took %d seconds", str(predicate_row),
perf_counter() - pre_time)
self.print_progress_bar(len(all_predicates),
len(all_predicates),
prefix='Processing predicates:',
suffix='Complete',
length=50)
self.prune_for_inheritance()
if self.show_shacl:
self.add_shacl_coloring()
def build_class_hierarchy(self):
inheritance_query = Template("""
prefix owl: <http://www.w3.org/2002/07/owl#>
prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
prefix skos: <http://www.w3.org/2004/02/skos/core#>
select distinct ?class ?c_label ?parent ?p_label where {
{
?class rdfs:subClassOf ?parent .
}
UNION
{
?class (owl:equivalentClass|rdfs:subClassOf)/(owl:unionOf|owl:intersectionOf)/rdf:rest*/rdf:first ?parent .
?parent a owl:Class
}
filter (!isblank(?class) && !isblank(?parent))
OPTIONAL {
?class rdfs:label|skos:prefLabel ?c_label
FILTER(lang(?c_label) = '$language' || lang(?c_label) = '')
}
OPTIONAL {
?parent rdfs:label|skos:prefLabel ?p_label
FILTER(lang(?p_label) = '$language' || lang(?p_label) = '')
}
}
""").substitute(language=self.label_lang)
if self.repo:
parents = list(self.remote_select_query(inheritance_query))
else:
parents = list(self.local_select_query(inheritance_query))
for inheritance_info in parents:
self.superclasses[inheritance_info['class']].add(
inheritance_info['parent'])
self.class_names[inheritance_info['class']] = \
inheritance_info.get('c_label') or self.strip_uri(inheritance_info['class'])
self.class_names[inheritance_info['parent']] = \
inheritance_info.get('p_label') or self.strip_uri(inheritance_info['parent'])
# Determine evaluation order, root classes to leaves
remaining_classes = set(self.superclasses.keys())
root_classes = set(parent
for cls, parents in self.superclasses.items()
for parent in parents
if parent not in remaining_classes)
eval_order = list(root_classes)
while remaining_classes:
next_set = set(cls for cls in remaining_classes if all(
parent in eval_order for parent in self.superclasses[cls]))
# Make superclasses transitive
for cls in next_set:
parents = set(self.superclasses[cls])
for parent in parents:
self.superclasses[cls].update(
self.superclasses.get(parent, set()))
eval_order.extend(next_set)
remaining_classes.difference_update(next_set)
logging.debug(
'Inheritance evaluation order:\n%s', "\n".join(
f"\t{self.strip_uri(cls)}: {list(self.strip_uri(sup) for sup in self.superclasses[cls])}"
for cls in eval_order))
self.inheritance = eval_order
class_query = self.create_class_count_query(self.limit)
if self.repo:
class_counts = list(self.remote_select_query(class_query))
else:
class_counts = list(self.local_select_query(class_query))
for instance_info in class_counts:
self.class_counts[self.deepest_class(instance_info['src'])] += int(
instance_info['num'])
def read_template(name, values):
t = Template(
file(resource_filename('canari.resources.template',
'%s.plate' % name)).read())
return t.substitute(**values)
def get_data_at_sites(self, data_path, sites, site_idx, period):
self.data = {}
for i, v in enumerate(cfg.SM_MODEL_VARNAMES):
csv = Template(data_path).substitute(v=v)
df = self.load_from_csv(csv)
self.data[i] = extract_sites(df, sites, site_idx, period)
soapenv11_namespace = 'http://schemas.xmlsoap.org/soap/envelope/'
soapenv12_namespace = 'http://www.w3.org/2003/05/soap-envelope'
wsse_namespace = 'http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd'
wsu_namespace = 'http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd'
common_namespaces = {
'soapenv': soapenv11_namespace,
'wsse': wsse_namespace,
'wsu': wsu_namespace,
'zato': zato_namespace
}
soap_doc = Template(
"""<soap:Envelope xmlns:soap='%s'><soap:Body>$body</soap:Body></soap:Envelope>"""
% soapenv11_namespace)
soap_body_path = '/soapenv:Envelope/soapenv:Body'
soap_body_xpath = etree.XPath(soap_body_path, namespaces=common_namespaces)
soap_fault_path = '/soapenv:Envelope/soapenv:Body/soapenv:Fault'
soap_fault_xpath = etree.XPath(soap_fault_path, namespaces=common_namespaces)
wsse_password_type_text = 'http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText'
supported_wsse_password_types = (wsse_password_type_text, )
wsse_username_path = '/soapenv:Envelope/soapenv:Header/wsse:Security/wsse:UsernameToken/wsse:Username'
wsse_username_xpath = etree.XPath(wsse_username_path,
namespaces=common_namespaces)
def translate_path(self, path_template, **kwargs):
context = kwargs
context['account_code'] = self.account_code
template = Template(path_template)
return template.substitute(**kwargs)
req = Request()
f = req.form
value_dic = {'message': '', 'title': '', 'url': '', 'bookmarks': ''}
if f.has_key('post'):
if not f.getvalue('title', '') or not f.getvalue('url', ''):
value_dic['message'] = u'タイトルとURLは必須項目です'
value_dic['title'] = unicode(f.getvalue('title', ''), 'utf-8',
'ignore')
value_dic['url'] = f.getvalue('url', '')
else:
cur.execute("""INSERT INTO bookmark(title, url) VALUES (?, ?)""",
(unicode(f.getvalue('title', ''), 'utf-8',
'ignore'), f.getvalue('url', '')))
con.commit()
listbody = ''
cur.execute("SELECT title, url FROM bookmark")
for item in cur.fetchall():
listbody += """<dt>%s</dt><dd>%s</dd>\n""" % (item)
listbody = """<ul>\n%s</ul>""" % listbody
value_dic['bookmarks'] = listbody
res = Response()
f = open(path.join(path.dirname(__file__), 'bookmarkform.html'))
t = Template(unicode(f.read(), 'utf-8', 'ignore'))
body = t.substitute(value_dic)
res.set_body(body)
print res
def get_output(data):
g = defaultdict(str, [(k, jinja2.escape(v)) for k, v in data.iteritems()])
required = ['mimetype', 'appID', 'appVersion', 'clientOS', 'chromeLocale']
# Some defaults we override depending on what we find below.
plugin = dict(mimetype='-1', name='-1', guid='-1', version='',
iconUrl='', XPILocation='', InstallerLocation='',
InstallerHash='', InstallerShowsUI='',
manualInstallationURL='', licenseURL='',
needsRestart='true')
# Special case for mimetype if they are provided.
plugin['mimetype'] = g['mimetype'] or '-1'
output = Template(xml_template)
for s in required:
if s not in data:
# A sort of 404, matching what was returned in the original PHP.
return output.substitute(plugin)
# Figure out what plugins we've got, and what plugins we know where
# to get.
# Begin our huge and embarrassing if-else statement.
if (g['mimetype'] in ['application/x-shockwave-flash',
'application/futuresplash'] and
re.match(flash_re, g['clientOS'])):
# Tell the user where they can go to get the installer.
plugin.update(
name='Adobe Flash Player',
manualInstallationURL='http://www.adobe.com/go/getflashplayer')
# Offer Windows users a specific flash plugin installer instead.
# Don't use a https URL for the license here, per request from
# Macromedia.
if g['clientOS'].startswith('Win'):
plugin.update(
guid='{4cfaef8a-a6c9-41a0-8e6f-967eb8f49143}',
XPILocation='',
iconUrl='http://fpdownload2.macromedia.com/pub/flashplayer/current/fp_win_installer.ico',
needsRestart='false',
InstallerShowsUI='true',
version='13.0.0.214',
InstallerHash='sha256:3429f62b63ea7e06298231be7a2cf80b15745ba09442a952cbf23942e1aeba42',
InstallerLocation='http://download.macromedia.com/pub/flashplayer/pdc/fp_pl_pfs_installer.exe')
elif (g['mimetype'] == 'application/x-director' and
g['clientOS'].startswith('Win')):
plugin.update(
name='Adobe Shockwave Player',
manualInstallationURL='http://get.adobe.com/shockwave/otherversions')
# Even though the shockwave installer is not a silent installer, we
# need to show its EULA here since we've got a slimmed down
# installer that doesn't do that itself.
if g['chromeLocale'] != 'ja-JP':
plugin.update(
licenseURL='http://www.adobe.com/go/eula_shockwaveplayer')
else:
plugin.update(
licenseURL='http://www.adobe.com/go/eula_shockwaveplayer_jp')
plugin.update(
guid='{45f2a22c-4029-4209-8b3d-1421b989633f}',
XPILocation='',
version='12.1.1.151',
InstallerHash='sha256:22105e72f390b72ee6120f43d83eec3d85f38df32d3a5d6b3d18dcf0cc46dbf7',
InstallerLocation='http://fpdownload.macromedia.com/pub/shockwave/default/english/win95nt/latest/Shockwave_Installer_FF.exe',
manualInstallationURL='http://get.adobe.com/shockwave/otherversions',
needsRestart='false',
InstallerShowsUI='false')
elif (g['mimetype'] in ['audio/x-pn-realaudio-plugin',
'audio/x-pn-realaudio'] and
re.match(r'^(Win|Linux|PPC Mac OS X)', g['clientOS'])):
plugin.update(
name='Real Player',
version='10.5',
manualInstallationURL='http://www.real.com')
if g['clientOS'].startswith('Win'):
plugin.update(
XPILocation='http://forms.real.com/real/player/download.html?type=firefox',
guid='{d586351c-cb55-41a7-8e7b-4aaac5172d39}')
else:
plugin.update(
guid='{269eb771-59de-4702-9209-ca97ce522f6d}')
elif (re.match(quicktime_re, g['mimetype']) and
re.match(r'^(Win|PPC Mac OS X)', g['clientOS'])):
# Well, we don't have a plugin that can handle any of those
# mimetypes, but the Apple Quicktime plugin can. Point the user to
# the Quicktime download page.
plugin.update(
name='Apple Quicktime',
guid='{a42bb825-7eee-420f-8ee7-834062b6fefd}',
InstallerShowsUI='true',
manualInstallationURL='http://www.apple.com/quicktime/download/')
elif (re.match(java_re, g['mimetype']) and
re.match(r'^(Win|Linux|PPC Mac OS X)', g['clientOS'])):
# We serve up the Java plugin for the following mimetypes:
#
# application/x-java-vm
# application/x-java-applet;jpi-version=1.5
# application/x-java-bean;jpi-version=1.5
# application/x-java-applet;version=1.3
# application/x-java-bean;version=1.3
# application/x-java-applet;version=1.2.2
# application/x-java-bean;version=1.2.2
# application/x-java-applet;version=1.2.1
# application/x-java-bean;version=1.2.1
# application/x-java-applet;version=1.4.2
# application/x-java-bean;version=1.4.2
# application/x-java-applet;version=1.5
# application/x-java-bean;version=1.5
# application/x-java-applet;version=1.3.1
# application/x-java-bean;version=1.3.1
# application/x-java-applet;version=1.4
# application/x-java-bean;version=1.4
# application/x-java-applet;version=1.4.1
# application/x-java-bean;version=1.4.1
# application/x-java-applet;version=1.2
# application/x-java-bean;version=1.2
# application/x-java-applet;version=1.1.3
# application/x-java-bean;version=1.1.3
# application/x-java-applet;version=1.1.2
# application/x-java-bean;version=1.1.2
# application/x-java-applet;version=1.1.1
# application/x-java-bean;version=1.1.1
# application/x-java-applet;version=1.1
# application/x-java-bean;version=1.1
# application/x-java-applet
# application/x-java-bean
#
#
# We don't want to link users directly to the Java plugin because
# we want to warn them about ongoing security problems first. Link
# to SUMO.
plugin.update(
name='Java Runtime Environment',
manualInstallationURL='https://support.mozilla.org/kb/use-java-plugin-to-view-interactive-content',
needsRestart='false',
guid='{fbe640ef-4375-4f45-8d79-767d60bf75b8}')
elif (g['mimetype'] in ['application/pdf', 'application/vnd.fdf',
'application/vnd.adobe.xfdf',
'application/vnd.adobe.xdp+xml',
'application/vnd.adobe.xfd+xml'] and
re.match(r'^(Win|PPC Mac OS X|Linux(?! x86_64))', g['clientOS'])):
plugin.update(
name='Adobe Acrobat Plug-In',
guid='{d87cd824-67cb-4547-8587-616c70318095}',
manualInstallationURL='http://www.adobe.com/products/acrobat/readstep.html')
elif (g['mimetype'] == 'application/x-mtx' and
re.match(r'^(Win|PPC Mac OS X)', g['clientOS'])):
plugin.update(
name='Viewpoint Media Player',
guid='{03f998b2-0e00-11d3-a498-00104b6eb52e}',
manualInstallationURL='http://www.viewpoint.com/pub/products/vmp.html')
elif re.match(wmp_re, g['mimetype']):
# We serve up the Windows Media Player plugin for the following
# mimetypes:
#
# application/asx
# application/x-mplayer2
# audio/x-ms-wax
# audio/x-ms-wma
# video/x-ms-asf
# video/x-ms-asf-plugin
# video/x-ms-wm
# video/x-ms-wmp
# video/x-ms-wmv
# video/x-ms-wmx
# video/x-ms-wvx
#
# For all windows users who don't have the WMP 11 plugin, give them
# a link for it.
if g['clientOS'].startswith('Win'):
plugin.update(
name='Windows Media Player',
version='11',
guid='{cff1240a-fd24-4b9f-8183-ccd96e5300d0}',
manualInstallationURL='http://port25.technet.com/pages/windows-media-player-firefox-plugin-download.aspx')
# For OSX users -- added Intel to this since flip4mac is a UB.
# Contact at MS was okay w/ this, plus MS points to this anyway.
elif re.match(r'^(PPC|Intel) Mac OS X', g['clientOS']):
plugin.update(
name='Flip4Mac',
version='2.1',
guid='{cff0240a-fd24-4b9f-8183-ccd96e5300d0}',
manualInstallationURL='http://www.flip4mac.com/wmv_download.htm')
elif (g['mimetype'] == 'application/x-xstandard' and
re.match(r'^(Win|PPC Mac OS X)', g['clientOS'])):
plugin.update(
name='XStandard XHTML WYSIWYG Editor',
guid='{3563d917-2f44-4e05-8769-47e655e92361}',
iconUrl='http://xstandard.com/images/xicon32x32.gif',
XPILocation='http://xstandard.com/download/xstandard.xpi',
InstallerShowsUI='false',
manualInstallationURL='http://xstandard.com/download/',
licenseURL='http://xstandard.com/license/')
elif (g['mimetype'] == 'application/x-dnl' and
g['clientOS'].startswith('Win')):
plugin.update(
name='DNL Reader',
guid='{ce9317a3-e2f8-49b9-9b3b-a7fb5ec55161}',
version='5.5',
iconUrl='http://digitalwebbooks.com/reader/dwb16.gif',
XPILocation='http://digitalwebbooks.com/reader/xpinst.xpi',
InstallerShowsUI='false',
manualInstallationURL='http://digitalwebbooks.com/reader/')
elif (g['mimetype'] == 'application/x-videoegg-loader' and
g['clientOS'].startswith('Win')):
plugin.update(
name='VideoEgg Publisher',
guid='{b8b881f0-2e07-11db-a98b-0800200c9a66}',
iconUrl='http://videoegg.com/favicon.ico',
XPILocation='http://update.videoegg.com/Install/Windows/Initial/VideoEggPublisher.xpi',
InstallerShowsUI='true',
manualInstallationURL='http://www.videoegg.com/')
elif (g['mimetype'] == 'video/vnd.divx' and
g['clientOS'].startswith('Win')):
plugin.update(
name='DivX Web Player',
guid='{a8b771f0-2e07-11db-a98b-0800200c9a66}',
iconUrl='http://images.divx.com/divx/player/webplayer.png',
XPILocation='http://download.divx.com/player/DivXWebPlayer.xpi',
InstallerShowsUI='false',
licenseURL='http://go.divx.com/plugin/license/',
manualInstallationURL='http://go.divx.com/plugin/download/')
elif (g['mimetype'] == 'video/vnd.divx' and
re.match(r'^(PPC|Intel) Mac OS X', g['clientOS'])):
plugin.update(
name='DivX Web Player',
guid='{a8b771f0-2e07-11db-a98b-0800200c9a66}',
iconUrl='http://images.divx.com/divx/player/webplayer.png',
XPILocation='http://download.divx.com/player/DivXWebPlayerMac.xpi',
InstallerShowsUI='false',
licenseURL='http://go.divx.com/plugin/license/',
manualInstallationURL='http://go.divx.com/plugin/download/')
# End ridiculously huge and embarrassing if-else block.
return output.substitute(plugin)
def create_predicate_query(self, predicate, predicate_type, limit):
if predicate_type == str(OWL.ObjectProperty):
type_query = """
prefix owl: <http://www.w3.org/2002/07/owl#>
prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
prefix xsd: <http://www.w3.org/2001/XMLSchema#>
prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
prefix gist: <https://ontologies.semanticarts.com/gist/>
prefix skos: <http://www.w3.org/2004/02/skos/core#>
select ?src ?tgt (COUNT(?src) as ?num) where {
{
select (group_concat(?src_c;separator=' ') as ?src) (group_concat(?tgt_c;separator=' ') as ?tgt) where {
$pattern
FILTER(!ISBLANK(?s))
?s a ?src_c .
FILTER (!STRSTARTS(STR(?src_c), 'http://www.w3.org/2002/07/owl#'))
?o a ?tgt_c .
} group by ?s ?o LIMIT $limit
}
} group by ?src ?tgt
"""
elif predicate_type == str(OWL.DatatypeProperty):
type_query = """
prefix owl: <http://www.w3.org/2002/07/owl#>
prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
prefix xsd: <http://www.w3.org/2001/XMLSchema#>
prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
prefix gist: <https://ontologies.semanticarts.com/gist/>
prefix skos: <http://www.w3.org/2004/02/skos/core#>
select ?src ?dt (COUNT(?src) as ?num) where {
{
select (group_concat(?src_c;separator=' ') as ?src) (SAMPLE(COALESCE(?dtype, xsd:string)) as ?dt) where {
$pattern
FILTER(!ISBLANK(?s) && ISLITERAL(?o))
?s a ?src_c .
FILTER (!STRSTARTS(STR(?src_c), 'http://www.w3.org/2002/07/owl#'))
BIND(DATATYPE(?o) as ?dtype) .
} group by ?s LIMIT $limit
}
} group by ?src ?dt
"""
else:
type_query = """
prefix owl: <http://www.w3.org/2002/07/owl#>
prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
prefix xsd: <http://www.w3.org/2001/XMLSchema#>
prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
prefix gist: <https://ontologies.semanticarts.com/gist/>
prefix skos: <http://www.w3.org/2004/02/skos/core#>
select ?src
?tgt
?dt
?num
where {
{
{
select ?src ?tgt (COUNT(?src) as ?num) where {
{
select (group_concat(?src_c;separator=' ') as ?src)
(group_concat(?tgt_c;separator=' ') as ?tgt) where {
$pattern
FILTER(!ISBLANK(?s))
?s a ?src_c .
FILTER (!STRSTARTS(STR(?src_c), 'http://www.w3.org/2002/07/owl#'))
?o a ?tgt_c .
} group by ?s ?o LIMIT $limit
}
} group by ?src ?tgt
}
}
UNION
{
{
select ?src ?dt (COUNT(?src) as ?num) where {
{
select (group_concat(?src_c;separator=' ') as ?src)
(SAMPLE(COALESCE(?dtype, xsd:string)) as ?dt) where {
$pattern
FILTER(!ISBLANK(?s) && ISLITERAL(?o))
?s a ?src_c .
FILTER (!STRSTARTS(STR(?src_c), 'http://www.w3.org/2002/07/owl#'))
BIND(DATATYPE(?o) as ?dtype) .
} group by ?s LIMIT $limit
}
} group by ?src ?dt
}
}
}
"""
query_text = Template(type_query).substitute(
pattern=self.filtered_graph_pattern(predicate), limit=limit)
return query_text
alpha,
"BETA":
beta,
"WRAP":
wrap,
"MALLOC":
malloc,
"SYNC": [0, 1][0 != sync],
"JIT": [0, 1][0 != jit],
"LIBXSMM_OFFLOAD_BUILD":
["", "\n#define LIBXSMM_OFFLOAD_BUILD"][0 != offload],
"MNK_PREPROCESSOR_LIST":
"",
}
template = Template(open(filename, "r").read())
if fnmatch.fnmatch(filename, "*.h*"):
if mnklist:
first = mnklist[0]
for mnk in mnklist:
mnkstr = "_".join(map(str, mnk))
if mnk != first:
substitute["MNK_PREPROCESSOR_LIST"] += "\n"
if 2 != precision:
substitute["MNK_PREPROCESSOR_LIST"] += (
"#define LIBXSMM_SMM_" + mnkstr)
if mnk != first or 0 == precision:
substitute["MNK_PREPROCESSOR_LIST"] += "\n"
if 1 != precision:
substitute["MNK_PREPROCESSOR_LIST"] += (
"#define LIBXSMM_DMM_" + mnkstr)
def getTemplate(input):
return Template(str(input))
import smtplib
from email.message import EmailMessage
from pathlib import Path
from string import Template
html = Template(Path('index.html').read_text())
email = EmailMessage()
email['from'] = 'W3bh4ck'
email['to'] = '*****@*****.**'
email['subject'] = 'Email subject'
email.set_content(html.substitute({'name': 'uche'}), 'html')
with smtplib.SMTP(host='smtp.gmail.com', port=587) as smtp:
smtp.ehlo()
smtp.starttls()
smtp.login('source email', 'password')
smtp.send_message(email)
print('email sent')
def main(argv=None):
parser = argparse.ArgumentParser()
parser.add_argument('-t',
'--tag',
help='the SapMachine tag',
metavar='TAG',
required=True)
parser.add_argument(
'-d',
'--dual',
help='this is going to be a dual architecture cask (x64 and aarch64)',
action='store_true',
default=False)
args = parser.parse_args()
work_dir = join(os.getcwd(), 'cask_work')
utils.remove_if_exists(work_dir)
os.makedirs(work_dir)
raw_tag = args.tag
sapMachineTag = SapMachineTag.from_string(raw_tag)
if sapMachineTag is None:
print(str.format("Tag {0} seems to be invalid. Aborting...", args.tag))
sys.exit(1)
os_name = 'osx' if sapMachineTag.get_major() < 17 or (
sapMachineTag.get_major() == 17 and sapMachineTag.get_update() is None
and sapMachineTag.get_build_number() < 21) else 'macos'
prerelease = not sapMachineTag.is_ga()
if prerelease:
jdk_cask_file_name = str.format('sapmachine{0}-ea-jdk.rb',
sapMachineTag.get_major())
jre_cask_file_name = str.format('sapmachine{0}-ea-jre.rb',
sapMachineTag.get_major())
cask_tag = str.format('{0}-ea', sapMachineTag.get_major())
cask_version = str.format(
'{0},{1}', sapMachineTag.get_version_string_without_build(),
sapMachineTag.get_build_number())
ruby_version = 'version.before_comma'
ea_ext = '-ea.'
url_version1 = '#{version.before_comma}%2B#{version.after_comma}'
url_version2 = '#{version.before_comma}-ea.#{version.after_comma}'
else:
jdk_cask_file_name = str.format('sapmachine{0}-jdk.rb',
sapMachineTag.get_major())
jre_cask_file_name = str.format('sapmachine{0}-jre.rb',
sapMachineTag.get_major())
cask_tag = str.format('{0}', sapMachineTag.get_major())
cask_version = str.format(
'{0}', sapMachineTag.get_version_string_without_build())
ruby_version = 'version'
ea_ext = '.'
url_version1 = '#{version}'
url_version2 = '#{version}'
if args.dual:
try:
aarch_jdk_sha_url, aarch_jre_sha_url = utils.get_asset_url(
raw_tag, os_name + '-aarch64', '.sha256.dmg.txt')
intel_jdk_sha_url, intel_jre_sha_url = utils.get_asset_url(
raw_tag, os_name + '-x64', '.sha256.dmg.txt')
except Exception as e:
print('Not both platforms ready yet')
sys.exit(0)
aarch_jdk_sha, code1 = utils.download_asset(aarch_jdk_sha_url)
aarch_jre_sha, code2 = utils.download_asset(aarch_jre_sha_url)
intel_jdk_sha, code3 = utils.download_asset(intel_jdk_sha_url)
intel_jre_sha, code4 = utils.download_asset(intel_jre_sha_url)
if code1 != 200 or code2 != 200 or code3 != 200 or code4 != 200:
print('Download failed')
sys.exit(1)
aarch_jdk_sha = aarch_jdk_sha.split(' ')[0]
aarch_jre_sha = aarch_jre_sha.split(' ')[0]
intel_jdk_sha = intel_jdk_sha.split(' ')[0]
intel_jre_sha = intel_jre_sha.split(' ')[0]
jdk_cask_content = Template(duplex_cask_template).substitute(
CASK_TAG=cask_tag,
IMAGE_TYPE='jdk',
CASK_VERSION=cask_version,
URL_VERSION1=url_version1,
URL_VERSION2=url_version2,
OS_NAME=os_name,
INTELSHA256=intel_jdk_sha,
AARCHSHA256=aarch_jdk_sha,
RUBY_VERSION=ruby_version,
EA_EXT=ea_ext)
jre_cask_content = Template(duplex_cask_template).substitute(
CASK_TAG=cask_tag,
IMAGE_TYPE='jre',
CASK_VERSION=cask_version,
URL_VERSION1=url_version1,
URL_VERSION2=url_version2,
OS_NAME=os_name,
INTELSHA256=intel_jre_sha,
AARCHSHA256=aarch_jre_sha,
RUBY_VERSION=ruby_version,
EA_EXT=ea_ext)
else:
try:
intel_jdk_sha_url, intel_jre_sha_url = utils.get_asset_url(
raw_tag, os_name + '-x64', '.sha256.dmg.txt')
except Exception as e:
print('Asset not found')
sys.exit(1)
intel_jdk_sha, code1 = utils.download_asset(intel_jdk_sha_url)
intel_jre_sha, code2 = utils.download_asset(intel_jre_sha_url)
if code1 != 200 or code2 != 200:
print('Download failed')
sys.exit(1)
intel_jdk_sha = intel_jdk_sha.split(' ')[0]
intel_jre_sha = intel_jre_sha.split(' ')[0]
try:
intel_jdk_url, intel_jre_url = utils.get_asset_url(
raw_tag, os_name + '-x64', '.dmg')
except Exception as e:
print('Asset not found')
sys.exit(1)
jdk_cask_content = Template(cask_template).substitute(
CASK_TAG=cask_tag,
IMAGE_TYPE='jdk',
CASK_VERSION=cask_version,
SHA256=intel_jdk_sha,
URL_VERSION1=url_version1,
URL_VERSION2=url_version2,
OS_NAME=os_name,
RUBY_VERSION=ruby_version,
EA_EXT=ea_ext)
jre_cask_content = Template(cask_template).substitute(
CASK_TAG=cask_tag,
IMAGE_TYPE='jre',
CASK_VERSION=cask_version,
SHA256=intel_jre_sha,
URL_VERSION1=url_version1,
URL_VERSION2=url_version2,
OS_NAME=os_name,
RUBY_VERSION=ruby_version,
EA_EXT=ea_ext)
homebrew_dir = join(work_dir, 'homebrew')
utils.git_clone('github.com/SAP/homebrew-SapMachine', 'master',
homebrew_dir)
jdk_replaced = replace_cask(jdk_cask_file_name, jdk_cask_content,
sapMachineTag, homebrew_dir)
jre_replaced = replace_cask(jre_cask_file_name, jre_cask_content,
sapMachineTag, homebrew_dir)
if jdk_replaced or jre_replaced:
utils.git_push(homebrew_dir)
utils.remove_if_exists(work_dir)
return 0
def make_nmonths_table_from_template(con,
dataset,
date_column,
n_months,
max_dist,
template,
load=False,
columns='all'):
'''
Load inspections table matched with events that happened X months
before. Returns pandas dataframe with the data loaded
'''
#Create a cursor
cur = con.cursor()
#Get the current schema
cur.execute('SELECT current_schema;')
current_schema = cur.fetchone()[0]
#Build the table name
table_name = ('insp2{dataset}_{n_months}months'
'_{max_dist}m').format(dataset=dataset,
n_months=n_months,
max_dist=max_dist)
#Check if table already exists in current schema
#If not, create it
if table_name not in tables_in_schema(con, current_schema):
logger.info(
'Table {} does not exist... Creating it'.format(table_name))
path_to_template = os.path.join(os.environ['ROOT_FOLDER'],
'blight_risk_prediction', 'features',
template)
#Load template with SQL statement
with open(path_to_template, 'r') as f:
sql_script = Template(f.read())
#Replace values in template
sql_script = sql_script.substitute(TABLE_NAME=table_name,
DATASET=dataset,
DATE_COLUMN=date_column,
N_MONTHS=n_months,
MAX_DIST=max_dist)
#Run the code using the connection
#this is going to take a while
cur.execute(sql_script)
#Commit changes to db
con.commit()
#If table created has a geom column which type USER DEFINED,
#delete it, we don't need it here
cols = columns_for_table_in_schema(con, table_name, current_schema)
if ('geom', 'USER-DEFINED') in cols:
#Important: this is not prouction ready since it's
#vulnerable to SQL injection, I haven't found any solution
#to dynamically pass table names as parameters in psycopg2
#it seems like the only solution is to prevent SQL injection
#in the code
q = ('ALTER TABLE {} DROP COLUMN geom').format(table_name)
cur.execute(q)
con.commit()
logger.info('Table {} has a PostGIS column, deleting...'.format(
table_name))
else:
logger.info('Table {} already exists. Skipping...'.format(table_name))
cur.close()
#Load data
e = create_engine(uri)
logger.info('Loading {} month table...'.format(table_name))
if columns == 'all':
#Since the table contains a geom column, you need to subselect columns
#to load otherwise pandas will complain
cols = columns_for_table_in_schema(con, table_name, current_schema)
valid_cols = filter(lambda x: x[1] != 'USER-DEFINED', cols)
cols_to_load = [x[0] for x in valid_cols]
#If the user passed and array in the columns parameter, only
#select those columns
else:
cols_to_load = columns
if load:
df = pd.read_sql_table(table_name,
e,
schema=current_schema,
columns=cols_to_load)
return df
help="password for exchange/outlook",
required=True)
parser.add_argument("-d",
"--deep",
help="Attemp a deep search (takes longer).",
action="store_true")
args = parser.parse_args()
url = args.url
user = args.user
#password = getpass.getpass("Password:"******"&&Dec2016"
xml_template = open("resolvenames_template.xml", "r").read()
xml = Template(xml_template)
rooms = {}
for prefix in args.prefix:
rooms.update(findRooms(prefix))
print "After searching for prefix '" + prefix + "' we found " + str(
len(rooms)) + " rooms."
deep = args.deep
if deep:
symbols = letters + digits
for symbol in symbols:
prefix_deep = prefix + " " + symbol
rooms.update(findRooms(prefix_deep))
def process_input(self):
# type: () -> None
""" Process Input: Processes input video(s) and generates output as per CLI commands.
Run after all command line options/sub-commands have been parsed.
"""
logging.debug('Processing input...')
if not self.options_processed:
logging.debug('Skipping processing, CLI options were not parsed successfully.')
return
self.check_input_open()
assert self.scene_manager.get_num_detectors() >= 0
if self.scene_manager.get_num_detectors() == 0:
logging.error(
'No scene detectors specified (detect-content, detect-threshold, etc...),\n'
' or failed to process all command line arguments.')
return
# Handle scene detection commands (detect-content, detect-threshold, etc...).
self.video_manager.start()
base_timecode = self.video_manager.get_base_timecode()
start_time = time.time()
logging.info('Detecting scenes...')
num_frames = self.scene_manager.detect_scenes(
frame_source=self.video_manager, frame_skip=self.frame_skip,
show_progress=not self.quiet_mode)
# Handle case where video fails with multiple audio tracks (#179).
# TODO: Is there a fix for this? See #179.
if num_frames <= 0:
logging.critical('\n'.join([
'Failed to read any frames from video file. This could be caused'
' by the video having multiple audio tracks. If so, please try'
' removing the audio tracks or muxing to mkv via:'
' ffmpeg -i input.mp4 -c copy -an output.mp4'
'or:'
' mkvmerge -o output.mkv input.mp4'
' For details, see https://pyscenedetect.readthedocs.io/en/latest/faq/']))
return
duration = time.time() - start_time
logging.info('Processed %d frames in %.1f seconds (average %.2f FPS).',
num_frames, duration, float(num_frames)/duration)
# Handle -s/--statsfile option.
if self.stats_file_path is not None:
if self.stats_manager.is_save_required():
with open(self.stats_file_path, 'wt') as stats_file:
logging.info('Saving frame metrics to stats file: %s',
os.path.basename(self.stats_file_path))
self.stats_manager.save_to_csv(
stats_file, base_timecode)
else:
logging.debug('No frame metrics updated, skipping update of the stats file.')
# Get list of detected cuts and scenes from the SceneManager to generate the required output
# files with based on the given commands (list-scenes, split-video, save-images, etc...).
cut_list = self.scene_manager.get_cut_list(base_timecode)
scene_list = self.scene_manager.get_scene_list(base_timecode)
video_paths = self.video_manager.get_video_paths()
video_name = os.path.basename(video_paths[0])
if video_name.rfind('.') >= 0:
video_name = video_name[:video_name.rfind('.')]
# Ensure we don't divide by zero.
if scene_list:
logging.info('Detected %d scenes, average shot length %.1f seconds.',
len(scene_list),
sum([(end_time - start_time).get_seconds()
for start_time, end_time in scene_list]) / float(len(scene_list)))
else:
logging.info('No scenes detected.')
# Handle list-scenes command.
if self.scene_list_output:
scene_list_filename = Template(self.scene_list_name_format).safe_substitute(
VIDEO_NAME=video_name)
if not scene_list_filename.lower().endswith('.csv'):
scene_list_filename += '.csv'
scene_list_path = get_and_create_path(
scene_list_filename,
self.scene_list_directory if self.scene_list_directory is not None
else self.output_directory)
logging.info('Writing scene list to CSV file:\n %s', scene_list_path)
with open(scene_list_path, 'wt') as scene_list_file:
write_scene_list(scene_list_file, scene_list, cut_list)
# Handle `list-scenes`.
if self.print_scene_list:
logging.info("""Scene List:
-----------------------------------------------------------------------
| Scene # | Start Frame | Start Time | End Frame | End Time |
-----------------------------------------------------------------------
%s
-----------------------------------------------------------------------
""", '\n'.join(
[' | %5d | %11d | %s | %11d | %s |' % (
i+1,
start_time.get_frames(), start_time.get_timecode(),
end_time.get_frames(), end_time.get_timecode())
for i, (start_time, end_time) in enumerate(scene_list)]))
if cut_list:
logging.info('Comma-separated timecode list:\n %s',
','.join([cut.get_timecode() for cut in cut_list]))
# Handle save-images command.
if self.save_images:
self._generate_images(scene_list=scene_list, video_name=video_name,
image_name_template=self.image_name_format,
output_dir=self.image_directory)
# Handle export-html command.
if self.export_html:
html_filename = Template(self.html_name_format).safe_substitute(
VIDEO_NAME=video_name)
if not html_filename.lower().endswith('.html'):
html_filename += '.html'
html_path = get_and_create_path(
html_filename,
self.image_directory if self.image_directory is not None
else self.output_directory)
logging.info('Exporting to html file:\n %s:', html_path)
if not self.html_include_images:
self.image_filenames = None
write_scene_list_html(html_path, scene_list, cut_list,
image_filenames=self.image_filenames,
image_width=self.image_width,
image_height=self.image_height)
# Handle split-video command.
if self.split_video:
# Add proper extension to filename template if required.
dot_pos = self.split_name_format.rfind('.')
if self.split_mkvmerge and not self.split_name_format.endswith('.mkv'):
self.split_name_format += '.mkv'
# Don't add if we find an extension between 2 and 4 characters
elif not (dot_pos >= 0) or (
dot_pos >= 0 and not
((len(self.split_name_format) - (dot_pos+1) <= 4 >= 2))):
self.split_name_format += '.mp4'
output_file_prefix = get_and_create_path(
self.split_name_format,
self.split_directory if self.split_directory is not None
else self.output_directory)
mkvmerge_available = is_mkvmerge_available()
ffmpeg_available = is_ffmpeg_available()
if mkvmerge_available and (self.split_mkvmerge or not ffmpeg_available):
if not self.split_mkvmerge:
logging.warning(
'ffmpeg not found, falling back to fast copy mode (split-video -c/--copy).')
split_video_mkvmerge(video_paths, scene_list, output_file_prefix, video_name,
suppress_output=self.quiet_mode or self.split_quiet)
elif ffmpeg_available:
if self.split_mkvmerge:
logging.warning('mkvmerge not found, falling back to normal splitting'
' mode (split-video).')
split_video_ffmpeg(video_paths, scene_list, output_file_prefix,
video_name, arg_override=self.split_args,
hide_progress=self.quiet_mode,
suppress_output=self.quiet_mode or self.split_quiet)
else:
if not (mkvmerge_available or ffmpeg_available):
error_strs = ["ffmpeg/mkvmerge is required for split-video [-c/--copy]."]
else:
error_strs = [
"{EXTERN_TOOL} is required for split-video{EXTRA_ARGS}.".format(
EXTERN_TOOL='mkvmerge' if self.split_mkvmerge else 'ffmpeg',
EXTRA_ARGS=' -c/--copy' if self.split_mkvmerge else '')]
error_strs += ["Install one of the above tools to enable the split-video command."]
error_str = '\n'.join(error_strs)
logging.debug(error_str)
raise click.BadParameter(error_str, param_hint='split-video')
if scene_list:
logging.info('Video splitting completed, individual scenes written to disk.')
_JVPP_IMPL_TEMPLATE = Template("""package $plugin_package;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.Set;
import java.util.logging.Logger;
import java.util.logging.Level;
import io.fd.jvpp.callback.JVppCallback;
import io.fd.jvpp.VppConnection;
import io.fd.jvpp.JVppRegistry;
/**
* <p>Default implementation of JVpp interface.
* <br>It was generated by jvpp_impl_gen.py based on $json_filename.
* <br>(python representation of api file generated by vppapigen)
*/
public final class JVpp${plugin_name}Impl implements $plugin_package.JVpp${plugin_name} {
private final static Logger LOG = Logger.getLogger(JVpp${plugin_name}Impl.class.getName());
private static final java.lang.String LIBNAME = "libjvpp_${plugin_name_underscore}.so";
// FIXME using NativeLibraryLoader makes load fail could not find (WantInterfaceEventsReply).
static {
try {
loadLibrary();
} catch (Exception e) {
LOG.severe("Can't find jvpp jni library: " + LIBNAME);
throw new ExceptionInInitializerError(e);
}
}
private static void loadStream(final InputStream is) throws IOException {
final Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxr-x---");
final Path p = Files.createTempFile(LIBNAME, null, PosixFilePermissions.asFileAttribute(perms));
try {
Files.copy(is, p, StandardCopyOption.REPLACE_EXISTING);
try {
Runtime.getRuntime().load(p.toString());
} catch (UnsatisfiedLinkError e) {
throw new IOException("Failed to load library " + p, e);
}
} finally {
try {
Files.deleteIfExists(p);
} catch (IOException e) {
}
}
}
private static void loadLibrary() throws IOException {
try (final InputStream is = JVpp${plugin_name}Impl.class.getResourceAsStream('/' + LIBNAME)) {
if (is == null) {
throw new IOException("Failed to open library resource " + LIBNAME);
}
loadStream(is);
}
}
private VppConnection connection;
private JVppRegistry registry;
private static native void init0(final JVppCallback callback, final long queueAddress, final int clientIndex);
@Override
public void init(final JVppRegistry registry, final JVppCallback callback, final long queueAddress, final int clientIndex) {
this.registry = java.util.Objects.requireNonNull(registry, "registry should not be null");
this.connection = java.util.Objects.requireNonNull(registry.getConnection(), "connection should not be null");
connection.checkActive();
init0(callback, queueAddress, clientIndex);
}
private static native void close0();
@Override
public void close() {
close0();
}
@Override
public int send(io.fd.jvpp.dto.JVppRequest request) throws io.fd.jvpp.VppInvocationException {
return request.send(this);
}
@Override
public final int controlPing(final io.fd.jvpp.dto.ControlPing controlPing) throws io.fd.jvpp.VppInvocationException {
return registry.controlPing(JVpp${plugin_name}Impl.class);
}
$methods
}
""")
def generate_docstrings_h(self):
template = Template('extern char* $name;')
return '\n\n'.join(
template.substitute(name=decl['docstring_var'])
for decl in chain(self.declarations, self.stateless_declarations)
if 'docstring_var' in decl)
class Neo4jClient:
def __init__(self, neo4j_url, user, password):
self.neo4j_url = neo4j_url
self.neo4j_user = user
self.neo4j_password = password
self.neo4j_driver = None
def connect(self):
self.neo4j_driver = GraphDatabase.driver(self.neo4j_url,
auth=basic_auth(
self.neo4j_user,
self.neo4j_password))
return self.neo4j_driver
def run_in_tx(self, batch, chunk_count=None, dry_run=False):
if not chunk_count:
chunk_count = sys.maxsize
if isinstance(batch, Generator):
it = batch
elif isinstance(batch, Iterable):
def gen():
for j in batch:
yield j
it = gen()
else:
err = "batch_job must be iterable or callable but {0} passed"
err = err.format(type(batch))
logger.error(err)
raise ValueError(err)
if dry_run:
return list(it)
session = self.neo4j_driver.session()
try:
result_set = []
consumed_result = None
more_chunks = True
while more_chunks:
logger.debug('neo4j transaction beginning')
with session.begin_transaction() as tx:
chunk_i = 0
try:
while chunk_i < chunk_count:
# noinspection PyNoneFunctionAssignment
query, params = it.send(consumed_result)
logger.debug(
'chunk %s will run query %s'
'in transaction', chunk_i, query)
result = tx.run(query, params)
consumed_result = list(result)
result_set.append(consumed_result)
chunk_i += 1
except StopIteration:
more_chunks = False
tx.success = True
logger.debug('neo4j transaction committed')
return result_set
finally:
session.close()
import_nodes_template = Template("UNWIND {props} AS map "
"CREATE (n$labels) SET n = map")
def import_nodes(self,
nodes: List[dict],
labels: List[str] = None,
chunk_size: int = 1000):
node_labels = ':{0}'.format(':'.join(labels)) \
if labels else ''
query = self.import_nodes_template.safe_substitute(labels=node_labels)
chunk_count = 1
def batch():
for i in range(0, len(nodes), chunk_size):
logger.debug('starting chunk %s', i)
result = (yield query, dict(props=nodes[i:i + chunk_size]))
logger.debug(result)
result = self.run_in_tx(batch(), chunk_count=chunk_count)
return result
def run_batch_query(self,
query: str,
labels: List[str] = None,
params: List[dict] = None,
chunk_size: int = 1000):
node_labels = ':{0}'.format(':'.join(labels)) \
if labels else ''
query_template = Template("UNWIND {params} AS params " + query)
labeled_query = query_template.safe_substitute(labels=node_labels)
chunk_count = 1
def batch():
for i in range(0, len(params), chunk_size):
logger.debug('starting chunk %s', i)
result = (yield labeled_query,
dict(params=params[i:i + chunk_size]))
logger.debug(result)
result = self.run_in_tx(batch(), chunk_count=chunk_count)
return result
def run_query(self,
query: str,
labels: List[str] = None,
params: dict = None) -> List:
node_labels = ':{0}'.format(':'.join(labels)) \
if labels else ''
query_template = Template(query)
labeled_query = query_template.safe_substitute(labels=node_labels)
logger.debug('will run query %s', labeled_query)
def batch():
yield labeled_query, params
result = self.run_in_tx(batch(), chunk_count=1)
return result[0]
def set_neo4j_password(self, new_password):
"""
sets new password for the neo4j server updating the server
:param new_password: new password to set
:return: boolean: True if update is successful
An excerpt from neo4j documentation on password change API:
Changing the user password
Given that you know the current password, you can ask the server to
change a users password.
You can choose any password you like, as long as it is different from
the current password.
Example request
POST http://localhost:7474/user/neo4j/password
Accept: application/json; charset=UTF-8
Authorization: Basic bmVvNGo6bmVvNGo=
Content-Type: application/json
{
"password" : "secret"
}
Example response
200: OK
"""
value_error_msg = ''
if new_password == self.neo4j_password:
value_error_msg = "New password must not equal old password"
elif not new_password:
value_error_msg = "New password must not be empty"
if value_error_msg:
raise ValueError(value_error_msg)
url = "{host_url}/user/{user_name}/password".format(
host_url=self.neo4j_url, user_name=self.neo4j_user)
headers = {
"Accept": "application/json; charset=UTF-8",
"Content-Type": "application/json"
}
payload = {'password': new_password}
r = requests.post(url,
auth=(self.neo4j_user, self.neo4j_password),
headers=headers,
json=payload)
if r.ok:
self.neo4j_password = new_password
return True
else:
r.raise_for_status()
def appendline(self, list, text, param, value=None):
if value is None:
val = ""
else:
val = self.crepr(param, param[value])
list.append(Template('${doline} $srcline "$srcfile"\n ' + text).safe_substitute(param, v=val, doline=LINEDEBUG, configname=self.name))
psi4_cc_template = Template('''#! Psi4 input generated by QSoME
memory $MEMORY MB # total memory not per thread memory for parallel jobs
molecule acrolein {
$CHARGE $SPIN
# acrolein geometry from MD snapshots
$GEOM
units angstrom # default in psi4
symmetry c1 # no symmetry with embedding potential
noreorient # prevent reorienting molecules
nocom # prevent recentering molecules
}
set {
freeze_core false
#df_basis_scf aug-cc-pvdz-jkfit
#df_basis_cc aug-cc-pvdz-ri
#scf_type df
#cc_type df
scf_type pk
CC_NUM_THREADS $NCORE
}
basis {
# generated by PySCF
$BASIS_TYPE
$BASIS
}
energy('$METHOD')
''')
import os
from string import Template
# Generate conf/nutcracker.yml
target_port = os.environ['TARGET_PORT']
redis_host = os.environ['REDIS_HOST']
redis_port = os.environ.get('REDIS_PORT', 6379)
redis_db = os.environ.get('REDIS_DB', 0)
redis_auth = os.environ['REDIS_AUTH']
nutcracker_yml_tmpl = open(
'/usr/src/twemproxy/conf/templates/nutcracker.yml.tmpl', 'r').read()
nutcracker_yml_body = (Template(nutcracker_yml_tmpl).substitute(
target_port=target_port,
redis_host=redis_host,
redis_port=redis_port,
redis_db=redis_db,
redis_auth=redis_auth))
nutcracker_yml = open('/usr/src/twemproxy/conf/nutcracker.yml', 'w')
nutcracker_yml.write(nutcracker_yml_body)
nutcracker_yml.close()
from string import Template
# The format uses placeholder names formed by $ with valid Python identifiers
# (alphanumeric characters and underscores).
# Surrounding the placeholder with braces allows it to be followed
# by more alphanumeric letters with no intervening spaces.
# Writing $$ creates a single escaped $:
t = Template('${village}folks send $$10 to $cause.')
x = t.substitute(village='Nottingham', cause='the ditch fund')
print(x)
t = Template('Return the $item to $owner.')
d = dict(item='unladen swallow')
# this will raise KeyError: 'owner' exception
# t.substitute(d)
t.safe_substitute(d) # will return 'Return the unladen swallow to $owner.'
