def create_minimal(cls, uris, configloc, schemas, config_password,
pidfile=None):
if _os.path.isdir(configloc) and _os.access(configloc, _os.W_OK):
pass
elif not _os.path.exists(configloc):
try:
_os.mkdir(configloc, 0700)
except OSError as exc:
raise RuntimeError('cannot create config dir {!r}: {}'
.format(configloc, str(exc)))
else:
raise ValueError('invalid config dir {!r}; expecting a writable'
' directory path'
.format(configloc))
if cls._PASSWORD_WITH_SCHEME_RE.match(config_password):
config_password_configvalue = config_password
else:
config_password_salt = _os.urandom(4)
config_password_configvalue = \
'{SSHA}' + _b64encode(_sha1(config_password
+ config_password_salt)
.digest()
+ config_password_salt)
if not pidfile:
if _os.path.isdir(cls._PIDFILE_STD_DIR) \
and _os.access(cls._PIDFILE_STD_DIR, _os.W_OK):
pidfile_dir = cls._PIDFILE_STD_DIR
else:
pidfile_dir = None
pidfile_tmp = _NamedTemporaryFile(dir=pidfile_dir, prefix='slapd-',
suffix='.pid')
with pidfile_tmp:
pass
pidfile = pidfile_tmp.name
configfile = _NamedTemporaryFile(delete=False)
with configfile:
for schema in schemas:
configfile.write('include {}\n'.format(schema))
configfile.write('pidfile {}\n'.format(pidfile))
configfile.write('database config\n')
configfile.write('rootpw {}\n'.format(config_password_configvalue))
service = cls.create_from_configfile(uris=uris,
configfile=configfile.name,
configdir=configloc)
_os.remove(configfile.name)
return service
def visualize(self, layout_dir='LR', display=False):
"""
Create and optionally display an image of the net structure.
:param layout_dir: string in ['LR', 'TB', 'BT'].
Short string for graph layout direction.
:param display: bool.
If set to ``True``, displays the graphic in a window. Press enter
to close it.
:returns: 3D numpy array.
Graphic of the visualization as (H, W, C) image in BGR format.
"""
if _draw is None or _cv2 is None: # pragma: no cover
raise Exception('Drawing is not available!')
else: # pragma: no cover
with _NamedTemporaryFile(mode='w+b', suffix='.png') as tmpfile:
_draw.draw_net_to_file(self.to_pbuf_message(),
tmpfile.name,
rankdir=layout_dir)
result_image = _cv2.imread(tmpfile.name)
assert result_image is not None
if display: # pragma: no cover
_cv2.imshow(self.name, result_image)
_cv2.waitKey(0)
_cv2.destroyWindow(self.name)
return result_image
def visualize(self,
layout_dir='LR',
display=False):
"""
Create and optionally display an image of the net structure.
:param layout_dir: string in ['LR', 'TB', 'BT'].
Short string for graph layout direction.
:param display: bool.
If set to ``True``, displays the graphic in a window. Press enter
to close it.
:returns: 3D numpy array.
Graphic of the visualization as (H, W, C) image in BGR format.
"""
if _draw is None or _cv2 is None:
raise Exception('Drawing is not available!')
with _NamedTemporaryFile(mode='w+b', suffix='.png') as tmpfile:
_draw.draw_net_to_file(self.to_pbuf_message(),
tmpfile.name,
rankdir=layout_dir)
result_image = _cv2.imread(tmpfile.name)
assert result_image is not None
if display:
_cv2.imshow(self.name, result_image)
_cv2.waitKey(0)
_cv2.destroyWindow(self.name)
return result_image
def NamedTemporaryFile(self, *args, **kwargs):
f = tempfile._NamedTemporaryFile(*args, **kwargs)
try:
log.debug(("Opened tempfile %s with NamedTemporaryFile:\n" % f.name) + "".join(traceback.format_stack()))
except AttributeError:
pass
return f
def NamedTemporaryFile(self, *args, **kwargs):
f = tempfile._NamedTemporaryFile(*args, **kwargs)
try:
log.debug(("Opened tempfile %s with NamedTemporaryFile:\n" %
f.name) + "".join(traceback.format_stack()))
except AttributeError:
pass
return f
def _save_state_to_s3(self):
# Dump immutable state data to a config
state = _ConfigParser(allow_no_value=True)
state.optionxform = str
state.add_section(PredictiveService._SERVICE_INFO_SECTION_NAME)
state.set(PredictiveService._SERVICE_INFO_SECTION_NAME, 'Name', self.name)
state.set(PredictiveService._SERVICE_INFO_SECTION_NAME, 'Description', self._description)
state.set(PredictiveService._SERVICE_INFO_SECTION_NAME, 'API Key', self._api_key)
# Save environment, if we have one
if self._environment:
state.add_section(PredictiveService._ENVIRONMENT_SECTION_NAME)
for (key, value) in self._environment._get_state().iteritems():
state.set(PredictiveService._ENVIRONMENT_SECTION_NAME, key, value)
# Save deployment version data to config
state.add_section(PredictiveService._DEPLOYMENT_SECTION_NAME)
current_predictive_objects = _copy(self._all_predictive_objects)
for (model_name, info) in current_predictive_objects.iteritems():
state.set(PredictiveService._DEPLOYMENT_SECTION_NAME, model_name, info['version'])
state.add_section(PredictiveService._PREDICTIVE_OBJECT_DOCSTRING)
for (model_name, info) in current_predictive_objects.iteritems():
state.set(PredictiveService._PREDICTIVE_OBJECT_DOCSTRING, model_name, info['docstring'].encode('string_escape'))
if self._has_state_changed_on_s3():
raise IOError("Can not save changes. The Predictive Service has changed on S3. Please "
"reload from S3.")
# Save any new predictive objects to S3.
for predictive_object_name in self._local_changes:
(predictive_object, po_info) = self._local_changes[predictive_object_name]
if predictive_object: # if this is not a model deletion:
save_path = self._get_predictive_object_save_path(predictive_object_name, po_info['version'])
dependency_path = self._get_dependency_save_path(predictive_object_name, po_info['version'])
predictive_object.save(save_path, dependency_path, self.aws_credentials)
# Update the revision number after we have successfully written all predictive objects
self._revision_number += 1
state.add_section(self._META_SECTION_NAME)
state.set(self._META_SECTION_NAME, 'Revision Number', self._revision_number)
state.set(self._META_SECTION_NAME, 'Schema Version', self._schema_version)
# Write state file to S3
with _NamedTemporaryFile() as temp_file:
state.write(temp_file)
temp_file.flush()
conn = _connect_s3(**self.aws_credentials)
bucket = conn.get_bucket(self._s3_bucket_name, validate=False)
key = _s3_key(bucket)
key.key = self._s3_state_key
key.set_contents_from_filename(temp_file.name)
temp_file.close() # deletes temp file
# Update our state
self._local_changes = {}
self._predictive_objects = dict(zip(current_predictive_objects.keys(),
[{'version':info['version'], 'docstring': info['docstring']} for info in current_predictive_objects.values()]))
def instantiate(self):
"""Create an instantiated net with the current object configuration."""
# Write spec to a temporary file.
with _NamedTemporaryFile(mode="w", suffix=".prototxt") as tmpfile:
tmpfile.write(self.to_prototxt())
tmpfile.flush()
_specification = self.copy()
net = _Net(tmpfile.name, specification=_specification)
return net
def _fill_and_save(file_from, file_to, as_root=False):
with _NamedTemporaryFile() as ff:
_fill_constants_from_settings(file_from, ff.name)
command = 'cp {} {}'.format(ff.name, file_to)
if as_root:
local('sudo ' + command)
else:
local(command)
_message_ok('File {} generated form {}'.format(file_to, file_from))
def pdb2traj(code,
filename=None,
verbose=True,
url="https://files.rcsb.org/download/",
):
r""" Return a :obj:`~mdtraj.Trajectory` from a four-letter PDB code via RSCB PBB lookup
Thinly wraps around :obj:`mdtraj.load_pdb`, printing the corresponding citation.
Will return None if lookup fails
Parameters
----------
code : str
four-letter code, e.g. 3SN6
filename : str, default is None
if str, save to this file,
eventually overwriting
verbose : bool, default is False
url : str, default is 'https://files.rcsb.org/download'
base URL for lookups
Returns
-------
traj : :obj:`~mdtraj.Trajectory` or None
"""
url1 = "%s/%s.pdb" % (url.strip("/"),code.strip("/"))
#TODO use print_v elsewhere
print_v = lambda s, **kwargs: [print(s, **kwargs) if verbose else None][0]
print_v("Checking %s" % url1, end=" ...", flush=True)
geom = None
try:
a = _requests.get(url1)
if a.ok:
with _NamedTemporaryFile(mode="w", suffix=".pdb") as f:
f.writelines(a.text)
print_v("done")
geom = _load_pdb(f.name) #TODO use string.IO
if filename is not None:
print_v("Saving to %s..." % filename, end="", flush=True)
if filename.lower().endswith(".pdb"):
_copy(f.name, filename)
else:
geom.save(filename)
print_v(filename)
else:
raise _URLError(a.text,filename=url1)
except (_HTTPError, _URLError) as e:
print(url1, ":", e)
if geom is not None:
pdb2ref(code)
return geom
def NamedTemporaryFile():
global temp_files
if sys.platform == "linux" or sys.platform == "linux2":
OPT_DELETE = True
elif sys.platform == "darwin":
OPT_DELETE = True
elif sys.platform == "win32":
OPT_DELETE = False
tempFile = _NamedTemporaryFile(delete=OPT_DELETE)
temp_files.append(tempFile.name)
return tempFile
def NamedTemporaryFile():
global temp_files
if sys.platform == "linux" or sys.platform == "linux2":
OPT_DELETE = True
elif sys.platform == "darwin":
OPT_DELETE= True
elif sys.platform == "win32":
OPT_DELETE = False
tempFile = _NamedTemporaryFile(delete = OPT_DELETE)
temp_files.append(tempFile.name)
return tempFile
def instantiate(self):
"""Create an instantiated net with the current object configuration."""
# Write spec to a temporary file.
with _NamedTemporaryFile(mode='w', suffix='.prototxt') as tmpfile:
tmpfile.write(self.to_prototxt())
tmpfile.flush()
_specification = self.copy()
net = _Net(tmpfile.name,
mode=self.phase,
specification=_specification)
return net
def Get_caffe_solver_instance(cls, solver_parameter_dict, net):
"""Get a caffe solver object."""
# now we actually create a instance of the solver
solver_message = _caffe_pb2.SolverParameter(**solver_parameter_dict)
messagestr = _gprototext.MessageToString(solver_message)
with _NamedTemporaryFile(mode='w+b', suffix='.prototxt') as tmpfile:
tmpfile.write(bytes(messagestr.encode('utf-8')))
tmpfile.flush()
return cls.Get_caffe_solver_class(
solver_parameter_dict['solver_type'])._caffe_solver_class(
tmpfile.name, net)
raise Exception('could not initialize solver class')
def Get_caffe_solver_instance(cls, solver_parameter_dict, net):
"""Get a caffe solver object."""
# now we actually create a instance of the solver
solver_message = _caffe_pb2.SolverParameter(**solver_parameter_dict)
messagestr = _gprototext.MessageToString(solver_message)
with _NamedTemporaryFile(mode='w+b', suffix='.prototxt') as tmpfile:
tmpfile.write(bytes(messagestr.encode('utf-8')))
tmpfile.flush()
return cls.Get_caffe_solver_class(
solver_parameter_dict['solver_type'])._caffe_solver_class(
tmpfile.name, net)
raise Exception('could not initialize solver class')
def _get_s3_state_config(s3_bucket_name, s3_key_name, credentials):
conn = _connect_s3(**credentials)
bucket = conn.get_bucket(s3_bucket_name, validate=False)
key = bucket.get_key(s3_key_name)
if not key:
raise IOError("No Predictive Service at the specified location.")
with _NamedTemporaryFile() as temp_file:
key.get_contents_to_file(temp_file)
temp_file.flush()
config = _ConfigParser(allow_no_value=True)
config.optionxform = str
config.read(temp_file.name)
temp_file.close() # deletes temp file
return config
def Get_caffe_solver_instance(cls, solver_parameter_dict, net):
"""Get a caffe solver object."""
# now we actually create a instance of the solver
solver_message = _caffe_pb2.SolverParameter(**solver_parameter_dict)
messagestr = _gprototext.MessageToString(solver_message)
with _NamedTemporaryFile(mode='w+b', suffix='.prototxt') as tmpfile:
tmpfile.write(bytes(messagestr.encode('utf-8')))
tmpfile.flush()
try:
# Newer version of caffe with full solver init support.
return cls.Get_caffe_solver_class(
solver_parameter_dict['solver_type'])._caffe_solver_class(
tmpfile.name, net, _caffe._caffe.NetVec(), True)
except TypeError:
# Fallback for older, patched versions.
return cls.Get_caffe_solver_class(
solver_parameter_dict['solver_type'])._caffe_solver_class(
tmpfile.name, net)
raise Exception('could not initialize solver class')
def Get_caffe_solver_instance(cls, solver_parameter_dict, net):
"""Get a caffe solver object."""
# now we actually create a instance of the solver
solver_message = _caffe_pb2.SolverParameter(**solver_parameter_dict)
messagestr = _gprototext.MessageToString(solver_message)
with _NamedTemporaryFile(mode='w+b', suffix='.prototxt') as tmpfile:
tmpfile.write(bytes(messagestr.encode('utf-8')))
tmpfile.flush()
try:
# Newer version of caffe with full solver init support.
return cls.Get_caffe_solver_class(
solver_parameter_dict['solver_type'])._caffe_solver_class(
tmpfile.name, net, _caffe._caffe.NetVec(), True)
except TypeError:
# Fallback for older, patched versions.
return cls.Get_caffe_solver_class(
solver_parameter_dict['solver_type'])._caffe_solver_class(
tmpfile.name, net)
raise Exception('could not initialize solver class')
def buildNet(fname_prototxt, fname_caffemodel, deconv_as_resamp=True, forward=False):
fname = fname_prototxt.split(".prototxt")[0]
if os.path.isfile(fname_caffemodel):
n_instance = caffe.Net(fname_prototxt, fname_caffemodel, caffe.TEST)
else:
n_instance = caffe.Net(fname_prototxt, caffe.TEST)
n_instance.save(fname+".caffemodel")
if forward:
n_instance.forward()
upgrade_bin = os.path.join(CAFFE_BIN_FOLDER, "upgrade_net_proto_text.bin")
with _NamedTemporaryFile(mode='r', suffix='.prototxt') as tmpfile:
_subprocess.check_call([upgrade_bin,
fname + ".prototxt",
tmpfile.name])
text = tmpfile.read()
n = _caffe_pb2.NetParameter()
_gprototext.Merge(text, n)
layers = n.ListFields()[-1][1]
net = AbstractNet(fname.split("/")[-1])
net.buffer_names_to_replace = {}
for i in range(0, len(layers)):
l = layers[i]
log(l.type + " " + l.name)
if l.type == "Input" or l.type == "Python" or l.type == "Data":
createInput(l, net, n_instance)
elif l.type == "Interp":
createInterp(l, net, n_instance)
elif l.type == "LRN":
createLRN(l, net, n_instance)
elif l.type == "Convolution":
createConv(l, net, n_instance)
elif l.type == "Pooling":
createPool(l, net, n_instance)
elif l.type == "Deconvolution":
createDeconv(l, net, n_instance, deconv_as_resamp)
elif l.type == "ReLU":
createRelu(l, net, n_instance)
elif l.type == "Concat":
createMerge(l, net, n_instance)
elif l.type == "Flatten":
createFlatten(l, net, n_instance)
elif l.type == "Softmax":
createSoftmax(l, net, n_instance)
elif l.type == "Sigmoid":
createSigmoid(l, net, n_instance)
elif l.type == "AbsVal":
createAbsVal(l, net, n_instance)
elif l.type == "TanH":
createTanH(l, net, n_instance)
elif l.type == "Eltwise":
createEltwise(l, net, n_instance)
elif l.type == "BatchNorm":
createBatchNorm(l, net, n_instance)
elif l.type == "Scale":
createScale(l, net, n_instance)
elif l.type == "Power":
createPower(l, net, n_instance)
elif l.type == "Dropout":
log("Dropout ignored")
elif l.type == "InnerProduct":
createInnerProduct(l, net, n_instance)
elif l.type == "BNLL":
createBNLL(l, net, n_instance)
elif l.type == "Reshape":
createReshape(l, net, n_instance)
elif l.type == "ArgMax":
createArgmax(l, net, n_instance)
elif l.type == "Crop":
net.buffer_names_to_replace[l.top[0]] = l.bottom[0]
else:
log("============= NOT IMPLEMENTED YET ==============")
log(l.type)
log(l.argmax_param)
read_heatmaps_data(net, n_instance)
read_weights(net, n_instance)
replace_buffer_names(net)
return net
def _make_temp(data):
tmpfile = _NamedTemporaryFile(delete=False)
tmpfile.write(data)
tmpfile.close()
return tmpfile
def from_prototxt(text=None, filename=None):
r"""
Create an :py:class:`NetSpecification` object from a text spec.
Either ``text`` or ``filename`` may be set, and is accordingly used.
Files may be of any caffe prototxt version.
"""
# Check if the user erroneously specified a filename as text.
if text is not None:
if _os.linesep not in text:
if _os.path.exists(text):
_LOGGER.warn('You probably mistakenly specified a filename '
'as text: "%s"! Trying to recover...', text)
filename = text
text = None
if filename is not None:
assert text is None
# Do a conversion if necessary.
with _NamedTemporaryFile(mode='r', suffix='.prototxt') as tmpfile:
net_upgrader_exec = _os.path.join(_CAFFE_BIN_FOLDER,
'upgrade_net_proto_text')
assert _os.path.exists(net_upgrader_exec),\
("The executable 'upgrade_net_proto_text' was not found "
"in your _CAFFE_BIN_FOLDER! Please set it from the "
"module `barrista.config`. The current folder is set "
"to: " + _CAFFE_BIN_FOLDER + ".")
_subprocess.check_call([net_upgrader_exec,
filename,
tmpfile.name])
text = tmpfile.read()
message = _caffe_pb2.NetParameter()
_gprototext.Merge(text, message)
# Check for completeness of the parsing process.
fields = message.ListFields()
for fielddesc in map(_itemgetter(0), fields): # pylint: disable=W0141
if fielddesc.name not in ['name',
'input_shape',
'debug_info',
'input',
'input_dim',
'layer',
'force_backward',
'state']:
_LOGGER.warn('Parsed net prototxt contained unknown field ' +
fielddesc.name + '. Ignored.')
if len(message.input_dim) > 0:
_LOGGER.warn('The loaded prototxt contains `input_dim` fields. '
'They are deprecated! Use `input_shape` instead.')
if _HAS_BLOB_SHAPE:
assert len(message.input_shape) == 0
assert len(message.input_dim) % 4 == 0
input_shape = _copy.deepcopy(list(_chunks(message.input_dim, 4)))
else:
input_shape = _copy.deepcopy([bshape.dim for
bshape in message.input_shape])
inputs = _copy.deepcopy(message.input)
layerspecs = [LayerSpecification.from_pbuf_message(layer)
for layer in message.layer]
pbforcebw = message.force_backward
phase = message.state.phase
level = message.state.level
stages = _copy.deepcopy(message.state.stage)
debug_info = message.debug_info
name = message.name
spec = NetSpecification(input_shape,
inputs,
layerspecs,
pbforcebw,
phase,
level,
stages,
debug_info,
name)
return spec
def from_prototxt(text=None, filename=None):
r"""
Create an :py:class:`NetSpecification` object from a text spec.
Either ``text`` or ``filename`` may be set, and is accordingly used.
Files may be of any caffe prototxt version.
"""
# Check if the user erroneously specified a filename as text.
if text is not None:
if _os.linesep not in text:
if _os.path.exists(text): # pragma: no cover
_LOGGER.warn(
'You probably mistakenly specified a filename '
'as text: "%s"! Trying to recover...', text)
filename = text
text = None
if filename is not None:
assert text is None
# Do a conversion if necessary.
with _NamedTemporaryFile(mode='r', suffix='.prototxt') as tmpfile:
net_upgrader_exec = _os.path.join(_CAFFE_BIN_FOLDER,
'upgrade_net_proto_text')
assert _os.path.exists(net_upgrader_exec),\
("The executable 'upgrade_net_proto_text' was not found "
"in your _CAFFE_BIN_FOLDER! Please set it from the "
"module `barrista.config`. The current folder is set "
"to: " + _CAFFE_BIN_FOLDER + ".")
_subprocess.check_call(
[net_upgrader_exec, filename, tmpfile.name])
text = tmpfile.read()
message = _caffe_pb2.NetParameter()
_gprototext.Merge(text, message)
# Check for completeness of the parsing process.
fields = message.ListFields()
for fielddesc in map(_itemgetter(0), fields): # pylint: disable=W0141
if fielddesc.name not in [
'name', 'input_shape', 'debug_info', 'input', 'input_dim',
'layer', 'force_backward', 'state'
]:
_LOGGER.warn('Parsed net prototxt contained unknown field ' +
fielddesc.name + '. Ignored.')
if len(message.input_dim) > 0:
_LOGGER.warn('The loaded prototxt contains `input_dim` fields. '
'They are deprecated! Use `input_shape` instead.')
if _HAS_BLOB_SHAPE:
assert len(message.input_shape) == 0
assert len(message.input_dim) % 4 == 0
input_shape = _copy.deepcopy(list(_chunks(message.input_dim, 4)))
else: # pragma: no cover
input_shape = _copy.deepcopy(
[bshape.dim for bshape in message.input_shape])
inputs = _copy.deepcopy(message.input)
layerspecs = [
LayerSpecification.from_pbuf_message(layer)
for layer in message.layer
]
pbforcebw = message.force_backward
phase = message.state.phase
level = message.state.level
stages = _copy.deepcopy(message.state.stage)
debug_info = message.debug_info
name = message.name
spec = NetSpecification(input_shape, inputs, layerspecs, pbforcebw,
phase, level, stages, debug_info, name)
return spec
