def _decompress_xz(filename):
"""Eumlates an option function in read mode for xz.
See the comment in _compress_xz for more information.
This function tries to emulate the lzma module as much as
possible
"""
if not filename.endswith('.xz'):
filename = '{}.xz'.format(filename)
try:
with open(os.devnull, 'w') as null:
string = subprocess.check_output(
['xz', '--decompress', '--stdout', filename],
stderr=null)
except OSError as e:
if e.errno == errno.ENOENT:
raise exceptions.PiglitFatalError(
'No xz binary available')
raise
# We need a file-like object, so the contents must be placed in
# a StringIO object.
io = StringIO()
io.write(string)
io.seek(0)
yield io
io.close()
def _decompress_xz(filename):
"""Eumlates an option function in read mode for xz.
See the comment in _compress_xz for more information.
This function tries to emulate the lzma module as much as
possible
"""
if not filename.endswith('.xz'):
filename = '{}.xz'.format(filename)
try:
with open(os.devnull, 'w') as null:
string = subprocess.check_output(
['xz', '--decompress', '--stdout', filename],
stderr=null)
except OSError as e:
if e.errno == errno.ENOENT:
raise exceptions.PiglitFatalError(
'No xz binary available')
raise
# We need a file-like object, so the contents must be placed in
# a StringIO object.
io = StringIO()
io.write(string)
io.seek(0)
yield io
io.close()
class IncludedResponse(object):
def __init__(self):
self.headers = None
self.status = None
self.output = StringIO()
self.str = None
def close(self):
self.str = self.output.getvalue()
self.output.close()
self.output = None
def write(self, s):
assert self.output is not None, (
"This response has already been closed and no further data "
"can be written.")
self.output.write(s)
def __str__(self):
return self.body
def body__get(self):
if self.str is None:
return self.output.getvalue()
else:
return self.str
body = property(body__get)
def picture_view(request, user_id, year=None):
"""Displays a view of a user's picture.
Args:
user_id
The ID of the user whose picture is being fetched.
year
The user's picture from this year is fetched. If not
specified, use the preferred picture.
"""
try:
user = User.get_user(id=user_id)
except User.DoesNotExist:
raise Http404
default_image_path = os.path.join(settings.PROJECT_ROOT, "static/img/default_profile_pic.png")
if user is None:
raise Http404
else:
if year is None:
preferred = user.preferred_photo
if preferred is not None:
if preferred.endswith("Photo"):
preferred = preferred[:-len("Photo")]
if preferred == "AUTO":
data = user.default_photo()
if data is None:
image_buffer = io.open(default_image_path, mode="rb")
else:
image_buffer = StringIO(data)
# Exclude 'graduate' from names array
elif preferred in Grade.names:
data = user.photo_binary(preferred)
if data:
image_buffer = StringIO(data)
else:
image_buffer = io.open(default_image_path, mode="rb")
else:
image_buffer = io.open(default_image_path, mode="rb")
else:
data = user.photo_binary(year)
if data:
image_buffer = StringIO(data)
else:
image_buffer = io.open(default_image_path, mode="rb")
response = HttpResponse(content_type="image/jpeg")
response["Content-Disposition"] = "filename={}_{}.jpg".format(user_id, year or preferred)
try:
img = image_buffer.read()
except UnicodeDecodeError:
img = io.open(default_image_path, mode="rb").read()
image_buffer.close()
response.write(img)
return response
def __str__(self):
buffer = StringIO()
if self.refTime is not None:
refTimeInSecs = self.refTime.getTime() / 1000
micros = (self.refTime.getTime() % 1000) * 1000
dtObj = datetime.datetime.utcfromtimestamp(refTimeInSecs)
dtObj = dtObj.replace(microsecond=micros)
buffer.write(dtObj.isoformat(' '))
if "FCST_USED" in self.utilityFlags:
hrs = int(self.fcstTime / 3600)
mins = int((self.fcstTime - (hrs * 3600)) / 60)
buffer.write(" (" + str(hrs))
if mins != 0:
buffer.write(":" + str(mins))
buffer.write(")")
if "PERIOD_USED" in self.utilityFlags:
buffer.write("[")
buffer.write(self.validPeriod.start.isoformat(' '))
buffer.write("--")
buffer.write(self.validPeriod.end.isoformat(' '))
buffer.write("]")
strVal = buffer.getvalue()
buffer.close()
return strVal
class IncludedResponse(object):
def __init__(self):
self.headers = None
self.status = None
self.output = StringIO()
self.str = None
def close(self):
self.str = self.output.getvalue()
self.output.close()
self.output = None
def write(self, s):
assert self.output is not None, (
"This response has already been closed and no further data "
"can be written.")
self.output.write(s)
def __str__(self):
return self.body
def body__get(self):
if self.str is None:
return self.output.getvalue()
else:
return self.str
body = property(body__get)
def test_write_ini_with_custom_converters(self):
def dict_encoder(dict_):
return ','.join('%s:%s' % (k, v) for (k, v) in dict_.items())
def dict_decoder(string):
return dict(x.split(':') for x in string.split(','))
n = Namespace(doc='top')
n.add_option(
'a',
default={'one': 'One'},
doc='the doc string',
to_string_converter=dict_encoder,
from_string_converter=dict_decoder,
)
c = ConfigurationManager([n],
use_admin_controls=True,
use_auto_help=False,
argv_source=[])
expected = "# the doc string\n#a=one:One\n"
out = StringIO()
c.write_conf(for_configobj, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
self.assertEqual(expected.strip(), received.strip())
def test_write_json(self):
n = Namespace(doc='top')
n.add_option(
'aaa',
'2011-05-04T15:10:00',
'the a',
short_form='a',
from_string_converter=datetime_from_ISO_string
)
c = ConfigurationManager(
[n],
use_admin_controls=True,
use_auto_help=False,
argv_source=[]
)
out = StringIO()
c.write_conf(for_json, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
jrec = json.loads(received)
expect_to_find = {
"short_form": "a",
"default": "2011-05-04T15:10:00",
"doc": "the a",
"value": "2011-05-04T15:10:00",
"from_string_converter":
"configman.datetime_util.datetime_from_ISO_string",
"name": "aaa"
}
for key, value in expect_to_find.items():
self.assertEqual(jrec['aaa'][key], value)
def test_write_json(self):
n = Namespace(doc='top')
n.add_option('aaa',
'2011-05-04T15:10:00',
'the a',
short_form='a',
from_string_converter=datetime_from_ISO_string)
c = ConfigurationManager([n],
use_admin_controls=True,
use_auto_help=False,
argv_source=[])
out = StringIO()
c.write_conf(for_json, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
jrec = json.loads(received)
expect_to_find = {
"short_form": "a",
"default": "2011-05-04T15:10:00",
"doc": "the a",
"value": "2011-05-04T15:10:00",
"from_string_converter":
"configman.datetime_util.datetime_from_ISO_string",
"name": "aaa"
}
for key, value in expect_to_find.items():
self.assertEqual(jrec['aaa'][key], value)
def test_write_ini_with_custom_converters(self):
def dict_encoder(dict_):
return ','.join('%s:%s' % (k, v) for (k, v) in dict_.items())
def dict_decoder(string):
return dict(x.split(':') for x in string.split(','))
n = Namespace(doc='top')
n.add_option(
'a',
default={'one': 'One'},
doc='the doc string',
to_string_converter=dict_encoder,
from_string_converter=dict_decoder,
)
c = ConfigurationManager(
[n],
use_admin_controls=True,
use_auto_help=False,
argv_source=[]
)
expected = "# the doc string\n#a=one:One\n"
out = StringIO()
c.write_conf(for_configobj, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
self.assertEqual(expected.strip(), received.strip())
def test_for_mapping_long_doc_in_write_conf(self):
n = self._some_namespaces()
n = Namespace(doc='top')
n.add_option(
'aaa',
'Default Value Goes In Here',
'This time the documentation string is really long. So long '
'that we have to write it on multiple lines.',
)
cm = ConfigurationManager(
n,
values_source_list=[],
)
out = StringIO()
cm.write_conf(for_mapping, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
for line in received.splitlines():
self.assertTrue(len(line) < 80, line)
expected = """
# This time the documentation string is really long. So long that we have to
# write it on multiple lines. (default: 'Default Value Goes In Here')
aaa='Default Value Goes In Here'
""".strip()
if six.PY3:
expected = expected.replace("='", "=b'")
self.assertEqual(received.strip(), expected)
def __str__(self):
buffer = StringIO()
if self.refTime is not None:
refTimeInSecs = self.refTime.getTime() / 1000
micros = (self.refTime.getTime() % 1000) * 1000
dtObj = datetime.datetime.utcfromtimestamp(refTimeInSecs)
dtObj = dtObj.replace(microsecond=micros)
# This won't be compatible with java or string from java since its to microsecond
buffer.write(dtObj.isoformat(' '))
if "FCST_USED" in self.utilityFlags:
hrs = int(self.fcstTime / 3600)
mins = int((self.fcstTime - (hrs * 3600)) / 60)
buffer.write(" (" + str(hrs))
if mins != 0:
buffer.write(":" + str(mins))
buffer.write(")")
if "PERIOD_USED" in self.utilityFlags:
buffer.write("[")
buffer.write(self.validPeriod.start.isoformat(' '))
buffer.write("--")
buffer.write(self.validPeriod.end.isoformat(' '))
buffer.write("]")
strVal = buffer.getvalue()
buffer.close()
return strVal
def scan(path):
"""
Performs an in-process binary module scan. That means the module is
loaded (imported) into the current Python interpreter.
"path" - a path to a binary module to scan
Returns a CIX 2.0 XML string.
"""
from gencix.python import gencixcore as gencix
name,_ = os.path.splitext(os.path.basename(path))
dir = os.path.dirname(path)
root = gencix.Element('codeintel', version='2.0', name=name)
gencix.docmodule(name, root, usefile=True, dir=dir)
gencix.perform_smart_analysis(root)
gencix.prettify(root)
tree = gencix.ElementTree(root)
stream = StringIO()
try:
stream.write('<?xml version="1.0" encoding="UTF-8"?>\n')
tree.write(stream)
return stream.getvalue()
finally:
stream.close()
def scan(path):
"""
Performs an in-process binary module scan. That means the module is
loaded (imported) into the current Python interpreter.
"path" - a path to a binary module to scan
Returns a CIX 2.0 XML string.
"""
from gencix.python import gencixcore as gencix
name, _ = os.path.splitext(os.path.basename(path))
dir = os.path.dirname(path)
root = gencix.Element('codeintel', version='2.0', name=name)
gencix.docmodule(name, root, usefile=True, dir=dir)
gencix.perform_smart_analysis(root)
gencix.prettify(root)
tree = gencix.ElementTree(root)
stream = StringIO()
try:
stream.write('<?xml version="1.0" encoding="UTF-8"?>\n')
tree.write(stream)
return stream.getvalue()
finally:
stream.close()
def run_pylint():
buff = StringIO()
reporter = text.ParseableTextReporter(output=buff)
args = ["--include-ids=y", "--errors-only", "rbd_iscsi_client"]
lint.Run(args, reporter=reporter, exit=False)
val = buff.getvalue()
buff.close()
return val
def run_pylint():
buff = StringIO()
reporter = text.ParseableTextReporter(output=buff)
args = ["--include-ids=y", "-E", "sahara"]
lint.Run(args, reporter=reporter, exit=False)
val = buff.getvalue()
buff.close()
return val
def run_pylint():
buff = StringIO()
reporter = text.ParseableTextReporter(output=buff)
args = ["-rn", "--disable=all", "--enable=" + enabled_codes ,"murano"]
lint.Run(args, reporter=reporter, exit=False)
val = buff.getvalue()
buff.close()
return val
def run_pylint():
buff = StringIO()
reporter = text.ParseableTextReporter(output=buff)
args = ["--include-ids=y", "-E", "savanna"]
lint.Run(args, reporter=reporter, exit=False)
val = buff.getvalue()
buff.close()
return val
def write_default_ansible_cfg(work_dir,
remote_user,
ssh_private_key=None,
transport=None,
base_ansible_cfg='/etc/ansible/ansible.cfg',
override_ansible_cfg=None):
ansible_config_path = os.path.join(work_dir, 'ansible.cfg')
shutil.copy(base_ansible_cfg, ansible_config_path)
config = configparser.ConfigParser()
config.read(ansible_config_path)
config.set('defaults', 'retry_files_enabled', 'False')
log_path = os.path.join(work_dir, 'ansible.log')
config.set('defaults', 'log_path', log_path)
if os.path.exists(log_path):
new_path = (log_path + '-' +
datetime.now().strftime("%Y-%m-%dT%H:%M:%S"))
os.rename(log_path, new_path)
config.set('defaults', 'forks', '25')
config.set('defaults', 'timeout', '30')
config.set('defaults', 'gather_timeout', '30')
# mistral user has no home dir set, so no place to save a known hosts file
config.set(
'ssh_connection', 'ssh_args', '-o UserKnownHostsFile=/dev/null '
'-o StrictHostKeyChecking=no '
'-o ControlMaster=auto '
'-o ControlPersist=30m '
'-o ServerAliveInterval=5 '
'-o ServerAliveCountMax=5')
config.set('ssh_connection', 'control_path_dir',
os.path.join(work_dir, 'ansible-ssh'))
config.set('ssh_connection', 'retries', '8')
config.set('ssh_connection', 'pipelining', 'True')
# Set connection info in config file so that subsequent/nested ansible
# calls can re-use it
if remote_user:
config.set('defaults', 'remote_user', remote_user)
if ssh_private_key:
config.set('defaults', 'private_key_file', ssh_private_key)
if transport:
config.set('defaults', 'transport', transport)
if override_ansible_cfg:
sio_cfg = StringIO()
sio_cfg.write(override_ansible_cfg)
sio_cfg.seek(0)
config.readfp(sio_cfg)
sio_cfg.close()
with open(ansible_config_path, 'w') as configfile:
config.write(configfile)
return ansible_config_path
def run_pylint():
buff = StringIO()
reporter = text.ParseableTextReporter(output=buff)
args = ["-rn", "--disable=all", "--enable=" + ",".join(ENABLED_CODES),
"murano"]
lint.Run(args, reporter=reporter, exit=False)
val = buff.getvalue()
buff.close()
return val
def run_pylint():
buff = StringIO()
args = ["--msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg}",
"-E",
"ceilometer"]
lint.Run(args, exit=False)
val = buff.getvalue()
buff.close()
return val
def run_pylint():
buff = StringIO()
reporter = text.TextReporter(output=buff)
args = [
"--msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}'",
"-E", "cinderclient"]
lint.Run(args, reporter=reporter, exit=False)
val = buff.getvalue()
buff.close()
return val
def get_cix_string(cix, prettyFormat=True):
# Get the CIX.
if prettyFormat:
prettify(cix)
cixstream = StringIO()
cixtree = ElementTree(cix)
cixstream.write('<?xml version="1.0" encoding="UTF-8"?>\n')
cixtree.write(cixstream)
cixcontent = cixstream.getvalue()
cixstream.close()
return cixcontent
def load_conf(xfile,
options=None,
parse_conf_func=None,
load_creds=False,
envvar=None,
custom_file=None):
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
conf = {'_config_directory': None}
if os.path.exists(xfile):
with open(xfile, 'r') as f:
conf = helpers.load_yaml(f)
config_directory = os.path.dirname(os.path.abspath(xfile))
conf['_config_directory'] = config_directory
if custom_file:
conf = helpers.merge(
helpers.load_conf_yaml_file(custom_file, config_directory),
conf)
conf['_config_directory'] = config_directory
elif envvar and os.environ.get(envvar):
c = StringIO(os.environ[envvar])
conf = helpers.load_yaml(c.getvalue())
c.close()
conf['_config_directory'] = None
if parse_conf_func:
conf = parse_conf_func(conf)
else:
conf = parse_conf(conf, load_creds)
for x in ('modules', 'plugins'):
conf = import_conf_files(x, conf)
init_modules(conf)
init_plugins(conf)
if _INOTIFY:
init_inotify(conf)
if not options or not isinstance(options, object):
return conf
for def_option in iterkeys(get_default_options()):
if getattr(options, def_option, None) is None \
and def_option in conf['general']:
setattr(options, def_option, conf['general'][def_option])
setattr(options, 'configuration', conf)
return options
def get_cix_string(cix, prettyFormat=True):
# Get the CIX.
if prettyFormat:
prettify(cix)
cixstream = StringIO()
cixtree = ElementTree(cix)
cixstream.write('<?xml version="1.0" encoding="UTF-8"?>\n')
cixtree.write(cixstream)
cixcontent = cixstream.getvalue()
cixstream.close()
return cixcontent
def _format_stack(frame):
"""
Pretty-print the stack of `frame` like logging would.
"""
sio = StringIO()
sio.write("Stack (most recent call last):\n")
traceback.print_stack(frame, file=sio)
sinfo = sio.getvalue()
if sinfo[-1] == "\n":
sinfo = sinfo[:-1]
sio.close()
return sinfo
def _format_exception(exc_info):
"""
Prettyprint an `exc_info` tuple.
Shamelessly stolen from stdlib's logging module.
"""
sio = StringIO()
traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], None, sio)
s = sio.getvalue()
sio.close()
if s[-1:] == "\n":
s = s[:-1]
return s
def vmd_visualize_script(pdb_filename, trajectory_filename, topology,
cplx_filename, coloring, step):
"""Function that creates the string in a syntax, that vmd can interpret. This
string can be saved to a text/data file.
Includes a comment-header. The very last printed lines prevent vmd from
automatically creating an additional Representation.
Parameters:
-----------
pdb_filename: String
Containing name of pdb-file (parsed argument)
trajectory_filename: String
Containing name of trajectory file (parsed argument)
topology: dictionary
yaml-dictionary from cplx
cplx_filename: String
Containing name of cplx-file (parsed argument)
coloring: String
(parsed argument) for choosing coloring scheme
Returns:
--------
String
the complete string to be written to data-file
"""
trajectory_type = os.path.splitext(trajectory_filename)[-1][1:]
out = StringIO()
out.write(
HEADER_FORMAT.format(
generation_time=datetime.datetime.time(datetime.datetime.now()),
topology_inputfile=cplx_filename,
pdb_inputfile=pdb_filename,
trajectory_inputfile=trajectory_filename,
))
out.write(FUNCTIONS)
out.write(LOAD_VMD_TOP_FORMAT.format(vmd_top=pdb_filename))
out.write(
LOADTRAJ_FORMAT.format(traj_file=trajectory_filename,
traj_type=trajectory_type,
step=step))
out.write(
"#adding representation to molecule 0 \n #################################### \n"
)
out.write(vmd_rep_gen(topology, coloring))
out.write(SHOW_SIM_BOX)
rep = out.getvalue()
out.close()
return rep
def dumps(root):
"""Convert obj into an xml string.
"""
if not isinstance(root, ET.Element):
raise ValueError
document = ET.ElementTree(root)
flow = StringIO()
document.write(flow, encoding='utf-8', xml_declaration=True)
contents = flow.getvalue()
flow.close()
return minidom.parseString(contents).toprettyxml(indent=" ")
def vmd_rep_gen(topology, coloring):
"""Goes through the given topology and creates representations from that. The
domains will be in a certain coloring scheme (specified by COLORING
input-param). Different domain types will be shown in different rep_types by
vmd.
Parameter:
----------
topology : dictionary
Dictionary from the cplx read by yaml
coloring : string
random or domain. Parsed as input argument for choosing color-scheme
Returns:
--------
string
The string containing all the interpretable stuff for vmd to create the
representations (also HEADER)
"""
out = StringIO()
bead_counter = 1
rep_counter = 0
name2type = get_type_translater(topology["definitions"])
for j, top in enumerate(topology["topologies"]):
domains = top["domains"]
for i, dom in six.iteritems(domains):
chainids = dom["chain-ids"]
rep_number = rep_counter
first_bead = bead_counter
last_bead = first_bead + dom["nbeads"]
bead_counter = last_bead + 1
dom_rep, rep_counter = vmd_domain_rep(
name2type[dom["type"]],
chainids,
coloring,
rep_counter,
rep_number,
i,
dom,
)
out.write(dom_rep)
# The Following has to be the LAST included instruction in the file !!
# Because otherwise vmd will automatically generate an additional
# representation including all atoms, displaying them as lines.
out.write(DEL_REP_FORMAT.format(rep_counter=rep_counter))
concatenated_string = out.getvalue()
out.close()
return concatenated_string
class LoggingFramework:
def __init__(self):
self.k = StringIO()
self.current_out = sys.stdout
self.current_err = sys.stderr
sys.stdout = self.k
sys.stderr = self.k
def __del__(self):
sys.stdout = self.current_out
sys.stderr = self.current_err
self.k.flush()
self.k.close()
def getvalue(self): return self.k.getvalue()
class _FileWrapper(object):
def set_file(self, filename):
self.f = open(filename, 'r')
def set_stdin(self):
self.f = sys.stdin
def set_text(self, text):
from six.moves import cStringIO as StringIO
self.f = StringIO(text)
def readline(self):
return self.f.readline()
def close(self):
self.f.close()
class _FileWrapper(object):
def set_file(self, filename):
self.f = open(filename, 'r')
def set_stdin(self):
self.f = sys.stdin
def set_text(self, text):
from six.moves import cStringIO as StringIO
self.f = StringIO(text)
def readline(self):
return self.f.readline()
def close(self):
self.f.close()
def old(self, no_reduce_db):
try:
if no_reduce_db:
touch('./dummydb')
fileobj = StringIO()
self.write_pdb(fileobj)
fileobj.seek(0)
reduce = os.path.join(os.getenv('LIBTBX_BUILD'), 'reduce', 'exe',
'reduce')
if not os.path.exists(reduce):
reduce = 'phenix.reduce'
cmd = [reduce, '-BUILD', '-NUC', '-NOFLIP', '-DB ./dummydb', '-']
if no_reduce_db:
process = subprocess.Popen([
reduce, '-BUILD', '-NUC', '-NOFLIP', '-DB ./dummydb', '-'
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
process = subprocess.Popen(
[reduce, '-BUILD', '-NUC', '-NOFLIP', '-'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = process.communicate(str.encode(fileobj.read()))
out = out.decode()
err = err.decode()
if process.wait():
logger.error("REDUCE returned non-zero exit status: "
"See reduce_info.log for more details")
# print out the reduce log even if it worked
with open('reduce_info.log', 'w') as fh:
fh.write(err)
pdbh = StringIO(out)
# not using load_file since it does not read StringIO
print('-' * 80)
print(pdbh)
print('-' * 80)
self.parm = parmed.read_PDB(pdbh)
finally:
fileobj.close()
if no_reduce_db:
os.unlink('./dummydb')
return self
def _get_config(self, section, progpath, parse_vers, check_version=True):
my_vers = False
if check_version:
my_vers = parse_vers(
subprocess.check_output((progpath, '--version')).strip())
myconf = StringIO("%s\n%s" % (self._read_file(
self._default_file), self._read_file(self._custom_file)))
self._myconf.readfp(myconf)
myconf.close()
if my_vers:
self._check_conf_versions(self._myconf, section,
my_vers.group('version'), 'ver-')
self._check_conf_versions(self._myconf, section,
my_vers.group('distrib'), 'dist-')
return self._myconf
def test_write_ini(self):
n = self._some_namespaces()
c = ConfigurationManager(
[n],
[{
"c.fred": "just like George Jetson",
}],
use_admin_controls=True,
#use_config_files=False,
use_auto_help=False,
argv_source=[]
)
expected = """# the a
#aaa=2011-05-04T15:10:00
[c]
# husband from Flintstones
fred=just like George Jetson
# wife from Flintstones
#wilma=waspish's
[d]
# female neighbor from I Love Lucy
#ethel=silly
# male neighbor from I Love Lucy
#fred=crabby
[x]
# the password
#password=secret "message"
# how big in tons
#size=100
"""
out = StringIO()
c.write_conf(for_configobj, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
self.assertEqual(expected.strip(), received.strip())
def test_write_ini(self):
n = self._some_namespaces()
c = ConfigurationManager(
[n],
[{
"c.fred": "just like George Jetson",
}],
use_admin_controls=True,
#use_config_files=False,
use_auto_help=False,
argv_source=[])
expected = """# the a
#aaa=2011-05-04T15:10:00
[c]
# husband from Flintstones
fred=just like George Jetson
# wife from Flintstones
#wilma=waspish's
[d]
# female neighbor from I Love Lucy
#ethel=silly
# male neighbor from I Love Lucy
#fred=crabby
[x]
# the password
#password=secret "message"
# how big in tons
#size=100
"""
out = StringIO()
c.write_conf(for_configobj, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
self.assertEqual(expected.strip(), received.strip())
def formatException(self, exc_info, record=None):
"""Format exception output with CONF.logging_exception_prefix."""
if not record:
return logging.Formatter.formatException(self, exc_info)
stringbuffer = StringIO()
traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], None,
stringbuffer)
lines = stringbuffer.getvalue().split('\n')
stringbuffer.close()
if CONF.logging_exception_prefix.find('%(asctime)') != -1:
record.asctime = self.formatTime(record, self.datefmt)
formatted_lines = []
for line in lines:
pl = CONF.logging_exception_prefix % record.__dict__
fl = '%s%s' % (pl, line)
formatted_lines.append(fl)
return '\n'.join(formatted_lines)
def test_for_mapping_nested_namespaces(self):
n = self._some_namespaces()
cm = ConfigurationManager(
n,
values_source_list=[],
)
out = StringIO()
cm.write_conf(for_mapping, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
expected = """
# the a (default: '2011-05-04T15:10:00')
aaa='2011-05-04T15:10:00'
# your uncle (default: 98)
c__dwight='98'
# husband from Flintstones (default: 'stupid')
c__fred='stupid'
# wife from Flintstones (default: 'waspish')
c__wilma='waspish'
# my uncle (default: 97)
c__e__dwight='97'
# female neighbor from I Love Lucy (default: 'silly')
d__ethel='silly'
# male neighbor from I Love Lucy (default: 'crabby')
d__fred='crabby'
# the password (default: 'secret')
x__password='******'
# how big in tons (default: 100)
x__size='100'
""".strip()
if six.PY3:
expected = expected.replace("='", "=b'")
self.assertEqual(received.strip(), expected)
def load_conf(xfile, options=None, envvar=None):
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
conf = {'_config_directory': None}
if os.path.exists(xfile):
with open(xfile, 'r') as f:
conf = parse_conf(load_yaml(f))
conf['_config_directory'] = os.path.dirname(os.path.abspath(xfile))
elif envvar and os.environ.get(envvar):
c = StringIO(os.environ[envvar])
conf = parse_conf(load_yaml(c.getvalue()))
c.close()
conf['_config_directory'] = None
for x in ('modules', 'plugins'):
conf = import_conf_files(x, conf)
init_modules(conf)
init_plugins(conf)
if not conf.get('dns'):
conf['dns'] = {}
if conf['dns'] and conf['dns'].get('domains'):
for name, domain_cfg in viewitems(conf['dns']['domains']):
cfg = {'rrsets': [], 'vars': {}}
for x in ('vars', 'rrsets'):
if isinstance(cfg[x], list):
append_func = getattr(cfg[x], 'extend')
else:
append_func = getattr(cfg[x], 'update')
if domain_cfg.get("import_%s" % x):
append_func(
import_file(domain_cfg["import_%s" % x],
conf['_config_directory'], cfg))
if x in domain_cfg:
append_func(domain_cfg[x])
if not cfg['rrsets']:
cfg = None
if cfg:
conf['dns']['domains'][name] = cfg
if not options or not isinstance(options, object):
return conf
for def_option in iterkeys(get_default_options()):
if getattr(options, def_option, None) is None \
and def_option in conf['general']:
setattr(options, def_option, conf['general'][def_option])
setattr(options, 'configuration', conf)
return options
class TextDumper(Communicator):
"""Class to dump data. """
def __init__(self, fname=None):
Communicator.__init__(self, self)
self._fname = fname
self._file = None
self.open(fname)
def open(self, fname=None):
"""Open a new file for writing. """
self.close()
self._fname = fname
if fname is None:
self._file = StringIO()
else:
self._file = open(fname, "w")
def close(self):
"""Transmit the objects. """
val = None
if self._file is not None:
Communicator.close(self)
if self._fname is None:
val = self._file.getvalue()
self._file.close()
self._file = None
self._fname = None
return val
def info(self, obj):
"""Write the object info"""
kind = xc_type(obj)
self._file.write("%c " % kind)
if kind in ['C', 'I', 'N', 'R']:
self._file.write('%d ' % ctypes.sizeof(c_type(obj)))
elif type(obj) is XCStruct:
self._file.write('%s ' % obj.name())
else:
self._file.write('%s ' % type(obj).__name__)
def data(self, obj, total=None):
"""Write the objects data. """
if total is not None:
tmp = '%s ' % CHAR_MAP.get(xc_type(obj[0]), None)
if type(obj[0]) in XC_TYPES:
for i in xrange(total):
self._file.write(tmp % obj[i].value)
elif type(obj[0]) in CTYPES:
for i in xrange(total):
self._file.write(tmp % obj[i])
else:
for i in xrange(total):
data(self, obj[i], "")
return
tmp = '%s ' % CHAR_MAP[xc_type(obj)]
if type(obj) not in XC_TYPES:
self._file.write(tmp % obj)
else:
self._file.write(tmp % obj.value)
def trans_type(self, obj):
"""Transfer a character. """
self._file.write("%c " % obj)
def trans_byte(self, obj):
"""Transfer a byte. """
self._file.write("%d " % obj)
def trans_num(self, num):
"""Transfer an integer. """
self._file.write("%d " % num)
def trans_varname(self, varname):
"""Transfer a variable name. """
self._file.write("%s " % varname)
def trans_name(self, name):
"""Transfer a name. """
self._file.write("%s " % name)
def trans_num_objects(self):
"""Transfer the number of objects. """
self._file.write("%d\n" % len(self.obj))
def trans_num_classes(self):
"""Tranfer the number of objects in cls_obj. """
if len(self.cls_obj) > 0:
self._file.write("%d\n" % len(self.cls_obj))
else:
self._file.write("%d " % len(self.cls_obj))
def trans_close(self):
"""Print character to denote when transfer of obj is done. """
self._file.write("\n")
class PythonWidget(HistoryConsoleWidget):
""" A basic in-process Python interpreter.
"""
# Emitted when a command has been executed in the interpeter.
executed = QtCore.Signal()
#--------------------------------------------------------------------------
# 'object' interface
#--------------------------------------------------------------------------
def __init__(self, parent=None):
super(PythonWidget, self).__init__(parent)
# PythonWidget attributes.
self.locals = dict(__name__='__console__', __doc__=None)
self.interpreter = InteractiveInterpreter(self.locals)
# PythonWidget protected attributes.
self._buffer = StringIO()
self._bracket_matcher = BracketMatcher(self._control)
self._call_tip_widget = CallTipWidget(self._control)
self._completion_lexer = CompletionLexer(PythonLexer())
self._hidden = False
self._highlighter = PythonWidgetHighlighter(self)
self._last_refresh_time = 0
# file-like object attributes.
self.encoding = sys.stdin.encoding
# Configure the ConsoleWidget.
self.tab_width = 4
self._set_continuation_prompt('... ')
# Configure the CallTipWidget.
self._call_tip_widget.setFont(self.font)
self.font_changed.connect(self._call_tip_widget.setFont)
# Connect signal handlers.
document = self._control.document()
document.contentsChange.connect(self._document_contents_change)
# Display the banner and initial prompt.
self.reset()
#--------------------------------------------------------------------------
# file-like object interface
#--------------------------------------------------------------------------
def flush(self):
""" Flush the buffer by writing its contents to the screen.
"""
self._buffer.seek(0)
text = self._buffer.getvalue()
self._buffer.close()
self._buffer = StringIO()
self._append_plain_text(text)
self._control.moveCursor(QtGui.QTextCursor.End)
def readline(self, prompt=None):
""" Read and return one line of input from the user.
"""
return self._readline(prompt)
def write(self, text, refresh=True):
""" Write text to the buffer, possibly flushing it if 'refresh' is set.
"""
if not self._hidden:
self._buffer.write(text)
if refresh:
current_time = time()
if current_time - self._last_refresh_time > 0.05:
self.flush()
self._last_refresh_time = current_time
def writelines(self, lines, refresh=True):
""" Write a list of lines to the buffer.
"""
for line in lines:
self.write(line, refresh=refresh)
#---------------------------------------------------------------------------
# 'ConsoleWidget' abstract interface
#---------------------------------------------------------------------------
def _is_complete(self, source, interactive):
""" Returns whether 'source' can be completely processed and a new
prompt created. When triggered by an Enter/Return key press,
'interactive' is True; otherwise, it is False.
"""
if interactive:
lines = source.splitlines()
if len(lines) == 1:
try:
return compile_command(source) is not None
except:
# We'll let the interpeter handle the error.
return True
else:
return lines[-1].strip() == ''
else:
return True
def _execute(self, source, hidden):
""" Execute 'source'. If 'hidden', do not show any output.
See parent class :meth:`execute` docstring for full details.
"""
# Save the current std* and point them here
old_stdin = sys.stdin
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stdin = sys.stdout = sys.stderr = self
# Run the source code in the interpeter
self._hidden = hidden
try:
more = self.interpreter.runsource(source)
finally:
self._hidden = False
# Restore std* unless the executed changed them
if sys.stdin is self:
sys.stdin = old_stdin
if sys.stdout is self:
sys.stdout = old_stdout
if sys.stderr is self:
sys.stderr = old_stderr
self.executed.emit()
self._show_interpreter_prompt()
def _prompt_started_hook(self):
""" Called immediately after a new prompt is displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = True
def _prompt_finished_hook(self):
""" Called immediately after a prompt is finished, i.e. when some input
will be processed and a new prompt displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = False
def _tab_pressed(self):
""" Called when the tab key is pressed. Returns whether to continue
processing the event.
"""
# Perform tab completion if:
# 1) The cursor is in the input buffer.
# 2) There is a non-whitespace character before the cursor.
text = self._get_input_buffer_cursor_line()
if text is None:
return False
complete = bool(text[:self._get_input_buffer_cursor_column()].strip())
if complete:
self._complete()
return not complete
#---------------------------------------------------------------------------
# 'ConsoleWidget' protected interface
#---------------------------------------------------------------------------
def _event_filter_console_keypress(self, event):
""" Reimplemented for smart backspace.
"""
if event.key() == QtCore.Qt.Key_Backspace and \
not event.modifiers() & QtCore.Qt.AltModifier:
# Smart backspace: remove four characters in one backspace if:
# 1) everything left of the cursor is whitespace
# 2) the four characters immediately left of the cursor are spaces
col = self._get_input_buffer_cursor_column()
cursor = self._control.textCursor()
if col > 3 and not cursor.hasSelection():
text = self._get_input_buffer_cursor_line()[:col]
if text.endswith(' ') and not text.strip():
cursor.movePosition(QtGui.QTextCursor.Left,
QtGui.QTextCursor.KeepAnchor, 4)
cursor.removeSelectedText()
return True
return super(PythonWidget, self)._event_filter_console_keypress(event)
def _insert_continuation_prompt(self, cursor):
""" Reimplemented for auto-indentation.
"""
super(PythonWidget, self)._insert_continuation_prompt(cursor)
source = self.input_buffer
space = 0
for c in source.splitlines()[-1]:
if c == '\t':
space += 4
elif c == ' ':
space += 1
else:
break
if source.rstrip().endswith(':'):
space += 4
cursor.insertText(' ' * space)
#---------------------------------------------------------------------------
# 'PythonWidget' public interface
#---------------------------------------------------------------------------
def execute_file(self, path, hidden=False):
""" Attempts to execute file with 'path'. If 'hidden', no output is
shown.
"""
self.execute("exec(open(%s).read())" % repr(path), hidden=hidden)
def reset(self):
""" Resets the widget to its initial state. Similar to ``clear``, but
also re-writes the banner.
"""
self._reading = False
self._highlighter.highlighting_on = False
self._control.clear()
self._append_plain_text(self._get_banner())
self._show_interpreter_prompt()
#---------------------------------------------------------------------------
# 'PythonWidget' protected interface
#---------------------------------------------------------------------------
def _call_tip(self):
""" Shows a call tip, if appropriate, at the current cursor location.
"""
# Decide if it makes sense to show a call tip
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.Left)
if cursor.document().characterAt(cursor.position()) != '(':
return False
context = self._get_context(cursor)
if not context:
return False
# Look up the context and show a tip for it
symbol, leftover = self._get_symbol_from_context(context)
doc = getattr(symbol, '__doc__', None)
if doc is not None and not leftover:
self._call_tip_widget.show_call_info(doc=doc)
return True
return False
def _complete(self):
""" Performs completion at the current cursor location.
"""
context = self._get_context()
if context:
symbol, leftover = self._get_symbol_from_context(context)
if len(leftover) == 1:
leftover = leftover[0]
if symbol is None:
names = list(self.interpreter.locals.keys())
names += list(six.moves.builtins.__dict__.keys())
else:
names = dir(symbol)
completions = [ n for n in names if n.startswith(leftover) ]
if completions:
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.Left,
n=len(context[-1]))
self._complete_with_items(cursor, completions)
def _get_banner(self):
""" Gets a banner to display at the beginning of a session.
"""
banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \
'"license" for more information.'
return banner % (sys.version, sys.platform)
def _get_context(self, cursor=None):
""" Gets the context for the specified cursor (or the current cursor
if none is specified).
"""
if cursor is None:
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.StartOfBlock,
QtGui.QTextCursor.KeepAnchor)
text = cursor.selection().toPlainText()
return self._completion_lexer.get_context(text)
def _get_symbol_from_context(self, context):
""" Find a python object in the interpeter namespace from a context (a
list of names).
"""
context = map(str, context)
if len(context) == 0:
return None, context
base_symbol_string = context[0]
symbol = self.interpreter.locals.get(base_symbol_string, None)
if symbol is None:
symbol = six.moves.builtins.__dict__.get(base_symbol_string, None)
if symbol is None:
return None, context
context = context[1:]
for i, name in enumerate(context):
new_symbol = getattr(symbol, name, None)
if new_symbol is None:
return symbol, context[i:]
else:
symbol = new_symbol
return symbol, []
def _show_interpreter_prompt(self):
""" Shows a prompt for the interpreter.
"""
self.flush()
self._show_prompt('>>> ')
#------ Signal handlers ----------------------------------------------------
def _document_contents_change(self, position, removed, added):
""" Called whenever the document's content changes. Display a call tip
if appropriate.
"""
# Calculate where the cursor should be *after* the change:
position += added
document = self._control.document()
if position == self._get_cursor().position():
self._call_tip()
def test_write_ini_with_reference_value_froms(
self
):
n = self._some_namespaces()
n.namespace('x1')
n.x1.add_option(
'password',
default='secret "message"',
doc='the password',
likely_to_be_changed=True,
reference_value_from='xxx.yyy'
)
n.namespace('x2')
n.x2.add_option(
'password',
default='secret "message"',
doc='the password',
reference_value_from='xxx.yyy'
)
external_values = {
'xxx': {
'yyy': {
'password': '******'
}
},
}
c = ConfigurationManager(
[n],
values_source_list=[external_values],
use_admin_controls=True,
use_auto_help=False,
argv_source=[]
)
expected = ("""# the a
#aaa=2011-05-04T15:10:00
[xxx]
#+include ./common_xxx.ini
[[yyy]]
#+include ./common_yyy.ini
# the password
password=dwight and wilma
[c]
# husband from Flintstones
#fred="stupid, deadly"
# wife from Flintstones
#wilma=waspish's
[d]
# female neighbor from I Love Lucy
#ethel=silly
# male neighbor from I Love Lucy
#fred=crabby
[x]
# the password
#password=secret "message"
# how big in tons
#size=100
[x1]
# the password
# see "xxx.yyy.password" for the default or override it here
password=dwight and wilma
[x2]
# the password
# see "xxx.yyy.password" for the default or override it here
password=dwight and wilma
""")
out = StringIO()
c.write_conf(for_configobj, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
self.assertEqual(expected.strip(), received.strip())
class PythonWidget(HistoryConsoleWidget):
""" A basic in-process Python interpreter.
"""
# Emitted when a command has been executed in the interpeter.
executed = QtCore.Signal()
#--------------------------------------------------------------------------
# 'object' interface
#--------------------------------------------------------------------------
def __init__(self, parent=None):
super(PythonWidget, self).__init__(parent)
# PythonWidget attributes.
self.locals = dict(__name__='__console__', __doc__=None)
self.interpreter = InteractiveInterpreter(self.locals)
# PythonWidget protected attributes.
self._buffer = StringIO()
self._bracket_matcher = BracketMatcher(self._control)
self._call_tip_widget = CallTipWidget(self._control)
self._completion_lexer = CompletionLexer(PythonLexer())
self._hidden = False
self._highlighter = PythonWidgetHighlighter(self)
self._last_refresh_time = 0
# file-like object attributes.
self.encoding = sys.stdin.encoding
# Configure the ConsoleWidget.
self.tab_width = 4
self._set_continuation_prompt('... ')
# Configure the CallTipWidget.
self._call_tip_widget.setFont(self.font)
self.font_changed.connect(self._call_tip_widget.setFont)
# Connect signal handlers.
document = self._control.document()
document.contentsChange.connect(self._document_contents_change)
# Display the banner and initial prompt.
self.reset()
#--------------------------------------------------------------------------
# file-like object interface
#--------------------------------------------------------------------------
def flush(self):
""" Flush the buffer by writing its contents to the screen.
"""
self._buffer.seek(0)
text = self._buffer.getvalue()
self._buffer.close()
self._buffer = StringIO()
self._append_plain_text(text)
self._control.moveCursor(QtGui.QTextCursor.End)
def readline(self, prompt=None):
""" Read and return one line of input from the user.
"""
return self._readline(prompt)
def write(self, text, refresh=True):
""" Write text to the buffer, possibly flushing it if 'refresh' is set.
"""
if not self._hidden:
self._buffer.write(text)
if refresh:
current_time = time()
if current_time - self._last_refresh_time > 0.05:
self.flush()
self._last_refresh_time = current_time
def writelines(self, lines, refresh=True):
""" Write a list of lines to the buffer.
"""
for line in lines:
self.write(line, refresh=refresh)
#---------------------------------------------------------------------------
# 'ConsoleWidget' abstract interface
#---------------------------------------------------------------------------
def _is_complete(self, source, interactive):
""" Returns whether 'source' can be completely processed and a new
prompt created. When triggered by an Enter/Return key press,
'interactive' is True; otherwise, it is False.
"""
if interactive:
lines = source.splitlines()
if len(lines) == 1:
try:
return compile_command(source) is not None
except:
# We'll let the interpeter handle the error.
return True
else:
return lines[-1].strip() == ''
else:
return True
def _execute(self, source, hidden):
""" Execute 'source'. If 'hidden', do not show any output.
See parent class :meth:`execute` docstring for full details.
"""
# Save the current std* and point them here
old_stdin = sys.stdin
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stdin = sys.stdout = sys.stderr = self
# Run the source code in the interpeter
self._hidden = hidden
try:
more = self.interpreter.runsource(source)
finally:
self._hidden = False
# Restore std* unless the executed changed them
if sys.stdin is self:
sys.stdin = old_stdin
if sys.stdout is self:
sys.stdout = old_stdout
if sys.stderr is self:
sys.stderr = old_stderr
self.executed.emit()
self._show_interpreter_prompt()
def _prompt_started_hook(self):
""" Called immediately after a new prompt is displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = True
def _prompt_finished_hook(self):
""" Called immediately after a prompt is finished, i.e. when some input
will be processed and a new prompt displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = False
def _tab_pressed(self):
""" Called when the tab key is pressed. Returns whether to continue
processing the event.
"""
# Perform tab completion if:
# 1) The cursor is in the input buffer.
# 2) There is a non-whitespace character before the cursor.
text = self._get_input_buffer_cursor_line()
if text is None:
return False
complete = bool(text[:self._get_input_buffer_cursor_column()].strip())
if complete:
self._complete()
return not complete
#---------------------------------------------------------------------------
# 'ConsoleWidget' protected interface
#---------------------------------------------------------------------------
def _event_filter_console_keypress(self, event):
""" Reimplemented for smart backspace.
"""
if event.key() == QtCore.Qt.Key_Backspace and \
not event.modifiers() & QtCore.Qt.AltModifier:
# Smart backspace: remove four characters in one backspace if:
# 1) everything left of the cursor is whitespace
# 2) the four characters immediately left of the cursor are spaces
col = self._get_input_buffer_cursor_column()
cursor = self._control.textCursor()
if col > 3 and not cursor.hasSelection():
text = self._get_input_buffer_cursor_line()[:col]
if text.endswith(' ') and not text.strip():
cursor.movePosition(QtGui.QTextCursor.Left,
QtGui.QTextCursor.KeepAnchor, 4)
cursor.removeSelectedText()
return True
return super(PythonWidget, self)._event_filter_console_keypress(event)
def _insert_continuation_prompt(self, cursor):
""" Reimplemented for auto-indentation.
"""
super(PythonWidget, self)._insert_continuation_prompt(cursor)
source = self.input_buffer
space = 0
for c in source.splitlines()[-1]:
if c == '\t':
space += 4
elif c == ' ':
space += 1
else:
break
if source.rstrip().endswith(':'):
space += 4
cursor.insertText(' ' * space)
#---------------------------------------------------------------------------
# 'PythonWidget' public interface
#---------------------------------------------------------------------------
def execute_file(self, path, hidden=False):
""" Attempts to execute file with 'path'. If 'hidden', no output is
shown.
"""
self.execute("exec(open(%s).read())" % repr(path), hidden=hidden)
def reset(self):
""" Resets the widget to its initial state. Similar to ``clear``, but
also re-writes the banner.
"""
self._reading = False
self._highlighter.highlighting_on = False
self._control.clear()
self._append_plain_text(self._get_banner())
self._show_interpreter_prompt()
#---------------------------------------------------------------------------
# 'PythonWidget' protected interface
#---------------------------------------------------------------------------
def _call_tip(self):
""" Shows a call tip, if appropriate, at the current cursor location.
"""
# Decide if it makes sense to show a call tip
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.Left)
if cursor.document().characterAt(cursor.position()) != '(':
return False
context = self._get_context(cursor)
if not context:
return False
# Look up the context and show a tip for it
symbol, leftover = self._get_symbol_from_context(context)
doc = getattr(symbol, '__doc__', None)
if doc is not None and not leftover:
self._call_tip_widget.show_call_info(doc=doc)
return True
return False
def _complete(self):
""" Performs completion at the current cursor location.
"""
context = self._get_context()
if context:
symbol, leftover = self._get_symbol_from_context(context)
if len(leftover) == 1:
leftover = leftover[0]
if symbol is None:
names = list(self.interpreter.locals.keys())
names += list(six.moves.builtins.__dict__.keys())
else:
names = dir(symbol)
completions = [n for n in names if n.startswith(leftover)]
if completions:
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.Left,
n=len(context[-1]))
self._complete_with_items(cursor, completions)
def _get_banner(self):
""" Gets a banner to display at the beginning of a session.
"""
banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \
'"license" for more information.'
return banner % (sys.version, sys.platform)
def _get_context(self, cursor=None):
""" Gets the context for the specified cursor (or the current cursor
if none is specified).
"""
if cursor is None:
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.StartOfBlock,
QtGui.QTextCursor.KeepAnchor)
text = cursor.selection().toPlainText()
return self._completion_lexer.get_context(text)
def _get_symbol_from_context(self, context):
""" Find a python object in the interpeter namespace from a context (a
list of names).
"""
context = map(str, context)
if len(context) == 0:
return None, context
base_symbol_string = context[0]
symbol = self.interpreter.locals.get(base_symbol_string, None)
if symbol is None:
symbol = six.moves.builtins.__dict__.get(base_symbol_string, None)
if symbol is None:
return None, context
context = context[1:]
for i, name in enumerate(context):
new_symbol = getattr(symbol, name, None)
if new_symbol is None:
return symbol, context[i:]
else:
symbol = new_symbol
return symbol, []
def _show_interpreter_prompt(self):
""" Shows a prompt for the interpreter.
"""
self.flush()
self._show_prompt('>>> ')
#------ Signal handlers ----------------------------------------------------
def _document_contents_change(self, position, removed, added):
""" Called whenever the document's content changes. Display a call tip
if appropriate.
"""
# Calculate where the cursor should be *after* the change:
position += added
document = self._control.document()
if position == self._get_cursor().position():
self._call_tip()
def test_write_ini_with_reference_value_froms(self):
n = self._some_namespaces()
n.namespace('x1')
n.x1.add_option('password',
default='secret "message"',
doc='the password',
likely_to_be_changed=True,
reference_value_from='xxx.yyy')
n.namespace('x2')
n.x2.add_option('password',
default='secret "message"',
doc='the password',
reference_value_from='xxx.yyy')
external_values = {
'xxx': {
'yyy': {
'password': '******'
}
},
}
c = ConfigurationManager([n],
values_source_list=[external_values],
use_admin_controls=True,
use_auto_help=False,
argv_source=[])
expected = ("""# the a
#aaa=2011-05-04T15:10:00
[xxx]
#+include ./common_xxx.ini
[[yyy]]
#+include ./common_yyy.ini
# the password
password=dwight and wilma
[c]
# husband from Flintstones
#fred="stupid, deadly"
# wife from Flintstones
#wilma=waspish's
[d]
# female neighbor from I Love Lucy
#ethel=silly
# male neighbor from I Love Lucy
#fred=crabby
[x]
# the password
#password=secret "message"
# how big in tons
#size=100
[x1]
# the password
# see "xxx.yyy.password" for the default or override it here
password=dwight and wilma
[x2]
# the password
# see "xxx.yyy.password" for the default or override it here
password=dwight and wilma
""")
out = StringIO()
c.write_conf(for_configobj, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
self.assertEqual(expected.strip(), received.strip())
def donttest_for_conf_nested_namespaces(self):
n = self._some_namespaces()
cm = ConfigurationManager(
n,
values_source_list=[],
)
out = StringIO()
cm.write_conf(for_conf, opener=stringIO_context_wrapper(out))
received = out.getvalue()
out.close()
expected = """# name: aaa
# doc: the a
aaa=2011-05-04T15:10:00
#------------------------------------------------------------------------------
# c - c space
# name: c.dwight
# doc: your uncle
c.dwight=98
# name: c.fred
# doc: husband from Flintstones
c.fred=stupid
# name: c.wilma
# doc: wife from Flintstones
c.wilma=waspish
#------------------------------------------------------------------------------
# e - e space
# name: c.e.dwight
# doc: my uncle
c.e.dwight=97
#------------------------------------------------------------------------------
# d - d space
# name: d.ethel
# doc: female neighbor from I Love Lucy
d.ethel=silly
# name: d.fred
# doc: male neighbor from I Love Lucy
d.fred=crabby
#------------------------------------------------------------------------------
# x - x space
# name: x.password
# doc: the password
x.password=secret
# name: x.size
# doc: how big in tons
x.size=100"""
self.assertEqual(received.strip(), expected)
strio = StringIO(expected)
n.c.dwight.default = 3823
n.c.e.dwight = 'fred'
cm2 = ConfigurationManager(
n,
[stringIO_context_wrapper(strio)],
use_admin_controls=False,
use_auto_help=False
)
result = cm2.get_config()
self.assertEqual(len(result), 4)
self.assertEqual(sorted(result.keys()), ['aaa', 'c', 'd', 'x'])
self.assertEqual(len(result.c), 4)
self.assertEqual(
sorted(result.c.keys()),
['dwight', 'e', 'fred', 'wilma']
)
self.assertEqual(result.c.dwight, 98)
self.assertEqual(len(result.c.e), 1)
self.assertEqual(result.c.e.dwight, '97')
def run(
params=None, # params for running from command line
map_data=None, # map_data, as_double()
pdb_inp=None,
pdb_hierarchy=None,
crystal_symmetry=None,
resolution=None,
scattering_table='n_gaussian',
smoothing_window=5,
crossover_atom='CA',
minimum_matching_atoms=3,
minimum_length=2,
dist_max=1.0,
minimum_improvement=0.01,
max_regions_to_test=10,
max_ends_per_region=5,
maximum_fraction=0.5,
max_keep=10,
map_coeffs_file=None,map_coeffs_labels=None,
pdb_in_file=None,
pdb_out=None,
verbose=None,
out=sys.stdout):
if out is None: out=sys.stdout # explode and refine calls it this way
# get info from params if present
if params:
verbose=params.control.verbose
map_coeffs_file=params.input_files.map_coeffs_file
map_coeffs_labels=params.input_files.map_coeffs_labels
pdb_in_file=params.input_files.pdb_in_file
resolution=params.crystal_info.resolution
scattering_table=params.crystal_info.scattering_table
smoothing_window=params.crossover.smoothing_window
crossover_atom=params.crossover.crossover_atom
minimum_matching_atoms=params.crossover.minimum_matching_atoms
minimum_length=params.crossover.minimum_length
dist_max=params.crossover.dist_max
minimum_improvement=params.crossover.minimum_improvement
max_regions_to_test=params.crossover.max_regions_to_test
max_ends_per_region=params.crossover.max_ends_per_region
maximum_fraction=params.crossover.maximum_fraction
max_keep=params.crossover.max_keep
pdb_out=params.output_files.pdb_out
# Consistency checks
if(pdb_hierarchy is not None):
assert pdb_in_file is None
assert pdb_inp is None
assert crystal_symmetry is not None
# XXX more checks here!
# Get map_data if not present
if not map_data:
if not map_coeffs_file or not os.path.isfile(map_coeffs_file):
raise Sorry("Cannot find the map_coeffs_file '%s'" %(
str(map_coeffs_file)))
from mmtbx.building.minimize_chain import get_map_coeffs
map_coeffs=get_map_coeffs(map_coeffs_file,
map_coeffs_labels=map_coeffs_labels)
fft_map = map_coeffs.fft_map(resolution_factor = 0.25)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
map_data=map_data.as_double()
if map_coeffs and not crystal_symmetry:
crystal_symmetry=map_coeffs.crystal_symmetry()
if map_coeffs and not resolution:
resolution=map_coeffs.d_min()
# Get the starting model
if(pdb_hierarchy is None):
if pdb_inp is None:
if not pdb_in_file or not os.path.isfile(pdb_in_file):
raise Sorry("Cannot read input PDB file '%s'" %(
str(pdb_in_file)))
else:
print("Taking models from %s" %(pdb_in_file), file=out)
pdb_string=open(pdb_in_file).read()
pdb_inp=iotbx.pdb.input(source_info=None, lines = pdb_string)
if pdb_inp is None:
raise Sorry("Need a model or models")
if not crystal_symmetry:
crystal_symmetry=pdb_inp.crystal_symmetry()
assert crystal_symmetry is not None
hierarchy = pdb_inp.construct_hierarchy()
else:
hierarchy = pdb_hierarchy # XXX FIXME
n_models=0
for model in hierarchy.models():
n_models+=1
if n_models==1: # nothing to do
return hierarchy
#xrs = pdb_inp.xray_structure_simple(crystal_symmetry=crystal_symmetry)
xrs = hierarchy.extract_xray_structure(crystal_symmetry=crystal_symmetry)
xrs.scattering_type_registry(table=scattering_table)
if not resolution:
from cctbx import maptbx
resolution=maptbx.resolution_from_map_and_model.run(
map_data=map_data, xray_structure=xrs).d_min
if(resolution is None):
raise Sorry("Resolution is required")
print("\nResolution limit: %7.2f" %(resolution), file=out)
print("\nSummary of input models", file=out)
xrs.show_summary(f=out, prefix=" ")
print("\nReady with %d models and map" %(n_models), file=out)
# Get CC by residue for each model and map
chain_id_and_resseq_list=[] # Instead set up chain_id and resseq (range)
from mmtbx.secondary_structure.find_ss_from_ca import \
split_model,get_first_resno, get_last_resno,get_chain_id
model_list=split_model(hierarchy=hierarchy,only_first_model=True)
for m in model_list:
h=m.hierarchy
first_resno=get_first_resno(h)
last_resno=get_last_resno(h)
chain_id=get_chain_id(h)
residue_range=[first_resno,last_resno]
chain_id_and_resseq=[chain_id,residue_range]
if not chain_id_and_resseq in chain_id_and_resseq_list:
chain_id_and_resseq_list.append(chain_id_and_resseq)
# Run through chains separately
# NOTE: All models of each chain must match exactly
# Save composite model, chain by chain
composite_model_stream=StringIO()
for chain_id_and_resseq in chain_id_and_resseq_list:
f=StringIO()
chain_id,[start_resno,end_resno]=chain_id_and_resseq
atom_selection=get_atom_selection(chain_id=chain_id,
start_resno=start_resno,end_resno=end_resno)
asc=hierarchy.atom_selection_cache()
sel=asc.selection(string = atom_selection)
sel_hierarchy=hierarchy.select(sel)
pdb_inp=sel_hierarchy.as_pdb_input(crystal_symmetry=crystal_symmetry)
ph=pdb_inp.construct_hierarchy()
print("\nWorking on chain_id='%s' resseq %d:%d\n" %(
chain_id_and_resseq[0],chain_id_and_resseq[1][0],chain_id_and_resseq[1][1]), file=out)
# get CC values for all residues
cc_dict=get_cc_dict(hierarchy=ph,map_data=map_data,d_min=resolution,
crystal_symmetry=crystal_symmetry,
table=scattering_table,out=out)
# smooth CC values with window of smoothing_window
smoothed_cc_dict=smooth_cc_values(cc_dict=cc_dict,
smoothing_window=smoothing_window,
verbose=verbose,out=out)
# figure out all the places where crossover can occur.
# FIXME: order of keys changes in py2/3 vthis could be bad
n_residues=cc_dict[list(cc_dict.keys())[0]].size()
crossover_dict=get_crossover_dict(
n_residues=n_residues,
hierarchy=ph,
crossover_atom=crossover_atom,
dist_max=dist_max,
minimum_matching_atoms=minimum_matching_atoms,
verbose=verbose,out=out)
# Now we are ready to identify the best composite model...
# A composite has reside 0 from model x, residue 1 from model y etc.
# Each change from model a to model b between residues i and i+1 must have
# a crossover between a and b at either residue i or i+1
keys=list(cc_dict.keys())
keys.sort()
sorted_working_model_list=[]
for key in keys:
working_model=model_object(source_id=key,
cc_dict=cc_dict,
smoothed_cc_dict=smoothed_cc_dict,
crossover_dict=crossover_dict,
minimum_length=minimum_length,
minimum_improvement=minimum_improvement,
max_regions_to_test=max_regions_to_test,
max_ends_per_region=max_ends_per_region,
maximum_fraction=maximum_fraction)
if verbose:
working_model.show_summary(out=out)
sorted_working_model_list.append(
[working_model.get_score(),working_model])
sorted_working_model_list.sort()
sorted_working_model_list.reverse()
sorted_working_model_list=\
sorted_working_model_list[:max_keep]
working_model_list=[]
for s,m in sorted_working_model_list:
working_model_list.append(m)
# Go through all the working models and cross them with other models to
# optimize...Then take all the best and cross...
best_score,best_model=sorted_working_model_list[0]
found=True
cycle=0
while found:
cycle+=1
print("\nCYCLE %d current best is %7.3f\n" %(
cycle,best_model.get_score()), file=out)
found=False
sorted_working_model_list=[]
new_best=best_model
id=0
for working_model in working_model_list:
id+=1
others=[]
for m in working_model_list:
if not working_model==m: others.append(m)
new_working_model=working_model.optimize_with_others(others=others)
if not new_working_model:
print()
continue
aa=[new_working_model.get_score(),new_working_model]
if not aa in sorted_working_model_list:
sorted_working_model_list.append(aa)
if not sorted_working_model_list:
break # nothing to do
sorted_working_model_list.sort()
sorted_working_model_list.reverse()
sorted_working_model_list=sorted_working_model_list[:max_keep]
new_working_score,new_working_model=sorted_working_model_list[0]
if new_working_score>best_model.get_score():
best_model=new_working_model
found=True
if verbose:
print("NEW BEST SCORE: %7.2f" %(best_model.get_score()), file=out)
best_model.show_summary(out=out)
print("\nDONE... best is %7.3f\n" %(
best_model.get_score()), file=out)
# Create composite of this chain
# Note residue values. We are going to pick each residue from one of
# the models
for model in ph.models():
for chain in model.chains():
if chain.id != chain_id: continue
residue_list=[]
for rg in chain.residue_groups():
residue_list.append(rg.resseq)
residue_list.sort()
assert len(best_model.source_list)==len(residue_list)
for i in range(len(residue_list)):
atom_selection=get_atom_selection(model_id=best_model.source_list[i],
resseq_sel=residue_list[i])
asc=ph.atom_selection_cache()
sel=asc.selection(string = atom_selection)
sel_hierarchy=ph.select(sel)
print(remove_ter(sel_hierarchy.as_pdb_string()), file=composite_model_stream)
# All done, make a new pdb_hierarchy
pdb_string=composite_model_stream.getvalue()
pdb_inp=iotbx.pdb.input(source_info=None, lines = pdb_string)
pdb_hierarchy=pdb_inp.construct_hierarchy()
if pdb_out:
f=open(pdb_out,'w')
print(pdb_hierarchy.as_pdb_string(crystal_symmetry=crystal_symmetry), file=f)
print("Final model is in: %s\n" %(f.name))
f.close()
return pdb_hierarchy
def main():
parser = argparse.ArgumentParser(description='Print tables pretty')
parser.add_argument('infile',
type=str,
nargs='?',
help='input file, default is stdin')
parser.add_argument(
'-d',
dest='delimiter',
type=str,
required=False,
help='delimiter of fields of input. Default is white space.')
parser.add_argument(
'-j',
dest='justify',
type=str,
required=False,
default='left',
choices=['left', 'right', 'center'],
help='justification, either left, right or center. Default is left')
parser.add_argument('-s',
dest='separator',
type=str,
required=False,
default=' ',
help='separator of fields in output')
args = parser.parse_args()
table = []
maxwidth = []
# default is to read from stdin
fin = sys.stdin
if args.infile:
try:
fin = open(args.infile, 'rt')
except IOError as e:
print('Error: %s: %s\n' % (e.strerror, args.infile),
file=sys.stderr)
sys.exit(e.errno)
try:
for line in fin:
fields = None
# split line by delimiter
if args.delimiter:
fields = line.strip().split(args.delimiter)
else:
fields = line.strip().split()
for i in xrange(len(fields)):
width = len(fields[i])
if (i + 1) > len(maxwidth):
maxwidth.append(width)
else:
if width > maxwidth[i]:
maxwidth[i] = width
table.append(fields)
fin.close()
for fields in table:
line = StringIO()
for i in xrange(len(fields)):
# format field with different justification
nSpace = maxwidth[i] - len(fields[i])
if args.justify == 'left':
line.write(fields[i])
for j in xrange(nSpace):
line.write(' ')
elif args.justify == 'right':
for j in xrange(nSpace):
line.write(' ')
line.write(fields[i])
elif args.justify == 'center':
for j in xrange(nSpace / 2):
line.write(' ')
line.write(fields[i])
for j in xrange(nSpace - nSpace / 2):
line.write(' ')
line.write(args.separator)
print(line.getvalue())
line.close()
except IOError as e:
if e.errno == errno.EPIPE:
sys.exit(-e.errno)
else:
raise e
except KeyboardInterrupt:
pass
def vmd_visualize_unified(pdb_filename, trajectory_filename, cplx_filename,
step):
"""Function that creates a string in a syntax that can be interpreted by vmd.
This string can be saved to a text/data file.
Includes a comment-header. The very last printed lines prevent vmd from
automatically creating an additional Representation.
This Function creates a very basic representation of the given topology:
All simulation beads are displayed as vdw,particles of defined radius
(default = 1.0)
Parameters
----------
pdb_filename: String
Containing name of pdb-file (parsed argument)
trajectory_filename: String
Containing name of trajectory file (parsed argument)
cplx_filename: String
Containing name of cplx-file (parsed argument)
Returns
-------
String
the complete string to be written to data-file
"""
vdw_radius = 1.0
trajectory_type = os.path.splitext(trajectory_filename)[-1][1:]
out = StringIO()
out.write(
HEADER_FORMAT.format(
generation_time=datetime.datetime.time(datetime.datetime.now()),
topology_inputfile=cplx_filename,
pdb_inputfile=pdb_filename,
trajectory_inputfile=trajectory_filename,
))
out.write(LOAD_VMD_TOP_FORMAT.format(vmd_top=pdb_filename))
out.write(
LOADTRAJ_FORMAT.format(traj_file=trajectory_filename,
traj_type=trajectory_type,
step=step))
out.write(
"#adding representation to molecule 0 \n #################################### \n"
)
colorid = 1
rep_type = "VDW"
rep_param = "{} 12.000000".format(vdw_radius)
rep_number = 0
out.write(
ADDREP_FORMAT.format(
colorid=colorid,
rep_number=rep_number,
mol_number=0,
resid_string="all",
rep_type=rep_type,
rep_param=rep_param,
))
rep_counter = 1
out.write(DEL_REP_FORMAT.format(rep_counter=rep_counter))
out.write(SHOW_SIM_BOX)
rep = out.getvalue()
out.close()
return rep
def write_default_ansible_cfg(work_dir,
remote_user,
ssh_private_key=None,
transport=None,
base_ansible_cfg='/etc/ansible/ansible.cfg',
override_ansible_cfg=None):
ansible_config_path = os.path.join(work_dir, 'ansible.cfg')
shutil.copy(base_ansible_cfg, ansible_config_path)
modules_path = ('/root/.ansible/plugins/modules:'
'/usr/share/ansible/plugins/modules:'
'%s/library' % constants.DEFAULT_VALIDATIONS_BASEDIR)
lookups_path = (
'root/.ansible/plugins/lookup:'
'/usr/share/ansible/plugins/lookup:'
'%s/lookup_plugins' % constants.DEFAULT_VALIDATIONS_BASEDIR)
callbacks_path = (
'~/.ansible/plugins/callback:'
'/usr/share/ansible/plugins/callback:'
'%s/callback_plugins' % constants.DEFAULT_VALIDATIONS_BASEDIR)
roles_path = ('%(work_dir)s/roles:'
'/root/.ansible/roles:'
'/usr/share/ansible/roles:'
'/etc/ansible/roles:'
'%(ooo_val_path)s/roles:'
'%(work_dir)s' % {
'work_dir': work_dir,
'ooo_val_path': constants.DEFAULT_VALIDATIONS_BASEDIR
})
config = configparser.ConfigParser()
config.read(ansible_config_path)
config.set('defaults', 'retry_files_enabled', 'False')
config.set('defaults', 'roles_path', roles_path)
config.set('defaults', 'library', modules_path)
config.set('defaults', 'callback_plugins', callbacks_path)
config.set('defaults', 'lookup_plugins', lookups_path)
log_path = os.path.join(work_dir, 'ansible.log')
config.set('defaults', 'log_path', log_path)
if os.path.exists(log_path):
new_path = (log_path + '-' +
datetime.now().strftime("%Y-%m-%dT%H:%M:%S"))
os.rename(log_path, new_path)
config.set('defaults', 'forks', '25')
config.set('defaults', 'timeout', '30')
config.set('defaults', 'gather_timeout', '30')
# mistral user has no home dir set, so no place to save a known hosts file
config.set('ssh_connection', 'ssh_args',
'-o UserKnownHostsFile=/dev/null '
'-o StrictHostKeyChecking=no '
'-o ControlMaster=auto '
'-o ControlPersist=30m '
'-o ServerAliveInterval=5 '
'-o ServerAliveCountMax=5')
config.set('ssh_connection', 'control_path_dir',
os.path.join(work_dir, 'ansible-ssh'))
config.set('ssh_connection', 'retries', '8')
config.set('ssh_connection', 'pipelining', 'True')
# Set connection info in config file so that subsequent/nested ansible
# calls can re-use it
if remote_user:
config.set('defaults', 'remote_user', remote_user)
if ssh_private_key:
config.set('defaults', 'private_key_file', ssh_private_key)
if transport:
config.set('defaults', 'transport', transport)
if override_ansible_cfg:
sio_cfg = StringIO()
sio_cfg.write(override_ansible_cfg)
sio_cfg.seek(0)
config.readfp(sio_cfg)
sio_cfg.close()
with open(ansible_config_path, 'w') as configfile:
config.write(configfile)
return ansible_config_path
