def test_find_replace_enable(self):
db_file = os.path.join(self.testdir, 'hash.db')
broker = ContainerBroker(db_file)
broker.account = 'a'
broker.container = 'c'
broker.initialize()
ts = utils.Timestamp.now()
broker.merge_items([
{'name': 'obj%02d' % i, 'created_at': ts.internal, 'size': 0,
'content_type': 'application/octet-stream', 'etag': 'not-really',
'deleted': 0, 'storage_policy_index': 0,
'ctype_timestamp': ts.internal, 'meta_timestamp': ts.internal}
for i in range(100)])
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
with mock_timestamp_now() as now:
main([broker.db_file, 'find_and_replace', '10', '--enable'])
expected = [
'No shard ranges found to delete.',
'Injected 10 shard ranges.',
'Run container-replicator to replicate them to other nodes.',
"Container moved to state 'sharding' with epoch %s." %
now.internal,
'Run container-sharder on all nodes to shard the container.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self._assert_enabled(broker, now)
self.assertEqual(
[(data['lower'], data['upper']) for data in self.shard_data],
[(sr.lower_str, sr.upper_str) for sr in broker.get_shard_ranges()])
def _parse_operator(segment, iterator):
"""Parses the operator (eg. '==' or '<')."""
stream = StringIO()
for character in iterator:
if character == constants.NEGATION[1]:
if stream.tell():
# Negation can only occur at the start of an operator.
raise ValueError('Unexpected negation.')
# We've been negated.
segment.negated = not segment.negated
continue
if (stream.getvalue() + character not in OPERATOR_SYMBOL_MAP and
stream.getvalue() + character not in OPERATOR_BEGIN_CHARS):
# We're no longer an operator.
break
# Expand the operator
stream.write(character)
# Check for existance.
text = stream.getvalue()
if text not in OPERATOR_SYMBOL_MAP:
# Doesn't exist because of a mis-placed negation in the middle
# of the path.
raise ValueError('Unexpected negation.')
# Set the found operator.
segment.operator = OPERATOR_SYMBOL_MAP[text]
# Return the remaining characters.
return chain(character, iterator)
def test_push(empty_db,newstatus):
db = empty_db
input = remove_email(inv_pre) if db.member_class else inv_pre
db.import_invitations(input,allfields=True,format='json')
input = copy.deepcopy(inv_down)
input['data'] = input['data'][2:]
input = StringIO(json.dumps(input))
output = StringIO()
# ignore registering
msg = dict(format='member',version=(1,0),status='',uuid='uid06')
for status in ('registering','new'):
msg['status'] = status
assert not InvitationDatabase.process_update(db,msg,input,output)
assert not output.getvalue()
# registered leads to sync
msg['status'] = newstatus
assert InvitationDatabase.process_update(db,msg,input,output)
output = json.loads(output.getvalue())
assert output
up = copy.deepcopy(inv_up)
up['data'] = up['data'][2:6]
output['data'] = sort_by_uuid(output['data'])
assert output == up
# not again
assert not db.process_update(msg,input,output)
class IncludedResponse(object):
def __init__(self):
self.headers = None
self.status = None
self.output = StringIO()
self.str = None
def close(self):
self.str = self.output.getvalue()
self.output.close()
self.output = None
def write(self, s):
assert self.output is not None, (
"This response has already been closed and no further data "
"can be written.")
self.output.write(s)
def __str__(self):
return self.body
def body__get(self):
if self.str is None:
return self.output.getvalue()
else:
return self.str
body = property(body__get)
def test_fail(self):
self.basedir = basedir = os.path.join("backupdb", "fail")
fileutil.make_dirs(basedir)
# put a non-DB file in the way
not_a_db = ("I do not look like a sqlite database\n" +
"I'M NOT" * 1000) # OS-X sqlite-2.3.2 takes some convincing
self.writeto("not-a-database", not_a_db)
stderr_f = StringIO()
bdb = backupdb.get_backupdb(os.path.join(basedir, "not-a-database"),
stderr_f)
self.failUnlessEqual(bdb, None)
stderr = stderr_f.getvalue()
self.failUnlessIn("backupdb file is unusable", stderr)
# sqlite-3.19.3 says "file is encrypted or is not a database"
# sqlite-3.20.0 says "file is not a database"
self.failUnlessIn("is not a database", stderr)
# put a directory in the way, to exercise a different error path
where = os.path.join(basedir, "roadblock-dir")
fileutil.make_dirs(where)
stderr_f = StringIO()
bdb = backupdb.get_backupdb(where, stderr_f)
self.failUnlessEqual(bdb, None)
stderr = stderr_f.getvalue()
self.failUnlessIn("Unable to create/open backupdb file %s" % (where,), stderr)
self.failUnlessIn("unable to open database file", stderr)
def test_export(member_db):
memfile = StringIO()
depfile = StringIO()
member_db.export_members([memfile,depfile],allfields=True,format='csv')
members = memfile.getvalue().replace('r2sub;sub2','sub2;r2sub') # normalize order
assert members==members_export
assert depfile.getvalue()==deps_export
def test_simple(self):
t = termlog.TermLog()
sio = StringIO()
t.configure(dump.Options(tfile = sio, verbosity = 2), set([]))
t.log(controller.LogEntry("one", "info"))
assert "one" in sio.getvalue()
t.log(controller.LogEntry("two", "debug"))
assert "two" not in sio.getvalue()
def test_print_obj_valid(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=self.testdir)
etag_msg = 'ETag: Not found in metadata'
length_msg = 'Content-Length: Not found in metadata'
self.assertTrue(etag_msg in out.getvalue())
self.assertTrue(length_msg in out.getvalue())
def test_print_obj_no_ring(self):
no_rings_dir = os.path.join(self.testdir, 'no_rings_here')
os.mkdir(no_rings_dir)
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=no_rings_dir)
self.assertTrue('d41d8cd98f00b204e9800998ecf8427e' in out.getvalue())
self.assertTrue('Partition' not in out.getvalue())
def test_print_ring_locations_account(self):
out = StringIO()
with mock.patch('sys.stdout', out):
acctring = ring.Ring(self.testdir, ring_name='account')
print_ring_locations(acctring, 'dir', 'acct')
exp_db = os.path.join('${DEVICE:-/srv/node*}', 'sdb1', 'dir', '3',
'b47', 'dc5be2aa4347a22a0fee6bc7de505b47')
self.assertTrue(exp_db in out.getvalue())
self.assertTrue('127.0.0.1' in out.getvalue())
self.assertTrue('127.0.0.2' in out.getvalue())
def test_print_item_locations_ring_policy_mismatch_no_target(self):
out = StringIO()
with mock.patch('sys.stdout', out):
objring = ring.Ring(self.testdir, ring_name='object')
# Test mismatch of ring and policy name (valid policy)
self.assertRaises(InfoSystemExit, print_item_locations,
objring, policy_name='zero')
self.assertTrue('Warning: mismatch between ring and policy name!'
in out.getvalue())
self.assertTrue('No target specified' in out.getvalue())
def test_print_obj_with_policy(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=self.testdir, policy_name='one')
etag_msg = 'ETag: Not found in metadata'
length_msg = 'Content-Length: Not found in metadata'
ring_loc_msg = 'ls -lah'
self.assertTrue(etag_msg in out.getvalue())
self.assertTrue(length_msg in out.getvalue())
self.assertTrue(ring_loc_msg in out.getvalue())
def test_print_item_locations_invalid_policy_no_target(self):
out = StringIO()
policy_name = 'nineteen'
with mock.patch('sys.stdout', out):
objring = ring.Ring(self.testdir, ring_name='object')
self.assertRaises(InfoSystemExit, print_item_locations,
objring, policy_name=policy_name)
exp_msg = 'Warning: Policy %s is not valid' % policy_name
self.assertTrue(exp_msg in out.getvalue())
self.assertTrue('No target specified' in out.getvalue())
def test_send_chunk():
v = "foobarfoobar"
for bs in range(1, len(v) + 2):
s = StringIO()
writer.send_chunk(s, v, bs, 0, len(v))
assert s.getvalue() == v
for start in range(len(v)):
for end in range(len(v)):
s = StringIO()
writer.send_chunk(s, v, bs, start, end)
assert s.getvalue() == v[start:end]
def test_write_values():
tst = "foobarvoing"
s = StringIO()
writer.write_values(s, [tst], [])
assert s.getvalue() == tst
for bs in range(1, len(tst) + 2):
for off in range(len(tst)):
s = StringIO()
writer.write_values(s, [tst], [(off, "disconnect")], blocksize=bs)
assert s.getvalue() == tst[:off]
def test_print_item_locations_account_no_ring(self):
out = StringIO()
account = 'account'
with mock.patch('sys.stdout', out):
print_item_locations(None, account=account,
swift_dir=self.testdir)
exp_acct_msg = 'Account \t%s' % account
exp_cont_msg = 'Container\tNone'
exp_obj_msg = 'Object \tNone'
self.assertTrue(exp_acct_msg in out.getvalue())
self.assertTrue(exp_cont_msg in out.getvalue())
self.assertTrue(exp_obj_msg in out.getvalue())
def test_raw(self):
s = StringIO()
r = language.parse_pathod("400:b'foo'").next()
language.serve(r, s, {})
v = s.getvalue()
assert "Content-Length" in v
s = StringIO()
r = language.parse_pathod("400:b'foo':r").next()
language.serve(r, s, {})
v = s.getvalue()
assert "Content-Length" not in v
def test_print_item_locations_policy_object(self):
out = StringIO()
part = '1'
with mock.patch('sys.stdout', out):
print_item_locations(None, partition=part, policy_name='zero',
swift_dir=self.testdir)
exp_part_msg = 'Partition\t%s' % part
exp_acct_msg = 'Account \tNone'
exp_cont_msg = 'Container\tNone'
exp_obj_msg = 'Object \tNone'
self.assertTrue(exp_part_msg in out.getvalue())
self.assertTrue(exp_acct_msg in out.getvalue())
self.assertTrue(exp_cont_msg in out.getvalue())
self.assertTrue(exp_obj_msg in out.getvalue())
def test_print_item_locations_account_container_ring(self):
out = StringIO()
account = 'account'
container = 'container'
with mock.patch('sys.stdout', out):
container_ring = ring.Ring(self.testdir, ring_name='container')
print_item_locations(container_ring, account=account,
container=container)
exp_acct_msg = 'Account \t%s' % account
exp_cont_msg = 'Container\t%s' % container
exp_obj_msg = 'Object \tNone'
self.assertTrue(exp_acct_msg in out.getvalue())
self.assertTrue(exp_cont_msg in out.getvalue())
self.assertTrue(exp_obj_msg in out.getvalue())
def test_write_values_inject():
tst = "foo"
s = StringIO()
writer.write_values(s, [tst], [(0, "inject", "aaa")], blocksize=5)
assert s.getvalue() == "aaafoo"
s = StringIO()
writer.write_values(s, [tst], [(1, "inject", "aaa")], blocksize=5)
assert s.getvalue() == "faaaoo"
s = StringIO()
writer.write_values(s, [tst], [(1, "inject", "aaa")], blocksize=5)
assert s.getvalue() == "faaaoo"
def test_print_item_locations_dashed_ring_name_partition(self):
out = StringIO()
part = '1'
with mock.patch('sys.stdout', out):
print_item_locations(None, policy_name='one',
ring_name='foo-bar', partition=part,
swift_dir=self.testdir)
exp_part_msg = 'Partition\t%s' % part
exp_acct_msg = 'Account \tNone'
exp_cont_msg = 'Container\tNone'
exp_obj_msg = 'Object \tNone'
self.assertIn(exp_part_msg, out.getvalue())
self.assertIn(exp_acct_msg, out.getvalue())
self.assertIn(exp_cont_msg, out.getvalue())
self.assertIn(exp_obj_msg, out.getvalue())
def generate_roles_data_from_directory(directory, roles, validate=True):
"""Generate a roles data file using roles from a local path
:param directory local filesystem path to the roles
:param roles ordered list of roles
:param validate validate the metadata format in the role yaml files
:returns string contents of the roles_data.yaml
"""
available_roles = get_roles_list_from_directory(directory)
check_role_exists(available_roles, roles)
output = StringIO()
header = ["#" * 79,
"# File generated by TripleO",
"#" * 79,
""]
output.write("\n".join(header))
for role in roles:
defined_role = role.split(':')[0]
file_path = os.path.join(directory, "{}.yaml".format(defined_role))
if validate:
validate_role_yaml(role_path=file_path)
with open(file_path, "r") as f:
if ':' in role:
generated_role = role.split(':')[1]
content = generate_role_with_colon_format(f.read(),
defined_role,
generated_role)
output.write(content)
else:
shutil.copyfileobj(f, output)
return output.getvalue()
def pathoc(
self,
specs,
timeout=None,
connect_to=None,
ssl=None,
ws_read_limit=None,
use_http2=False,
):
"""
Returns a (messages, text log) tuple.
"""
if ssl is None:
ssl = self.ssl
logfp = StringIO()
c = pathoc.Pathoc(
("localhost", self.d.port),
ssl=ssl,
ws_read_limit=ws_read_limit,
timeout=timeout,
fp=logfp,
use_http2=use_http2,
)
with c.connect(connect_to):
ret = []
for i in specs:
resp = c.request(i)
if resp:
ret.append(resp)
for frm in c.wait():
ret.append(frm)
c.stop()
return ret, logfp.getvalue()
def test_item_not_in_defaults(self):
_stdout = StringIO()
class RootController(object):
@expose()
def index(self):
return 'Hello, World!'
app = TestApp(
make_app(
RootController(),
hooks=lambda: [
RequestViewerHook(
config={'items': ['date']}, writer=_stdout
)
]
)
)
response = app.get('/')
out = _stdout.getvalue()
assert response.status_int == 200
assert response.body == b_('Hello, World!')
assert 'date' in out
assert 'method' not in out
assert 'status' not in out
assert 'method' not in out
assert 'params' not in out
assert 'hooks' not in out
assert '200 OK' not in out
assert "['RequestViewerHook']" not in out
assert '/' not in out
def test_print_item_locations_account_container_object_dashed_ring(self):
out = StringIO()
account = 'account'
container = 'container'
obj = 'object'
with mock.patch('sys.stdout', out):
object_ring = ring.Ring(self.testdir, ring_name='object-1')
print_item_locations(object_ring, ring_name='object-1',
account=account, container=container,
obj=obj)
exp_acct_msg = 'Account \t%s' % account
exp_cont_msg = 'Container\t%s' % container
exp_obj_msg = 'Object \t%s' % obj
self.assertTrue(exp_acct_msg in out.getvalue())
self.assertTrue(exp_cont_msg in out.getvalue())
self.assertTrue(exp_obj_msg in out.getvalue())
def test_debug_print_option_totals_color(self):
prob = Problem()
prob.model = FanInGrouped()
prob.model.linear_solver = LinearBlockGS()
prob.model.sub.linear_solver = LinearBlockGS()
prob.model.add_design_var('iv.x1', parallel_deriv_color='par_dv')
prob.model.add_design_var('iv.x2', parallel_deriv_color='par_dv')
prob.model.add_design_var('iv.x3')
prob.model.add_objective('c3.y')
prob.driver.options['debug_print'] = ['totals']
prob.setup(check=False, mode='fwd')
prob.set_solver_print(level=0)
prob.run_driver()
indep_list = ['iv.x1', 'iv.x2', 'iv.x3']
unknown_list = ['c3.y']
stdout = sys.stdout
strout = StringIO()
sys.stdout = strout
try:
_ = prob.compute_totals(unknown_list, indep_list, return_format='flat_dict',
debug_print=not prob.comm.rank)
finally:
sys.stdout = stdout
output = strout.getvalue()
if not prob.comm.rank:
self.assertTrue('Solving color: par_dv (iv.x1, iv.x2)' in output)
self.assertTrue('Solving variable: iv.x3' in output)
def __str__(self):
buffer = StringIO()
if self.refTime is not None:
refTimeInSecs = self.refTime.getTime() / 1000
micros = (self.refTime.getTime() % 1000) * 1000
dtObj = datetime.datetime.utcfromtimestamp(refTimeInSecs)
dtObj = dtObj.replace(microsecond=micros)
# This won't be compatible with java or string from java since its to microsecond
buffer.write(dtObj.isoformat(' '))
if "FCST_USED" in self.utilityFlags:
hrs = int(self.fcstTime / 3600)
mins = int((self.fcstTime - (hrs * 3600)) / 60)
buffer.write(" (" + str(hrs))
if mins != 0:
buffer.write(":" + str(mins))
buffer.write(")")
if "PERIOD_USED" in self.utilityFlags:
buffer.write("[")
buffer.write(self.validPeriod.start.isoformat(' '))
buffer.write("--")
buffer.write(self.validPeriod.end.isoformat(' '))
buffer.write("]")
strVal = buffer.getvalue()
buffer.close()
return strVal
class _MiniPPrinter(object):
def __init__(self):
self._out = StringIO()
self.indentation = 0
def text(self, text):
self._out.write(text)
def breakable(self, sep=" "):
self._out.write(sep)
def begin_group(self, _, text):
self.text(text)
def end_group(self, _, text):
self.text(text)
def pretty(self, obj):
if hasattr(obj, "_repr_pretty_"):
obj._repr_pretty_(self, False)
else:
self.text(repr(obj))
def getvalue(self):
return self._out.getvalue()
def test_no_historical(self):
out = StringIO()
with replace_registry():
management.call_command(self.command_name, auto=True,
stdout=out)
self.assertIn(populate_history.Command.NO_REGISTERED_MODELS,
out.getvalue())
def test_single_blacklist_item(self):
_stdout = StringIO()
class RootController(object):
@expose()
def index(self):
return 'Hello, World!'
app = TestApp(
make_app(
RootController(),
hooks=lambda: [
RequestViewerHook(
config={'blacklist': ['/']}, writer=_stdout
)
]
)
)
response = app.get('/')
out = _stdout.getvalue()
assert response.status_int == 200
assert response.body == b_('Hello, World!')
assert out == ''
def initialize_db(self, indices):
db = self.connection()
print "testing for tables"
cursor = db.cursor()
cursor.execute("SHOW TABLES from %s;"%self.params.mysql.database)
all_tables = cursor.fetchall()
# Beware of SQL injection vulnerability (here and elsewhere).
new_tables = self.merging_schema_tables(self.params.mysql.runtag)
for table in new_tables:
cursor.execute("DROP TABLE IF EXISTS %s;"%table[0])
cursor.execute("CREATE TABLE %s "%table[0]+table[1].replace("\n"," ")+" ;")
from six.moves import cStringIO as StringIO
query = StringIO()
query.write("INSERT INTO `%s_miller` (h,k,l) VALUES "%self.params.mysql.runtag)
firstcomma = ""
for item in indices:
query.write(firstcomma); firstcomma=","
query.write("('%d','%d','%d')"%(item[0],item[1],item[2]))
query.write(" ;")
cursor.execute( query.getvalue() )
db.commit()
def multipart_encode(vars, files, boundary=None, buf=None):
if boundary is None:
boundary = mimetools.choose_boundary()
if buf is None:
buf = StringIO()
for(key, value) in vars:
buf.write('--%s\r\n' % boundary)
buf.write('Content-Disposition: form-data; name="%s"' % key)
buf.write('\r\n\r\n' + value + '\r\n')
for(key, fd) in files:
file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
filename = fd.name.split('/')[-1]
contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
buf.write('--%s\r\n' % boundary)
buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
buf.write('Content-Type: %s\r\n' % contenttype)
# buffer += 'Content-Length: %s\r\n' % file_size
fd.seek(0)
buf.write('\r\n' + fd.read() + '\r\n')
buf.write('--' + boundary + '--\r\n\r\n')
buf = buf.getvalue()
return boundary, buf
def run_modernize(paths):
if libmodernize_main is None:
return 0
print('Running modernize checks')
try:
orig_stdout = getattr(sys, 'stdout')
out = StringIO()
setattr(sys, 'stdout', out)
ret = libmodernize_main(paths)
finally:
sys.stdout = orig_stdout
output = out.getvalue()
print(output)
has_patch_lines = any(
line.startswith(('+++', '---')) for line in output.splitlines())
if has_patch_lines or ret != 0:
print('modernize failed')
return max(ret, 1)
print('modernize passed')
return 0
def test_new_query_mocked(self, N, Process):
"""
A basic functionality test, in a case where everything is just normal.
"""
_configure_mock(N, Process)
gpustats = gpustat.new_query()
fp = StringIO()
gpustats.print_formatted(
fp=fp, no_color=False, show_user=True,
show_cmd=True, show_pid=True, show_power=True
)
result = fp.getvalue()
print(result)
unescaped = remove_ansi_codes(result)
# remove first line (header)
unescaped = '\n'.join(unescaped.split('\n')[1:])
self.maxDiff = 4096
self.assertEqual(unescaped, MOCK_EXPECTED_OUTPUT_FULL)
def tst_01():
print("Comparing mixed model with target...")
import iotbx.pdb
from cctbx.array_family import flex
model_pdb_inp=iotbx.pdb.input(source_info='model',
lines=flex.split_lines(model))
crystal_symmetry=model_pdb_inp.crystal_symmetry()
model_hierarchy=model_pdb_inp.construct_hierarchy()
query_hierarchy=iotbx.pdb.input(source_info='query',
lines=flex.split_lines(query)).construct_hierarchy()
f=StringIO()
r=run(crystal_symmetry=crystal_symmetry,
chain_hierarchy=query_hierarchy,target_hierarchy=model_hierarchy,out=f)
expected_text=\
"""SEQ SCORE is fraction (close and matching target sequence).
MEAN LENGTH is the mean length of contiguous segments in the match with target sequence. (Each gap/reverse of direction starts new segment).
----ALL RESIDUES--- CLOSE RESIDUES ONLY %
MODEL --CLOSE- --FAR-- FORWARD REVERSE MIXED FOUND CA SEQ
RMSD N N N N N SCORE SEQ MATCH(%) SCORE MEAN LENGTH FRAGMENTS BAD CONNECTIONS
Unique_target 1.55 54 7 14 29 11 39.7 0.26 9.3 0.04 6.0 7 6"""
found_text="\n".join(f.getvalue().splitlines()[-10:])
if remove_blank(found_text)!=remove_blank(expected_text):
print("Expected: \n%s \nFound: \n%s" %(expected_text,found_text))
raise AssertionError("FAILED")
from libtbx.test_utils import approx_equal
print(r.get_values("forward"))
assert approx_equal(r.get_values("forward"),(1.6751069901864204, 14))
print(r.get_values("reverse"))
assert approx_equal(r.get_values("reverse"),(1.388466550576198, 29))
print(r.get_values("close"))
assert approx_equal(r.get_values("close"),(1.545835235099158, 54))
print(r.get_values("all_far"))
assert approx_equal(r.get_values("all_far"),(0,0))
print("OK")
def test_compact_expansion_limit_less_than_shrink_threshold(self):
broker = self._make_broker()
shard_ranges = make_shard_ranges(broker, self.shard_data, '.shards_')
for i, sr in enumerate(shard_ranges):
if i % 2:
sr.object_count = 25
else:
sr.object_count = 3
sr.update_state(ShardRange.ACTIVE)
broker.merge_shard_ranges(shard_ranges)
self._move_broker_to_sharded_state(broker)
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
ret = main([
broker.db_file, 'compact', '--yes', '--shrink-threshold', '10',
'--expansion-limit', '5'
])
self.assertEqual(0, ret)
out_lines = out.getvalue().split('\n')
self.assertEqual(['No shards identified for compaction.'],
out_lines[:1])
def after_request(self, response):
accept_encoding = request.headers.get('Accept-Encoding', '')
if response.status_code < 200 or \
response.status_code >= 300 or \
response.content_length < self.minimum_size or \
'gzip' not in accept_encoding.lower() or \
'Content-Encoding' in response.headers:
return response
print 'GZIPPING'
response.direct_passthrough = False
gzip_buffer = StringIO()
gzip_file = gzip.GzipFile(mode='wb',
compresslevel=self.compress_level,
fileobj=gzip_buffer)
gzip_file.write(response.data)
gzip_file.close()
response.data = gzip_buffer.getvalue()
response.headers['Content-Encoding'] = 'gzip'
response.headers['Content-Length'] = len(response.data)
return response
def exec_expr(self, s):
out = StringIO()
exec_lock.acquire()
save_stdout = sys.stdout
try:
debugger = _OutputRedirectingPdb(save_stdout)
debugger.reset()
pdb.set_trace = debugger.set_trace
sys.stdout = out
try:
code = compile(s, '<web>', "single", 0, 1)
six.exec_(code, self.globs, self.namespace)
debugger.set_continue()
except KeyboardInterrupt:
raise
except:
traceback.print_exc(file=out)
debugger.set_continue()
finally:
sys.stdout = save_stdout
exec_lock.release()
return out.getvalue()
def test_single_blacklist_item(self):
_stdout = StringIO()
class RootController(object):
@expose()
def index(self):
return 'Hello, World!'
app = TestApp(
make_app(RootController(),
hooks=lambda: [
RequestViewerHook(config={'blacklist': ['/']},
writer=_stdout)
]))
response = app.get('/')
out = _stdout.getvalue()
assert response.status_int == 200
assert response.body == b_('Hello, World!')
assert out == ''
def display(self, add_comments=False):
'''Display options in a config file form.'''
output = StringIO()
keys = self._options.keys()
keys.sort()
currentSection = None
for sect, opt in keys:
if sect != currentSection:
if currentSection is not None:
output.write('\n')
output.write('[')
output.write(sect)
output.write("]\n")
currentSection = sect
if add_comments:
doc = self._options[sect, opt].doc()
if not doc:
doc = "No information available, sorry."
doc = re.sub(r"\s+", " ", doc)
output.write("\n# %s\n" % ("\n# ".join(wrap(doc)),))
self._options[sect, opt].write_config(output)
return output.getvalue()
def check(d, s, f, exception_expected=True):
if (s is not None): s = flex.double([s])
ma2 = ma.array(data=flex.double([d]), sigmas=s)
sio = StringIO()
ma2.export_as_shelx_hklf(sio, normalise_if_format_overflow=True)
assert not show_diff(
sio.getvalue(), """\
1 2 -1%s
0 0 0 0.00 0.00
""" % f)
if exception_expected:
try:
ma2.export_as_shelx_hklf(sio)
except RuntimeError:
pass
else:
raise Exception_expected
else:
try:
ma2.export_as_shelx_hklf(sio)
except RuntimeError:
raise Exception_not_expected
def exercise():
if (iotbx_mtz is None):
print(
"Skipping iotbx/examples/tst_mtz_free_flipper.py: ccp4io not available"
)
return
input_file_name = libtbx.env.find_in_repositories(
relative_path="phenix_regression/reflection_files/l.mtz",
test=os.path.isfile)
if (input_file_name is None):
print("Skipping exercise(): input file not available")
return
label = "R-free-flags(-)"
mtz_free_flipper.run(args=[input_file_name], label=label)
mtz_free_flipper.run(args=["free_flipped_l.mtz"], label=label)
mtz_convert_free_to_work.run(args=[input_file_name], label=label)
spreadsheets = []
for file_name, expected in [(input_file_name, (13469, 1065, 2323)),
("free_flipped_l.mtz", (1065, 13469, 2323)),
("free_flipped_free_flipped_l.mtz",
(13469, 1065, 2323)),
("less_free_l.mtz", (14002, 532, 2323))]:
s = StringIO()
mtz_obj = iotbx_mtz.object(file_name=file_name)
mtz_obj.show_column_data(out=s, format="spreadsheet")
s = s.getvalue()
spreadsheets.append(s)
n_0, n_1, n_rest = 0, 0, 0
for line in s.splitlines()[1:]:
if (line.endswith(",0")): n_0 += 1
elif (line.endswith(",1")): n_1 += 1
else:
assert line.endswith(",")
n_rest += 1
assert mtz_obj.n_reflections() == n_0 + n_1 + n_rest
assert (n_0, n_1, n_rest) == expected
assert not show_diff(spreadsheets[0], spreadsheets[2])
assert spreadsheets[0] != spreadsheets[1]
def test_debug_print_option_totals_color(self):
prob = om.Problem()
prob.model = FanInGrouped()
prob.model.linear_solver = om.LinearBlockGS()
prob.model.sub.linear_solver = om.LinearBlockGS()
prob.model.add_design_var('iv.x1', parallel_deriv_color='par_dv')
prob.model.add_design_var('iv.x2', parallel_deriv_color='par_dv')
prob.model.add_design_var('iv.x3')
prob.model.add_objective('c3.y')
prob.driver.options['debug_print'] = ['totals']
prob.setup(check=False, mode='fwd')
prob.set_solver_print(level=0)
prob.run_driver()
indep_list = ['iv.x1', 'iv.x2', 'iv.x3']
unknown_list = ['c3.y']
stdout = sys.stdout
strout = StringIO()
sys.stdout = strout
try:
_ = prob.compute_totals(unknown_list,
indep_list,
return_format='flat_dict',
debug_print=not prob.comm.rank)
finally:
sys.stdout = stdout
output = strout.getvalue()
if not prob.comm.rank:
self.assertTrue('Solving color: par_dv (iv.x1, iv.x2)' in output)
self.assertTrue('In mode: fwd, Solving variable(s):' in output)
self.assertTrue("('iv.x3', [2])" in output)
def tst_06():
print("Comparing mixed model with target with 2 chains...using ncs")
import iotbx.pdb
from cctbx.array_family import flex
model_pdb_inp = iotbx.pdb.input(source_info='model',
lines=flex.split_lines(model1))
crystal_symmetry = model_pdb_inp.crystal_symmetry()
f = open('ncs.ncs_spec', 'w')
print(ncs_spec, file=f)
f.close()
f = open('model.pdb', 'w')
print(model1, file=f) #model_hierarchy.as_pdb_string()
f.close()
f = open('query.pdb', 'w')
print(query, file=f) #query_hierarchy.as_pdb_string()
f.close()
f = StringIO()
args = ['model.pdb', 'query.pdb', 'ncs_file=ncs.ncs_spec']
r = run(args, out=f)
expected_text = """
SEQ SCORE is fraction (close and matching target sequence).
MEAN LENGTH is the mean length of contiguous segments in the match with target sequence. (Each gap/reverse of direction starts new segment).
----ALL RESIDUES--- CLOSE RESIDUES ONLY %
MODEL --CLOSE- --FAR-- FORWARD REVERSE MIXED FOUND CA SEQ
RMSD N N N N N SCORE SEQ MATCH(%) SCORE MEAN LENGTH
Unique_target 1.67 58 64 15 29 14 42.6 0.25 8.6 0.04 4.5"""
found_text = "\n".join(f.getvalue().splitlines()[-10:])
if remove_blank(found_text) != remove_blank(expected_text):
print("\n\nExpected: \n%s \n\nFound: \n%s" %
(expected_text, found_text))
raise AssertionError("FAILED")
print("OK")
def test_linesearch_vector_bound_enforcement(self):
top = self.top
ls = top.model.nonlinear_solver.linesearch = BoundsEnforceLS(
bound_enforcement='vector')
ls.options['print_bound_enforce'] = True
# Setup again because we assigned a new linesearch
top.setup(check=False)
# Test lower bounds: should go to the lower bound and stall
top['px.x'] = 2.0
top['comp.y'] = 0.
top['comp.z'] = 1.6
top.run_model()
for ind in range(3):
assert_rel_error(self, top['comp.z'][ind], [1.5], 1e-8)
# Test upper bounds: should go to the minimum upper bound and stall
top['px.x'] = 0.5
top['comp.y'] = 0.
top['comp.z'] = 2.4
stdout = sys.stdout
strout = StringIO()
sys.stdout = strout
try:
top.run_model()
finally:
sys.stdout = stdout
txt = strout.getvalue()
self.assertTrue("'comp.z' exceeds upper bound" in txt)
for ind in range(3):
assert_rel_error(self, top['comp.z'][ind], [2.5], 1e-8)
def exercise_group_args():
from libtbx import group_args
out = StringIO()
a = group_args(
a=1,
b=2,
c=3)
assert a.a==1
assert a.b==2
assert a.c==3
b = group_args(
d = 'd',
e = 'e')
assert b.d=='d'
assert b.e=='e'
print(a, file=out)
v = out.getvalue()
assert not show_diff(v, """group_args
a : 1
b : 2
c : 3\n""")
a.merge(b)
assert a.a==1
assert a.b==2
assert a.c==3
assert a.d=='d'
assert a.e=='e'
assert b.d=='d'
assert b.e=='e'
c = group_args(
a = 11,
b = 12)
a.merge(c)
assert a.a==11
assert a.b==12
assert a.c==3
assert c.a==11
assert c.b==12
def test_write_skip_aggregations(self):
required_config = Namespace()
required_config.add_option(
'minimal_version_for_understanding_refusal',
doc='ignore the Thottleable protocol',
default={'Firefox': '3.5.4'},
)
required_config.add_aggregation(
'an_agg',
lambda x, y, z: 'I refuse'
)
cm = ConfigurationManager(
required_config,
values_source_list=[],
)
cm.get_config()
s = StringIO()
@contextlib.contextmanager
def s_opener():
yield s
cm.write_conf('py', s_opener)
generated_python_module_text = s.getvalue()
expected = """# generated Python configman file
# ignore the Thottleable protocol
minimal_version_for_understanding_refusal = {
"Firefox": "3.5.4"
}
"""
self.assertEqual(generated_python_module_text, expected)
def exercise():
flex.set_random_seed(12345)
random.seed(12345)
xrs = random_structure.xray_structure(sgtbx.space_group_info("P21212"),
elements=["const"] * 100)
f_calc = xrs.structure_factors(d_min=2.5).f_calc()
i_calc = abs(f_calc).f_as_f_sq().set_observation_type_xray_intensity()
i_calc = i_calc.customized_copy(
space_group_info=sgtbx.space_group_info("P222"),
sigmas=flex.double(i_calc.size(), 1.0))
complete_set = i_calc.complete_set()
lone_set = complete_set.lone_set(other=i_calc)
i_abs = lone_set.array(data=flex.double(lone_set.size(), 5),
sigmas=flex.double(lone_set.size(), 10))
i_calc = i_calc.concatenate(
other=i_abs).set_observation_type_xray_intensity()
i_calc.export_as_scalepack_unmerged(file_name="tst_sys_absent.sca")
args = ["tst_sys_absent.sca"]
out = StringIO()
try:
show_systematic_absences.run(args=args, out=out)
except Sorry:
pass
else:
raise Exception_expected
args.append(",".join([str(x) for x in xrs.unit_cell().parameters()]))
show_systematic_absences.run(args=args, out=out)
assert (out.getvalue().count(" ( 0, 3, 0): i/sigi = 0.5") == 4)
i_calc = i_calc.customized_copy(sigmas=None)
i_calc.as_mtz_dataset(
column_root_label="I").mtz_object().write("tst_sys_absent.mtz")
args = ["tst_sys_absent.mtz"]
try:
show_systematic_absences.run(args=args, out=out)
except Sorry:
pass
else:
raise Exception_expected
def exercise():
pdb_inp = iotbx.pdb.input(source_info=None, lines=pdb_str)
ann = pdb_inp.extract_secondary_structure()
pdb_int_params = mmtbx.model.manager.get_default_pdb_interpretation_params(
)
pdb_int_params.pdb_interpretation.ncs_search.enabled = True
model = mmtbx.model.manager(model_input=pdb_inp,
pdb_interpretation_params=pdb_int_params,
build_grm=True)
model.setup_ncs_constraints_groups()
log = StringIO()
rm = ssb.substitute_ss(model, log=log)
# with open("res.pdb", 'w') as f:
# f.write(model.model_as_pdb())
# check that strict NCS was implemented throughout
dif = get_difference_chainsAB(strings=model.model_as_pdb())
for d in dif:
assert approx_equal(d, (0, 0, 0), eps=0.005)
log_cont = log.getvalue()
assert log_cont.find("skipping, not in NCS master") > 0
# print log_cont
print("OK")
def generate_roles_data_from_directory(directory, roles, validate=True):
"""Generate a roles data file using roles from a local path
:param directory local filesystem path to the roles
:param roles ordered list of roles
:param validate validate the metadata format in the role yaml files
:returns string contents of the roles_data.yaml
"""
available_roles = get_roles_list_from_directory(directory)
check_role_exists(available_roles, roles)
output = StringIO()
header = ["#" * 79, "# File generated by TripleO", "#" * 79, ""]
output.write("\n".join(header))
for role in roles:
file_path = os.path.join(directory, "{}.yaml".format(role))
if validate:
validate_role_yaml(role_path=file_path)
with open(file_path, "r") as f:
shutil.copyfileobj(f, output)
return output.getvalue()
def make_merging_statistics_summary(dataset_statistics):
"""Format merging statistics information into an output string."""
# Here use a StringIO to get around excessive padding/whitespace.
# Avoid using result.show as don't want redundancies printed.
out = StringIO()
# First make summary
make_sub_header("Merging statistics", out=out)
dataset_statistics.overall.show_summary(out=out)
# Next make statistics by resolution bin.
msg = "\n\nStatistics by resolution bin:\n"
msg += (" d_max d_min #obs #uniq mult. %comp <I> <I/sI>" +
" r_mrg r_meas r_pim cc1/2 cc_ano\n")
for bin_stats in dataset_statistics.bins:
msg += bin_stats.format() + "\n"
msg += dataset_statistics.overall.format() + "\n\n"
out.write(msg)
# Finally show estimated cutoffs
dataset_statistics.show_estimated_cutoffs(out=out)
return out.getvalue()
def export_mtz(params, experiments, reflections):
"""
Export reflections in MTZ format
:param params: The phil parameters
:param experiments: The experiment list
:param reflections: The reflection tables
"""
_check_input(experiments, reflections)
from dials.util.export_mtz import export_mtz
try:
m = export_mtz(reflections[0], experiments, params)
except ValueError as e:
sys.exit(e)
from six.moves import cStringIO as StringIO
summary = StringIO()
m.show_summary(out=summary)
logger.info("")
logger.info(summary.getvalue())
def test_auto_cleanup(self):
p = Poll.objects.create(question="Will this be deleted?",
pub_date=datetime.now())
self.assertEqual(Poll.history.all().count(), 1)
p.save()
self.assertEqual(Poll.history.all().count(), 2)
p.question = "Maybe this one won't...?"
p.save()
self.assertEqual(Poll.history.all().count(), 3)
out = StringIO()
h = p.history.first()
h.history_date -= timedelta(days=40)
h.save()
management.call_command(self.command_name,
auto=True,
stdout=out,
stderr=StringIO())
self.assertEqual(
out.getvalue(),
"Removed 1 historical records for "
"<class 'simple_history.tests.models.Poll'>\n",
)
self.assertEqual(Poll.history.all().count(), 2)
def exercise():
ext_module = libtbx.env.find_in_repositories(
relative_path="libtbx/xmlrpc_server_example.py", test=os.path.isfile)
full_cmd = "libtbx.python %s" % ext_module
log = StringIO()
server = xmlrpc_utils.external_program_server(command=full_cmd,
program_id="TEST",
timeout=None,
cache_requests=True,
log=log)
server.echo_test()
time.sleep(1)
run_server_tests(server)
port = server.get_port()
assert log.getvalue() == """XML-RPC server started on port %d
hello, world!
hello, world!
quitting
""" % port
server.restart()
time.sleep(1)
run_server_tests(server)
print("OK")
def __call__(self, environ, start_response):
try:
return self.app(environ, start_response)
except Exception as exc:
# get a formatted exception
out = StringIO()
print_exc(file=out)
# get formatted WSGI environment
formatted_environ = pformat(environ)
# render our template
result = debug_template.render(traceback=out.getvalue(),
environment=formatted_environ)
# construct and return our response
response = Response()
if isinstance(exc, HTTPException):
response.status_int = exc.status
else:
response.status_int = 500
response.unicode_body = result
return response(environ, start_response)
def test_auto_cleanup_custom_history_field(self):
m = CustomManagerNameModel.objects.create(name="John")
self.assertEqual(CustomManagerNameModel.log.all().count(), 1)
m.save()
self.assertEqual(CustomManagerNameModel.log.all().count(), 2)
m.name = "Ivan"
m.save()
h = m.log.first()
h.history_date -= timedelta(days=40)
h.save()
self.assertEqual(CustomManagerNameModel.log.all().count(), 3)
out = StringIO()
management.call_command(self.command_name,
auto=True,
stdout=out,
stderr=StringIO())
self.assertEqual(
out.getvalue(),
"Removed 1 historical records for "
"<class 'simple_history.tests.models.CustomManagerNameModel'>\n",
)
self.assertEqual(CustomManagerNameModel.log.all().count(), 2)
def get_next_strand(self,
sheet_id,
strand_id,
pdb_labels,
next_ladder=None):
from iotbx.pdb import secondary_structure
start_i_res = end_i_res = None
next_ladder_i_start = sys.maxsize
next_ladder_i_end = -sys.maxsize
if (next_ladder is not None):
next_ladder_i_start = min(next_ladder.bridges[0].i_res,
next_ladder.bridges[-1].i_res)
next_ladder_i_end = max(next_ladder.bridges[-1].i_res,
next_ladder.bridges[0].i_res)
j_res_start = min(self.bridges[0].j_res, self.bridges[-1].j_res)
j_res_end = max(self.bridges[-1].j_res, self.bridges[0].j_res)
start_i_res = min(j_res_start, next_ladder_i_start)
end_i_res = max(j_res_end, next_ladder_i_end)
if (start_i_res > end_i_res):
out = StringIO()
self.show(out=out)
raise RuntimeError("start_i_res(%d) > end_i_res(%d):\n%s" %
(start_i_res, end_i_res, out.getvalue()))
start = pdb_labels[start_i_res]
end = pdb_labels[end_i_res]
strand = secondary_structure.pdb_strand(sheet_id=sheet_id,
strand_id=strand_id,
start_resname=start.resname,
start_chain_id=start.chain_id,
start_resseq=start.resseq,
start_icode=start.icode,
end_resname=end.resname,
end_chain_id=end.chain_id,
end_resseq=end.resseq,
end_icode=end.icode,
sense=self.direction)
return strand
def run_model(prob):
"""
Call `run_model` on problem and capture output.
Parameters
----------
prob : Problem
an instance of Problem
Returns
-------
string
output from calling `run_model` on the Problem, captured from stdout
"""
stdout = sys.stdout
strout = StringIO()
sys.stdout = strout
try:
prob.run_model()
finally:
sys.stdout = stdout
return strout.getvalue()
def exercise():
class bad_class(object):
def __init__(self):
self.a = self.b
def b():
return 1 + 2
t = bad_class()
sio = StringIO()
print_cycles([t], outstream=sio)
lines = sio.getvalue().splitlines()
# 'Examining: <__main__.bad_class object at 0xb7eb5d2c>'
# " <class '__main__.bad_class'> -- <__main__.bad_class object at 0xb7eb5d2c> ->"
# " <type 'dict'> -- ->"
# ''
assert len(lines) == 4
assert lines[0].startswith("Examining: <")
assert lines[0].count("bad_class") == 1
assert lines[1].count("bad_class") == 2
assert lines[2].count("dict") == 1
print("OK")
def get_pdb_inp(crystal_symmetry=None,
pdb_inp=None,
pdb_string=None,
pdb_in=None):
if pdb_inp is None:
if not pdb_string:
if pdb_in:
pdb_string = open(pdb_in).read()
else:
raise Sorry("Need an input PDB file")
pdb_inp = iotbx.pdb.input(source_info=None, lines=pdb_string)
cryst1_line = iotbx.pdb.format_cryst1_record(
crystal_symmetry=crystal_symmetry)
if not pdb_inp.crystal_symmetry(): # get it
from six.moves import cStringIO as StringIO
f = StringIO()
print(cryst1_line, file=f)
print(pdb_string, file=f)
pdb_string = f.getvalue()
pdb_inp = iotbx.pdb.input(source_info=None, lines=pdb_string)
if pdb_string is None:
pdb_string = pdb_inp.as_pdb_string()
return pdb_inp, cryst1_line, pdb_string
