def test_file_download_authorization_public(self):
# type: () -> None
subscribed_users = [self.example_email("hamlet"), self.example_email("iago")]
unsubscribed_users = [self.example_email("othello"), self.example_email("prospero")]
realm = get_realm("zulip")
for email in subscribed_users:
self.subscribe(get_user(email, realm), "test-subscribe")
self.login(self.example_email("hamlet"))
fp = StringIO("zulip!")
fp.name = "zulip.txt"
result = self.client_post("/json/user_uploads", {'file': fp})
uri = result.json()['uri']
fp_path_id = re.sub('/user_uploads/', '', uri)
body = "First message ...[zulip.txt](http://localhost:9991/user_uploads/" + fp_path_id + ")"
self.send_stream_message(self.example_email("hamlet"), "test-subscribe", body, "test")
self.logout()
# Now all users should be able to access the files
for user in subscribed_users + unsubscribed_users:
self.login(user)
response = self.client_get(uri)
data = b"".join(response.streaming_content)
self.assertEqual(b"zulip!", data)
self.logout()
def test_question_choices():
# TODO: come up with a reusable fixture for testing here
choices = {
'a': '[a], b, cc',
'b': 'a, [b], cc',
'cc': 'a, b, [cc]'
}
for default_value in ['a', 'b']:
choices_str = choices[default_value]
for entered_value, expected_value in [(default_value, default_value),
('', default_value),
('cc', 'cc')]:
with patch_getpass(return_value=entered_value), \
patch_getpass(return_value=entered_value):
out = StringIO()
response = DialogUI(out=out).question("prompt", choices=sorted(choices), default=default_value)
eq_(response, expected_value)
# getpass doesn't use out -- goes straight to the terminal
eq_(out.getvalue(), '')
# TODO: may be test that the prompt was passed as a part of the getpass arg
#eq_(out.getvalue(), 'prompt (choices: %s): ' % choices_str)
# check some expected exceptions to be thrown
out = StringIO()
ui = DialogUI(out=out)
assert_raises(ValueError, ui.question, "prompt", choices=['a'], default='b')
eq_(out.getvalue(), '')
with patch_getpass(return_value='incorrect'):
assert_raises(RuntimeError, ui.question, "prompt", choices=['a', 'b'])
assert_re_in(".*ERROR: .incorrect. is not among choices.*", out.getvalue())
def test_list(self):
def fake_network_get_all(context):
return [db_fakes.FakeModel(self.net)]
self.stubs.Set(db, 'network_get_all', fake_network_get_all)
output = StringIO()
sys.stdout = output
self.commands.list()
sys.stdout = sys.__stdout__
result = output.getvalue()
_fmt = "\t".join(["%(id)-5s", "%(cidr)-18s", "%(cidr_v6)-15s",
"%(dhcp_start)-15s", "%(dns1)-15s", "%(dns2)-15s",
"%(vlan)-15s", "%(project_id)-15s", "%(uuid)-15s"])
head = _fmt % {'id': 'id',
'cidr': 'IPv4',
'cidr_v6': 'IPv6',
'dhcp_start': 'start address',
'dns1': 'DNS1',
'dns2': 'DNS2',
'vlan': 'VlanID',
'project_id': 'project',
'uuid': "uuid"}
body = _fmt % {'id': self.net['id'],
'cidr': self.net['cidr'],
'cidr_v6': self.net['cidr_v6'],
'dhcp_start': self.net['dhcp_start'],
'dns1': self.net['dns1'],
'dns2': self.net['dns2'],
'vlan': self.net['vlan'],
'project_id': self.net['project_id'],
'uuid': self.net['uuid']}
answer = '%s\n%s\n' % (head, body)
self.assertEqual(result, answer)
def test_rest_endpoint(self):
# type: () -> None
"""
Tests the /api/v1/user_uploads api endpoint. Here a single file is uploaded
and downloaded using a username and api_key
"""
fp = StringIO("zulip!")
fp.name = "zulip.txt"
# Upload file via API
auth_headers = self.api_auth(self.example_email("hamlet"))
result = self.client_post('/api/v1/user_uploads', {'file': fp}, **auth_headers)
self.assertIn("uri", result.json())
uri = result.json()['uri']
base = '/user_uploads/'
self.assertEqual(base, uri[:len(base)])
# Download file via API
self.logout()
response = self.client_get(uri, **auth_headers)
data = b"".join(response.streaming_content)
self.assertEqual(b"zulip!", data)
# Files uploaded through the API should be accesible via the web client
self.login(self.example_email("hamlet"))
self.assert_url_serves_contents_of_file(uri, b"zulip!")
def test_pydotprint_cond_highlight():
"""
This is a REALLY PARTIAL TEST.
I did them to help debug stuff.
"""
# Skip test if pydot is not available.
if not theano.printing.pydot_imported:
raise SkipTest('pydot not available')
x = tensor.dvector()
f = theano.function([x], x * 2)
f([1, 2, 3, 4])
s = StringIO()
new_handler = logging.StreamHandler(s)
new_handler.setLevel(logging.DEBUG)
orig_handler = theano.logging_default_handler
theano.theano_logger.removeHandler(orig_handler)
theano.theano_logger.addHandler(new_handler)
try:
theano.printing.pydotprint(f, cond_highlight=True,
print_output_file=False)
finally:
theano.theano_logger.addHandler(orig_handler)
theano.theano_logger.removeHandler(new_handler)
assert (s.getvalue() == 'pydotprint: cond_highlight is set but there'
' is no IfElse node in the graph\n')
def user_report():
strIO = None
# Create or update view
cursor = connection.cursor()
sql = 'CREATE OR REPLACE VIEW accounts_emailuser_report_v AS \
select md5(CAST((first_name,last_name,dob)AS text)) as hash,count(*) as occurence, first_name,last_name,\
dob from accounts_emailuser group by first_name,last_name,dob;'
cursor.execute(sql)
users = EmailUserReport.objects.filter(occurence__gt=1)
if users:
strIO = StringIO()
fieldnames = ['Occurence', 'First Name','Last Name','DOB']
writer = csv.DictWriter(strIO, fieldnames=fieldnames)
writer.writeheader()
for u in users:
info = {
'Occurence': u.occurence,
'First Name': u.first_name,
'Last Name': u.last_name,
'DOB': u.dob
}
writer.writerow(info)
strIO.flush()
strIO.seek(0)
return strIO
def do_check_on(value, nd, var=None):
"""
Checks `value` for NaNs / Infs. If detected, raises an exception
and / or prints information about `nd`, `f`, and `is_input` to
help the user determine the cause of the invalid values.
Parameters
----------
value : numpy.ndarray
The value to be checked.
nd : theano.gof.Apply
The Apply node being executed.
var : theano.gof.Variable
Not used if nd is there. Otherwise, used to print the stack
trace for inputs of the graph.
"""
error = False
sio = StringIO()
if nan_is_error:
if contains_nan(value, nd, var):
print('NaN detected', file=sio)
error = True
if inf_is_error:
if contains_inf(value, nd, var):
print('Inf detected', file=sio)
error = True
if big_is_error:
err = False
if not _is_numeric_value(value, var):
err = False
elif pygpu_available and isinstance(value, GpuArray):
err = (f_gpua_absmax(value.reshape(value.size)) > 1e10)
else:
err = (np.abs(value).max() > 1e10)
if err:
print('Big value detected', file=sio)
error = True
if error:
if nd:
print("NanGuardMode found an error in the "
"output of a node in this variable:", file=sio)
print(theano.printing.debugprint(nd, file='str'), file=sio)
else:
print("NanGuardMode found an error in an input of the "
"graph.", file=sio)
# Add the stack trace
if nd:
var = nd.outputs[0]
print(theano.gof.utils.get_variable_trace_string(var),
file=sio)
msg = sio.getvalue()
if config.NanGuardMode.action == 'raise':
raise AssertionError(msg)
elif config.NanGuardMode.action == 'pdb':
print(msg)
import pdb
pdb.set_trace()
elif config.NanGuardMode.action == 'warn':
logger.error(msg)
def test_file_upload_authed(self):
# type: () -> None
"""
A call to /json/upload_file should return a uri and actually create an object.
"""
conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
conn.create_bucket(settings.S3_AUTH_UPLOADS_BUCKET)
self.login("*****@*****.**")
fp = StringIO("zulip!")
fp.name = "zulip.txt"
result = self.client_post("/json/upload_file", {'file': fp})
self.assert_json_success(result)
json = ujson.loads(result.content)
self.assertIn("uri", json)
uri = json["uri"]
base = '/user_uploads/'
self.assertEquals(base, uri[:len(base)])
response = self.client_get(uri)
redirect_url = response['Location']
self.assertEquals(b"zulip!", urllib.request.urlopen(redirect_url).read().strip())
self.subscribe_to_stream("*****@*****.**", "Denmark")
body = "First message ...[zulip.txt](http://localhost:9991" + uri + ")"
self.send_message("*****@*****.**", "Denmark", Recipient.STREAM, body, "test")
self.assertIn('title="zulip.txt"', self.get_last_message().rendered_content)
def main(loops, level):
board, solution = LEVELS[level]
order = DESCENDING
strategy = Done.FIRST_STRATEGY
stream = StringIO()
board = board.strip()
expected = solution.rstrip()
range_it = xrange(loops)
t0 = perf.perf_counter()
for _ in range_it:
stream = StringIO()
solve_file(board, strategy, order, stream)
output = stream.getvalue()
stream = None
dt = perf.perf_counter() - t0
output = '\n'.join(line.rstrip() for line in output.splitlines())
if output != expected:
raise AssertionError("got a wrong answer:\n%s\nexpected: %s"
% (output, expected))
return dt
def test_file_upload_authed(self):
# type: () -> None
"""
A call to /json/upload_file should return a uri and actually create an
entry in the database. This entry will be marked unclaimed till a message
refers it.
"""
self.login("*****@*****.**")
fp = StringIO("zulip!")
fp.name = "zulip.txt"
result = self.client_post("/json/upload_file", {'file': fp})
self.assert_json_success(result)
json = ujson.loads(result.content)
self.assertIn("uri", json)
uri = json["uri"]
base = '/user_uploads/'
self.assertEquals(base, uri[:len(base)])
# In the future, local file requests will follow the same style as S3
# requests; they will be first authenthicated and redirected
response = self.client_get(uri)
data = b"".join(response.streaming_content)
self.assertEquals(b"zulip!", data)
# check if DB has attachment marked as unclaimed
entry = Attachment.objects.get(file_name='zulip.txt')
self.assertEquals(entry.is_claimed(), False)
self.subscribe_to_stream("*****@*****.**", "Denmark")
body = "First message ...[zulip.txt](http://localhost:9991" + uri + ")"
self.send_message("*****@*****.**", "Denmark", Recipient.STREAM, body, "test")
self.assertIn('title="zulip.txt"', self.get_last_message().rendered_content)
def test_rest_endpoint(self):
# type: () -> None
"""
Tests the /api/v1/user_uploads api endpoint. Here a single file is uploaded
and downloaded using a username and api_key
"""
fp = StringIO("zulip!")
fp.name = "zulip.txt"
# Upload file via API
auth_headers = self.api_auth('*****@*****.**')
result = self.client_post('/api/v1/user_uploads', {'file': fp}, **auth_headers)
json = ujson.loads(result.content)
self.assertIn("uri", json)
uri = json["uri"]
base = '/user_uploads/'
self.assertEquals(base, uri[:len(base)])
# Download file via API
self.client_post('/accounts/logout/')
response = self.client_get(uri, **auth_headers)
data = b"".join(response.streaming_content)
self.assertEquals(b"zulip!", data)
# Files uploaded through the API should be accesible via the web client
self.login("*****@*****.**")
response = self.client_get(uri)
data = b"".join(response.streaming_content)
self.assertEquals(b"zulip!", data)
def test_record_good():
"""
Tests that when we record a sequence of events, then
repeat it exactly, the Record class:
1) Records it correctly
2) Does not raise any errors
"""
# Record a sequence of events
output = StringIO()
recorder = Record(file_object=output, replay=False)
num_lines = 10
for i in xrange(num_lines):
recorder.handle_line(str(i) + '\n')
# Make sure they were recorded correctly
output_value = output.getvalue()
assert output_value == ''.join(str(i) + '\n' for i in xrange(num_lines))
# Make sure that the playback functionality doesn't raise any errors
# when we repeat them
output = StringIO(output_value)
playback_checker = Record(file_object=output, replay=True)
for i in xrange(num_lines):
playback_checker.handle_line(str(i) + '\n')
def test_stderr_to_StringIO():
s = StringIO()
with stderr_to(s):
sys.stderr.write(u("hello"))
assert s.getvalue() == 'hello'
def test_grid_01(self):
nid = 1
cp = 2
cd = 0
ps = ''
seid = 0
card_count = {'GRID': 1,}
model = BDF()
model.allocate(card_count)
data1 = BDFCard(['GRID', nid, cp, 0., 0., 0., cd, ps, seid])
nodes = model.grid
nodes.add(data1)
#print n1
f = StringIO()
nodes.write_bdf(f, size=8, write_header=False)
nodes.write_bdf(f, size=16, write_header=False)
nodes.write_bdf(f, size=16, is_double=True, write_header=False)
# small field
f = StringIO()
nodes.write_bdf(f, size=8, write_header=False)
msg = f.getvalue()
card = 'GRID 1 2 0. 0. 0.\n'
self.assertCardEqual(msg, card)
# large field
f = StringIO()
nodes.write_bdf(f, size=16, write_header=False)
card = ('GRID* 1 2 0. 0.\n'
'* 0.\n')
msg = f.getvalue()
self.assertCardEqual(msg, card)
def test_record_bad():
"""
Tests that when we record a sequence of events, then
do something different on playback, the Record class catches it.
"""
# Record a sequence of events
output = StringIO()
recorder = Record(file_object=output, replay=False)
num_lines = 10
for i in xrange(num_lines):
recorder.handle_line(str(i) + '\n')
# Make sure that the playback functionality doesn't raise any errors
# when we repeat some of them
output_value = output.getvalue()
output = StringIO(output_value)
playback_checker = Record(file_object=output, replay=True)
for i in xrange(num_lines // 2):
playback_checker.handle_line(str(i) + '\n')
# Make sure it raises an error when we deviate from the recorded sequence
try:
playback_checker.handle_line('0\n')
except MismatchError:
return
raise AssertionError("Failed to detect mismatch between recorded sequence "
" and repetition of it.")
def init_3d():
"""Initialise 3D plots within the IPython notebook, by injecting the
required javascript libraries.
"""
library_javascript = StringIO()
library_javascript.write("""
<script type="text/javascript">
/* Beginning of javascript injected by OpenModes */
var openmodes_javascript_injected = true;
""")
three_js_libraries = ("three.min.js", "OrbitControls.js",
"Lut.js", "Detector.js", "CanvasRenderer.js",
"Projector.js")
# Include required parts of three.js inline
for library in three_js_libraries:
with open(osp.join(three_js_dir, library)) as infile:
library_javascript.write(infile.read())
# include my custom javascript inline
with open(osp.join(static_dir, "three_js_plot.js")) as infile:
library_javascript.write(infile.read())
library_javascript.write(
"/* End of javascript injected by OpenModes */\n</script>\n")
display(HTML(library_javascript.getvalue()))
logging.info("Javascript injected for 3D interactive WebGL plots")
def create_hook_module(target):
safe_name = SAFE_MODULE_NAME.sub("_", target, len(target))
module_name = "bento_hook_%s" % safe_name
main_file = os.path.abspath(target)
module = imp.new_module(module_name)
module.__file__ = main_file
code = open(main_file).read()
sys.path.insert(0, os.path.dirname(main_file))
try:
exec(compile(code, main_file, 'exec'), module.__dict__)
sys.modules[module_name] = module
except Exception:
sys.path.pop(0)
e = extract_exception()
tb = sys.exc_info()[2]
s = StringIO()
traceback.print_tb(tb, file=s)
msg = """\
Could not import hook file %r: caught exception %r
Original traceback (most recent call last)
%s\
""" % (main_file, e, s.getvalue())
raise InvalidHook(msg)
module.root_path = main_file
return module
def render_and_save_variation(self, name, content, variation):
"""
Renders the image variations and saves them to the storage
"""
content.seek(0)
img = Image.open(content)
if self.is_smaller(img, variation):
factor = 1
while (img.size[0] / factor > 2 * variation['width'] and
img.size[1] * 2 / factor > 2 * variation['height']):
factor *= 2
if factor > 1:
img.thumbnail((int(img.size[0] / factor),
int(img.size[1] / factor)), resample=resample)
if variation['crop']:
img = ImageOps.fit(img, (variation['width'], variation['height']), method=resample)
else:
img.thumbnail((variation['width'], variation['height']), resample=resample)
variation_name = self.get_variation_name(self.instance, self.field, variation)
file_buffer = StringIO()
format = self.get_file_extension(name).lower().replace('jpg', 'jpeg')
img.save(file_buffer, format)
self.storage.save(variation_name, ContentFile(file_buffer.getvalue()))
file_buffer.close()
def setUp(self):
stream = StringIO("""
- hosts: localhost
vars:
number: 1
string: Ansible
utf8_string: Cafè Eñyei
dictionary:
webster: daniel
oed: oxford
list:
- a
- b
- 1
- 2
tasks:
- name: Test case
ping:
data: "{{ utf8_string }}"
- name: Test 2
ping:
data: "Cafè Eñyei"
- name: Test 3
command: "printf 'Cafè Eñyei\\n'"
""")
self.play_filename = '/path/to/myplay.yml'
stream.name = self.play_filename
self.loader = AnsibleLoader(stream)
self.data = self.loader.get_single_data()
def outstanding_bookings():
try:
outstanding = []
today = datetime.date.today()
for b in Booking.objects.filter(is_canceled=False,departure__gte=today).exclude(booking_type__in=['1','3']):
if not b.paid:
outstanding.append(b)
strIO = StringIO()
fieldnames = ['Confirmation Number','Customer','Campground','Arrival','Departure','Outstanding Amount']
writer = csv.writer(strIO)
writer.writerow(fieldnames)
for o in outstanding:
fullname = '{} {}'.format(o.details.get('first_name'),o.details.get('last_name'))
writer.writerow([o.confirmation_number,fullname,o.campground.name,o.arrival.strftime('%d/%m/%Y'),o.departure.strftime('%d/%m/%Y'),o.outstanding])
strIO.flush()
strIO.seek(0)
_file = strIO
dt = datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S')
recipients = []
recipients = OutstandingBookingRecipient.objects.all()
email = EmailMessage(
'Unpaid Bookings Summary as at {}'.format(dt),
'Unpaid Bookings as at {}'.format(dt),
settings.DEFAULT_FROM_EMAIL,
to=[r.email for r in recipients]if recipients else [settings.NOTIFICATION_EMAIL]
)
email.attach('OustandingBookings_{}.csv'.format(dt), _file.getvalue(), 'text/csv')
email.send()
except:
raise
def init_3d():
"""Initialise 3D plots within the IPython notebook, by injecting the
required javascript libraries.
"""
library_javascript = StringIO()
library_javascript.write("""
<!-- Beginning of javascript injected by multinet.js -->
<script type="text/javascript" src="multinet/static/js/jquery-2.1.4.js"></script>
<script type="text/javascript" src="multinet/static/js/jquery-ui-1.11.4.js"></script>
<script type="text/javascript" src="multinet/static/js/threejs/three-r71.js"></script>
<script type="text/javascript" src="multinet/static/js/threejs/orbitcontrols.js"></script>
<script type="text/javascript" src="multinet/static/js/threejs/stats-r12.min.js"></script>
<script type="text/javascript" src="multinet/static/js/threejs/detector.js"></script>
<script type="text/javascript" src="multinet/static/js/multinet-core.js"></script>
<script type="text/javascript" src="multinet/static/js/multinet-ipython.js"></script>
<script type="text/javascript">
var multinet_javascript_injected = true;
</script>
<!-- End of javascript injected by multinet.js -->
""")
display(HTML(library_javascript.getvalue()))
def test_contradictory_date_entries_warn(self):
"""4.8.5.3 Emit warning on contradictory date entries."""
stream = StringIO(
wrap_document_text(construct_document_from(**{
"Author": {
"ForeName": "John",
"LastName": "Smith"
},
"DateCompleted": {
"Year": "2011",
"Month": "01",
"Day": "01"
},
"DateRevised": {
"Year": "2010",
"Month": "01",
"Day": "01"
},
}))
)
stderr = StringIO()
self.patch(sys, "stderr", stderr)
result = parsexml.parse_element_tree(
parsexml.file_to_element_tree(stream)
)
stderr.seek(0)
stderr_out = stderr.read()
self.assertThat(result["pubDate"], Is(None))
self.assertThat(result["reviseDate"], Is(None))
self.assertThat(stderr_out,
Contains("is greater than"))
class UpdateAppsAndBackendsTest(TestCase):
def setUp(self):
self.output = StringIO()
# BASE_APPS are needed for the managment commands to load successfully
self.BASE_APPS = [
'rapidsms',
'django.contrib.auth',
'django.contrib.contenttypes',
]
def test_no_apps_then_none_added(self):
with self.settings(INSTALLED_APPS=self.BASE_APPS):
call_command('update_apps', stdout=self.output)
self.assertEqual(self.output.getvalue(), '')
def test_adds_app(self):
# Add an app that has a RapidSMS app
APPS = self.BASE_APPS + ['rapidsms.contrib.handlers']
with self.settings(INSTALLED_APPS=APPS):
call_command('update_apps', stdout=self.output)
self.assertEqual(self.output.getvalue(), 'Added persistent app rapidsms.contrib.handlers\n')
def test_no_backends_then_none_added(self):
with self.settings(INSTALLED_BACKENDS={}):
call_command('update_backends', stdout=self.output)
self.assertEqual(self.output.getvalue(), '')
def test_adds_backend(self):
INSTALLED_BACKENDS = {
"message_tester": {"ENGINE": "rapidsms.backends.database.DatabaseBackend"},
}
with self.settings(INSTALLED_BACKENDS=INSTALLED_BACKENDS):
call_command('update_backends', stdout=self.output)
self.assertEqual(self.output.getvalue(), 'Added persistent backend message_tester\n')
def _find_snippet_imports(module_data, module_path, strip_comments):
"""
Given the source of the module, convert it to a Jinja2 template to insert
module code and return whether it's a new or old style module.
"""
module_style = "old"
if REPLACER in module_data:
module_style = "new"
elif REPLACER_WINDOWS in module_data:
module_style = "new"
elif "from ansible.module_utils." in module_data:
module_style = "new"
elif "WANT_JSON" in module_data:
module_style = "non_native_want_json"
output = StringIO()
lines = module_data.split("\n")
snippet_names = []
for line in lines:
if REPLACER in line:
output.write(_slurp(os.path.join(_SNIPPET_PATH, "basic.py")))
snippet_names.append("basic")
if REPLACER_WINDOWS in line:
ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
output.write(ps_data)
snippet_names.append("powershell")
elif line.startswith("from ansible.module_utils."):
tokens = line.split(".")
import_error = False
if len(tokens) != 3:
import_error = True
if " import *" not in line:
import_error = True
if import_error:
raise AnsibleError(
"error importing module in %s, expecting format like 'from ansible.module_utils.basic import *'"
% module_path
)
snippet_name = tokens[2].split()[0]
snippet_names.append(snippet_name)
output.write(_slurp(os.path.join(_SNIPPET_PATH, snippet_name + ".py")))
else:
if strip_comments and line.startswith("#") or line == "":
pass
output.write(line)
output.write("\n")
if not module_path.endswith(".ps1"):
# Unixy modules
if len(snippet_names) > 0 and not "basic" in snippet_names:
raise AnsibleError("missing required import in %s: from ansible.module_utils.basic import *" % module_path)
else:
# Windows modules
if len(snippet_names) > 0 and not "powershell" in snippet_names:
raise AnsibleError("missing required import in %s: # POWERSHELL_COMMON" % module_path)
return (output.getvalue(), module_style)
def run(self, ctx):
argv = ctx.command_argv
p = ctx.options_context.parser
o, a = p.parse_args(argv)
if o.help:
p.print_help()
return
pkg = ctx.pkg
format = o.format
archive_root = "%s-%s" % (pkg.name, pkg.version)
if not o.output_file:
archive_name = archive_basename(pkg) + _FORMATS[format]["ext"]
else:
output = op.basename(o.output_file)
if output != o.output_file:
raise bento.errors.BentoError("Invalid output file: should not contain any directory")
archive_name = output
s = StringIO()
write_pkg_info(ctx.pkg, s)
n = ctx.build_node.make_node("PKG_INFO")
n.parent.mkdir()
n.write(s.getvalue())
ctx.register_source_node(n, "PKG_INFO")
# XXX: find a better way to pass archive name from other commands (used
# by distcheck ATM)
self.archive_root, self.archive_node = create_archive(archive_name, archive_root, ctx._node_pkg,
ctx.top_node, ctx.run_node, o.format, o.output_dir)
def render(self):
"""This is the tricky part, whith the rendered_content create a CSV"""
if not self._is_rendered:
# File pointer needed to create the CSV in memory
buffer = StringIO()
writer = UnicodeWriter(buffer)
for row in self.rows:
writer.writerow([six.text_type(value) for value
in row])
# Get the value of the StringIO buffer and write it to the response.
csv = buffer.getvalue()
buffer.close()
self.write(csv)
# Sets the appropriate CSV headers.
self['Content-Disposition'] = 'attachment; filename=%s' % (
self.filename, )
# The CSV has been generated
self._is_rendered = True
for post_callback in self._post_render_callbacks:
post_callback(self)
return self
def log_mem_usage(signum, frame, fname=None):
global _count
_count += 1
gc.collect()
if not fname:
fname = filename + '_memory_%02d.log' % _count
with open(fname, 'wb') as f:
f.write('gc.garbage: %d\n\n' % len(gc.garbage))
objgraph.show_most_common_types(limit=50, file=f)
f.write('\n\n')
buf = StringIO()
objgraph.show_growth(limit=50, file=buf)
buf = buf.getvalue()
f.write(buf)
if _count < 2:
return
for tn, l in enumerate(buf.splitlines()[:10]):
l = l.strip()
if not l:
continue
type_ = l.split()[0]
objects = objgraph.by_type(type_)
objects = random.sample(objects, min(50, len(objects)))
objgraph.show_chain(
objgraph.find_backref_chain(
objects[0],
objgraph.is_proper_module),
filename=fname[:-4] + '_type_%02d_backref.png' % tn
)
objgraph.show_backrefs(
objects,
max_depth=5,
extra_info=lambda x: hex(id(x)),
filename=fname[:-4] + '_type_%02d_backrefs.png' % tn,
)
def test_main_help(monkeypatch):
# Patch stdout
fakestdout = StringIO()
monkeypatch.setattr(sys, "stdout", fakestdout)
pytest.raises(SystemExit, main.main, ['--help'])
assert fakestdout.getvalue().lstrip().startswith("Usage: ")
def render(self):
"""This is the tricky part, whith the rendered_content create a PDF"""
# The following is required for PDF generation
import xhtml2pdf.pisa as pisa # The import is changed to xhtml2pdf
if not self._is_rendered:
# File pointer needed to create the PDF in memory
buffer = StringIO()
# Create the PDF object, using the StringIO object as its "file."
pisa.CreatePDF(self.rendered_content, buffer,
link_callback=fetch_resources)
# Get the value of the StringIO buffer and write it to the response.
pdf = buffer.getvalue()
buffer.close()
self.write(pdf)
# Sets the appropriate PDF headers.
self['Content-Disposition'] = 'attachment; filename=%s' % (
self.filename, )
# The PDF has been rendered
self._is_rendered = True
for post_callback in self._post_render_callbacks:
post_callback(self)
return self
def show(self, lswitch=None):
params = [lswitch] if lswitch else []
stdout = StringIO()
self.run("show", args=params, stdout=stdout)
output = stdout.getvalue()
return get_lswitch_info(output)
def __repr__(self):
return self.print_to(StringIO()).getvalue()
def exercise_non_crystallographic_conserving_bonds_and_angles():
sites_cart, geo = geometry_restraints.manager \
.construct_non_crystallographic_conserving_bonds_and_angles(
sites_cart=flex.vec3_double([
(10.949, 12.815, 15.189),
(10.405, 13.954, 15.917),
(10.779, 15.262, 15.227),
( 9.916, 16.090, 14.936)]),
edge_list_bonds=[(0, 1), (1, 2), (2, 3)],
edge_list_angles=[(0, 2), (1, 3)])
assert approx_equal(sites_cart, [(6.033, 5.000, 5.253),
(5.489, 6.139, 5.981),
(5.863, 7.447, 5.291),
(5.000, 8.275, 5.000)])
assert approx_equal(geo.energies_sites(sites_cart=sites_cart).target, 0)
sites_cart_noise = flex.vec3_double([ # Just to make all residuals unique,
(6.043, 5.030, 5.233), # so that the sorted bond list below
(5.469, 6.119, 5.941), # has the same order on all platforms.
(5.893, 7.487, 5.281),
(5.040, 8.225, 5.020)
])
sio = StringIO()
geo.show_sorted(sites_cart=sites_cart_noise, f=sio)
expected_first_part = """\
Bond restraints: 5
Sorted by residual:
bond 2
3
ideal model delta sigma weight residual
1.231 1.158 0.073 1.00e-01 1.00e+02 5.35e-01
bond 1
2
ideal model delta sigma weight residual
1.525 1.577 -0.052 1.00e-01 1.00e+02 2.66e-01
bond 1
3
ideal model delta sigma weight residual
2.401 2.338 0.063 1.41e-01 5.00e+01 1.96e-01
bond 0
1
ideal model delta sigma weight residual
1.457 1.420 0.037 1.00e-01 1.00e+02 1.37e-01
bond 0
2
ideal model delta sigma weight residual
2.453 2.462 -0.009 1.41e-01 5.00e+01 3.92e-03
"""
assert not show_diff(
sio.getvalue(), expected_first_part + """\
Nonbonded interactions: 0
""")
#
sites_cart, geo = geometry_restraints.manager \
.construct_non_crystallographic_conserving_bonds_and_angles(
sites_cart=flex.vec3_double([
(10.949, 12.815, 15.189),
(10.405, 13.954, 15.917),
(10.779, 15.262, 15.227),
( 9.916, 16.090, 14.936),
(10.749, 12.615, 15.389)]),
edge_list_bonds=[(0, 1), (1, 2), (2, 3)],
edge_list_angles=[(0, 2), (1, 3)])
sites_cart_noise.append(sites_cart[-1])
sio = StringIO()
geo.show_sorted(sites_cart=sites_cart_noise, f=sio)
assert not show_diff(
sio.getvalue(), expected_first_part + """\
Nonbonded interactions: 2
Sorted by model distance:
nonbonded 0
4
model vdw
0.306 1.200
nonbonded 1
4
model vdw
1.274 1.200
""")
def exercise_with_zeolite(verbose):
if (not libtbx.env.has_module("iotbx")):
print("Skipping exercise_with_zeolite(): iotbx not available")
return
from iotbx.kriber import strudat
atlas_file = libtbx.env.find_in_repositories(
relative_path="phenix_regression/misc/strudat_zeolite_atlas",
test=os.path.isfile)
if (atlas_file is None):
print("Skipping exercise_with_zeolite(): input file not available")
return
strudat_contents = strudat.read_all_entries(open(atlas_file))
strudat_entry = strudat_contents.get("YUG")
si_structure = strudat_entry.as_xray_structure()
if (verbose):
out = sys.stdout
else:
out = StringIO()
drls = distance_and_repulsion_least_squares(
si_structure=si_structure,
distance_cutoff=3.5,
nonbonded_repulsion_function_type="prolsq",
n_macro_cycles=2,
out=out)
#
nbp = drls.geometry_restraints_manager.pair_proxies().nonbonded_proxies
assert nbp.n_total() > 50
# expected is 60, but the exact number depends on the minimizer
#
site_labels = drls.minimized_structure.scatterers().extract_labels()
sites_cart = drls.start_structure.sites_cart()
pair_proxies = drls.geometry_restraints_manager.pair_proxies()
out = StringIO()
pair_proxies.bond_proxies.show_sorted(by_value="residual",
sites_cart=sites_cart,
site_labels=site_labels,
f=out)
if (verbose):
sys.stdout.write(out.getvalue())
assert len(out.getvalue().splitlines()) == 48 * 4 + 2
assert out.getvalue().splitlines()[-1].find("remaining") < 0
out = StringIO()
pair_proxies.bond_proxies.show_sorted(by_value="residual",
sites_cart=sites_cart,
site_labels=site_labels,
f=out,
prefix="0^",
max_items=28)
if (verbose):
sys.stdout.write(out.getvalue())
assert not show_diff(out.getvalue().replace("e-00", "e-0"),
"""\
0^Bond restraints: 48
0^Sorted by residual:
0^bond O3
0^ O4
0^ ideal model delta sigma weight residual
0^ 2.629 2.120 0.509 1.56e+00 4.10e-01 1.06e-01
...
0^bond SI1
0^ SI1
0^ ideal model delta sigma weight residual sym.op.
0^ 3.071 3.216 -0.145 2.08e+00 2.31e-01 4.83e-03 -x+1/2,-y+1/2,-z+1
0^... (remaining 20 not shown)
""",
selections=[range(6), range(-5, 0)])
out = StringIO()
pair_proxies.bond_proxies.show_sorted(by_value="delta",
sites_cart=sites_cart,
site_labels=site_labels,
f=out,
prefix="0^",
max_items=28)
if (verbose):
sys.stdout.write(out.getvalue())
assert not show_diff(out.getvalue().replace("e-00", "e-0"),
"""\
0^Bond restraints: 48
0^Sorted by delta:
0^bond O3
0^ O4
0^ ideal model delta sigma weight residual
0^ 2.629 2.120 0.509 1.56e+00 4.10e-01 1.06e-01
...
0^... (remaining 20 not shown)
""",
selections=[range(6), [-1]])
site_labels_long = ["abc" + label + "def" for label in site_labels]
out = StringIO()
pair_proxies.bond_proxies.show_sorted(by_value="residual",
sites_cart=sites_cart,
site_labels=site_labels_long,
f=out,
prefix="^0",
max_items=28)
if (verbose):
sys.stdout.write(out.getvalue())
assert not show_diff(out.getvalue().replace("e-00", "e-0"),
"""\
^0Bond restraints: 48
^0Sorted by residual:
^0bond abcO3def
^0 abcO4def
^0 ideal model delta sigma weight residual
^0 2.629 2.120 0.509 1.56e+00 4.10e-01 1.06e-01
...
^0bond abcSI1def
^0 abcSI1def
^0 ideal model delta sigma weight residual sym.op.
^0 3.071 3.216 -0.145 2.08e+00 2.31e-01 4.83e-03 -x+1/2,-y+1/2,-z+1
^0... (remaining 20 not shown)
""",
selections=[range(6), range(-5, 0)])
out = StringIO()
pair_proxies.bond_proxies.show_sorted(by_value="residual",
sites_cart=sites_cart,
f=out,
prefix=".=",
max_items=28)
if (verbose):
sys.stdout.write(out.getvalue())
assert not show_diff(out.getvalue().replace("e-00", "e-0"),
"""\
.=Bond restraints: 48
.=Sorted by residual:
.=bond 4
.= 5
.= ideal model delta sigma weight residual
.= 2.629 2.120 0.509 1.56e+00 4.10e-01 1.06e-01
...
.=bond 0
.= 0
.= ideal model delta sigma weight residual sym.op.
.= 3.071 3.216 -0.145 2.08e+00 2.31e-01 4.83e-03 -x+1/2,-y+1/2,-z+1
.=... (remaining 20 not shown)
""",
selections=[range(6), range(-5, 0)])
out = StringIO()
pair_proxies.bond_proxies.show_sorted(by_value="residual",
sites_cart=sites_cart,
f=out,
prefix="-+",
max_items=1)
if (verbose):
sys.stdout.write(out.getvalue())
assert not show_diff(
out.getvalue().replace("e-00", "e-0"), """\
-+Bond restraints: 48
-+Sorted by residual:
-+bond 4
-+ 5
-+ ideal model delta sigma weight residual
-+ 2.629 2.120 0.509 1.56e+00 4.10e-01 1.06e-01
-+... (remaining 47 not shown)
""")
out = StringIO()
pair_proxies.bond_proxies.show_sorted(by_value="residual",
sites_cart=sites_cart,
f=out,
prefix="=+",
max_items=0)
if (verbose):
sys.stdout.write(out.getvalue())
assert not show_diff(
out.getvalue(), """\
=+Bond restraints: 48
=+Sorted by residual:
=+... (remaining 48 not shown)
""")
#
sites_cart = si_structure.sites_cart()
site_labels = [sc.label for sc in si_structure.scatterers()]
asu_mappings = si_structure.asu_mappings(buffer_thickness=3.5)
for min_cubicle_edge in [0, 5]:
pair_generator = crystal.neighbors_fast_pair_generator(
asu_mappings=asu_mappings,
distance_cutoff=asu_mappings.buffer_thickness(),
minimal=False,
min_cubicle_edge=min_cubicle_edge)
sorted_asu_proxies = geometry_restraints.nonbonded_sorted_asu_proxies(
asu_mappings=asu_mappings)
while (not pair_generator.at_end()):
p = geometry_restraints.nonbonded_asu_proxy(
pair=next(pair_generator), vdw_distance=3)
sorted_asu_proxies.process(p)
out = StringIO()
sorted_asu_proxies.show_sorted(by_value="delta",
sites_cart=sites_cart,
site_labels=site_labels,
f=out,
prefix="d%")
if (verbose):
sys.stdout.write(out.getvalue())
assert not show_diff(
out.getvalue(),
"""\
d%Nonbonded interactions: 7
d%Sorted by model distance:
...
d%nonbonded SI2
d% SI2
d% model vdw sym.op.
d% 3.092 3.000 -x+1,y,-z
...
d%nonbonded SI1
d% SI1
d% model vdw sym.op.
d% 3.216 3.000 -x+1/2,-y+1/2,-z+1
""",
selections=[range(2), range(10, 14),
range(26, 30)])
out = StringIO()
sorted_asu_proxies.show_sorted(by_value="delta",
sites_cart=sites_cart,
f=out,
prefix="*j",
max_items=5)
if (verbose):
sys.stdout.write(out.getvalue())
assert not show_diff(out.getvalue(),
"""\
*jNonbonded interactions: 7
*jSorted by model distance:
...
*jnonbonded 0
*j 1
*j model vdw
*j 3.107 3.000
*jnonbonded 0
*j 0
*j model vdw sym.op.
*j 3.130 3.000 -x+1,y,-z+1
*j... (remaining 2 not shown)
""",
selections=[range(2), range(-9, 0)])
out = StringIO()
sorted_asu_proxies.show_sorted(by_value="delta",
sites_cart=sites_cart,
f=out,
prefix="@r",
max_items=0)
if (verbose):
sys.stdout.write(out.getvalue())
assert not show_diff(out.getvalue(), """\
@rNonbonded interactions: 7
""")
def write(self, data):
'Put, possibly replace, file contents with (new) data'
if not hasattr(data, 'read'):
data = StringIO(data)
self.jfs.up(self.path, data)
def test_cord2_rcs_03(self):
"""
all points are located at <30,40,50>
"""
model = BDF(debug=False)
cards = [
[
'CORD2S* 2 0 0. 0.',
'* 0. 0. 0. 1.* ',
'* 1. 0. 1.',
],
[
#'$ Coordinate System 30 : rectangular in spherical',
'CORD2R* 30 2 14. 30.',
'* 70. 13.431863852 32.1458443949 75.2107442927* ',
'* 14.4583462334 33.4569982885 68.2297989286',
],
[
#'$ Coordinate System 31 : cylindrical in spherical',
'CORD2C* 31 2 3. 42.',
'* -173. 2.86526881213 45.5425615252 159.180363517* ',
'* 3.65222385965 29.2536614627 -178.631312271',
],
[
#'$ Coordinate System 32 : spherical in spherical',
'CORD2S* 32 2 22. 14.',
'* 85. 22.1243073983 11.9537753718 77.9978191005* ',
'* 21.0997242967 13.1806120497 88.4824763008',
],
[
'GRID* 30 30 40.7437952957 -23.6254877994',
'* -33.09784854 0',
],
[
'GRID* 31 31 62.9378078196 15.9774797923',
'* 31.0484428362 0',
],
[
'GRID* 32 32 53.8270847449 95.8215692632',
'* 159.097767463 0',
],
]
for lines in cards:
card = model._process_card(lines)
model.add_card(card, card[0])
unused_xyz_cid0b = model.get_xyz_in_coord_no_xref(cid=0,
fdtype='float64')
unused_xyz_cid0c = model.get_xyz_in_coord_no_xref(cid=32,
fdtype='float64')
bdf_file = StringIO()
model.write_bdf(bdf_file, close=False)
bdf_file.seek(0)
#-------------------------------------------------
model.cross_reference()
model2 = BDFv(debug=False)
model2.read_bdf(bdf_file,
punch=True,
xref=False,
save_file_structure=False)
bdf_file.seek(0)
#-------------------------------------------------
xyz_cid0_actual = array([
[30., 40., 50.],
[30., 40., 50.],
[30., 40., 50.],
],
dtype='float64')
for nid in model.nodes:
node = model.Node(nid)
a = array([30., 40., 50.])
b = node.get_position()
self.assertTrue(
allclose(array([30., 40., 50.]), node.get_position()),
str(a - b))
xyz_cid0 = model.get_xyz_in_coord(cid=0, fdtype='float64')
assert np.allclose(xyz_cid0_actual,
xyz_cid0), '%s' % (xyz_cid0_actual - xyz_cid0)
out = model.get_displacement_index_xyz_cp_cd()
unused_icd_transform, icp_transform, xyz_cp, nid_cp_cd = out
nids = nid_cp_cd[:, 0]
xyz_cid0_xform = model.transform_xyzcp_to_xyz_cid(xyz_cp,
nids,
icp_transform,
cid=0)
array_equal(xyz_cid0_actual, xyz_cid0_xform)
assert array_equal(nids, array([30, 31, 32]))
model2.nodes.nids = nid_cp_cd[:, 0]
for cid in [30, 31, 32]:
unused_xyz_cid_a = model.transform_xyzcp_to_xyz_cid(xyz_cp,
nids,
icp_transform,
cid=cid)
unused_xyz_cid_b = model2.transform_xyzcp_to_xyz_cid(xyz_cp,
nids,
icp_transform,
cid=cid,
atol=None)
#assert np.allclose(xyz_cid_a, xyz_cid_b), '%s' % np.isclose(xyz_cid_a, xyz_cid_b)
#print(xyz_cid_a)
#print(xyz_cid_b)
#print(xyz_cid_a - xyz_cid_b)
#print('-------------')
#assert array_equal(xyz_cid_a, xyz_cid_b), 'error=%s' % (
#xyz_cid_a - xyz_cid_b)
#---------------------------------------------
xyz_cid0 = model.transform_xyzcp_to_xyz_cid(xyz_cp,
nids,
icp_transform,
cid=0,
atol=None)
array_equal(xyz_cid0_actual, xyz_cid0)
model.write_bdf(bdf_file, close=False)
model3 = BDF(debug=False)
origin = [14., 30., 70.]
zaxis = [13.431863852, 32.1458443949, 75.2107442927]
xzplane = [14.4583462334, 33.4569982885, 68.2297989286]
cord2r = model3.add_cord2r(30,
origin,
zaxis,
xzplane,
rid=2,
comment='')
origin = [3., 42., -173.]
zaxis = [2.86526881213, 45.5425615252, 159.180363517]
xzplane = [3.65222385965, 29.2536614627, -178.631312271]
cord2c = model3.add_cord2c(31,
origin,
zaxis,
xzplane,
rid=2,
comment='')
origin = [22., 14., 85.]
zaxis = [22.1243073983, 11.9537753718, 77.9978191005]
xzplane = [21.0997242967, 13.1806120497, 88.4824763008]
cord2s = model3.add_cord2s(32,
origin,
zaxis,
xzplane,
rid=2,
comment='')
assert cord2r == model.coords[
cord2r.cid], 'cord2r:\n%r\ncord2r[cid]:\n%r' % (
str(cord2r), str(model.coords[cord2r.cid]))
assert cord2c == model.coords[
cord2c.cid], 'cord2c:\n%r\ncord2c[cid]:\n%r' % (
str(cord2c), str(model.coords[cord2c.cid]))
assert cord2s == model.coords[
cord2s.cid], 'cord2s:\n%r\ncord2s[cid]:\n%r' % (
str(cord2s), str(model.coords[cord2s.cid]))
def __enter__(self): self._stdout = sys.stdout self._stderr = sys.stderr sys.stdout = self._io_stdout = StringIO() sys.stderr = self._io_stderr = StringIO() return self
def run(args, params=None, out=sys.stdout, log=sys.stderr):
# params keyword is for running program from GUI dialog
if ( ((len(args) == 0) and (params is None)) or
((len(args) > 0) and ((args[0] == "-h") or (args[0] == "--help"))) ):
show_usage()
return
# parse command-line arguments
if (params is None):
pcl = iotbx.phil.process_command_line_with_files(
args=args,
master_phil_string=master_phil_str,
pdb_file_def="file_name")
work_params = pcl.work.extract()
# or use parameters defined by GUI
else:
work_params = params
pdb_files = work_params.file_name
work_params.secondary_structure.enabled=True
assert work_params.format in ["phenix", "phenix_refine", "phenix_bonds",
"pymol", "refmac", "kinemage", "pdb", 'csv']
if work_params.quiet :
out = StringIO()
pdb_combined = iotbx.pdb.combine_unique_pdb_files(file_names=pdb_files)
pdb_structure = iotbx.pdb.input(source_info=None,
lines=flex.std_string(pdb_combined.raw_records))
cs = pdb_structure.crystal_symmetry()
corrupted_cs = False
if cs is not None:
if [cs.unit_cell(), cs.space_group()].count(None) > 0:
corrupted_cs = True
cs = None
elif cs.unit_cell().volume() < 10:
corrupted_cs = True
cs = None
if cs is None:
if corrupted_cs:
print("Symmetry information is corrupted, ", file=out)
else:
print("Symmetry information was not found, ", file=out)
print("putting molecule in P1 box.", file=out)
from cctbx import uctbx
atoms = pdb_structure.atoms()
box = uctbx.non_crystallographic_unit_cell_with_the_sites_in_its_center(
sites_cart=atoms.extract_xyz(),
buffer_layer=3)
atoms.set_xyz(new_xyz=box.sites_cart)
cs = box.crystal_symmetry()
defpars = mmtbx.model.manager.get_default_pdb_interpretation_params()
defpars.pdb_interpretation.automatic_linking.link_carbohydrates=False
defpars.pdb_interpretation.c_beta_restraints=False
defpars.pdb_interpretation.clash_guard.nonbonded_distance_threshold=None
model = mmtbx.model.manager(
model_input=pdb_structure,
crystal_symmetry=cs,
pdb_interpretation_params=defpars,
stop_for_unknowns=False)
pdb_hierarchy = model.get_hierarchy()
geometry = None
if pdb_hierarchy.contains_nucleic_acid():
geometry = model.get_restraints_manager().geometry
if len(pdb_hierarchy.models()) != 1 :
raise Sorry("Multiple models not supported.")
ss_from_file = None
if (hasattr(pdb_structure, "extract_secondary_structure") and
not work_params.ignore_annotation_in_file):
ss_from_file = pdb_structure.extract_secondary_structure()
m = manager(pdb_hierarchy=pdb_hierarchy,
geometry_restraints_manager=geometry,
sec_str_from_pdb_file=ss_from_file,
params=work_params.secondary_structure,
verbose=work_params.verbose)
# bp_p = nucleic_acids.get_basepair_plane_proxies(
# pdb_hierarchy,
# m.params.secondary_structure.nucleic_acid.base_pair,
# geometry)
# st_p = nucleic_acids.get_stacking_proxies(
# pdb_hierarchy,
# m.params.secondary_structure.nucleic_acid.stacking_pair,
# geometry)
# hb_b, hb_a = nucleic_acids.get_basepair_hbond_proxies(pdb_hierarchy,
# m.params.secondary_structure.nucleic_acid.base_pair)
result_out = StringIO()
# prefix_scope="refinement.pdb_interpretation"
# prefix_scope=""
prefix_scope=""
if work_params.format == "phenix_refine":
prefix_scope = "refinement.pdb_interpretation"
elif work_params.format == "phenix":
prefix_scope = "pdb_interpretation"
ss_phil = None
working_phil = m.as_phil_str(master_phil=sec_str_master_phil)
phil_diff = sec_str_master_phil.fetch_diff(source=working_phil)
if work_params.format in ["phenix", "phenix_refine"]:
comment = "\n".join([
"# These parameters are suitable for use in e.g. phenix.real_space_refine",
"# or geometry_minimization. To use them in phenix.refine add ",
"# 'refinement.' if front of pdb_interpretation."])
if work_params.format == "phenix_refine":
comment = "\n".join([
"# These parameters are suitable for use in phenix.refine only.",
"# To use them in other Phenix tools remove ",
"# 'refinement.' if front of pdb_interpretation."])
print(comment, file=result_out)
if (prefix_scope != ""):
print("%s {" % prefix_scope, file=result_out)
if work_params.show_all_params :
working_phil.show(prefix=" ", out=result_out)
else :
phil_diff.show(prefix=" ", out=result_out)
if (prefix_scope != ""):
print("}", file=result_out)
elif work_params.format == "pdb":
print(m.actual_sec_str.as_pdb_str(), file=result_out)
elif work_params.format == "phenix_bonds" :
raise Sorry("Not yet implemented.")
elif work_params.format in ["pymol", "refmac", "kinemage", 'csv'] :
m.show_summary(log=out)
model.process_input_model(make_restraints=True)
(hb_proxies, hb_angle_proxies, planarity_proxies,
parallelity_proxies) = m.create_all_new_restraints(
pdb_hierarchy=pdb_hierarchy,
grm=model.get_restraints_manager().geometry,
log=out)
if hb_proxies.size() > 0:
if work_params.format == "pymol" :
file_load_add = "load %s" % work_params.file_name[0]
# surprisingly, pymol handles filenames with whitespaces without quotes...
print(file_load_add, file=result_out)
bonds_in_format = hb_proxies.as_pymol_dashes(
pdb_hierarchy=pdb_hierarchy)
elif work_params.format == "kinemage" :
bonds_in_format = hb_proxies.as_kinemage(
pdb_hierarchy=pdb_hierarchy)
elif work_params.format == "csv" :
bonds_in_format = hb_proxies.as_csv(
pdb_hierarchy=pdb_hierarchy)
else :
bonds_in_format = hb_proxies.as_refmac_restraints(
pdb_hierarchy=pdb_hierarchy)
print(bonds_in_format, file=result_out)
if hb_angle_proxies.size() > 0:
if work_params.format == "pymol":
angles_in_format = hb_angle_proxies.as_pymol_dashes(
pdb_hierarchy=pdb_hierarchy)
print(angles_in_format, file=result_out)
result = result_out.getvalue()
out_prefix = os.path.basename(work_params.file_name[0])
if work_params.output_prefix is not None:
out_prefix = work_params.output_prefix
filename = "%s_ss.eff" % out_prefix
if work_params.format == "pymol":
filename = "%s_ss.pml" % out_prefix
outf = open(filename, "w")
outf.write(result)
outf.close()
print(result, file=out)
return os.path.abspath(filename)
def test_no_historical(self):
out = StringIO()
with replace_registry({"test_place": Place}):
management.call_command(self.command_name, auto=True, stdout=out)
self.assertIn(clean_old_history.Command.NO_REGISTERED_MODELS,
out.getvalue())
def four_lines_dataframe():
text = StringIO(FOUR_LINES)
return pd.read_csv(text, names=automobile_data.COLUMN_TYPES.keys(),
dtype=automobile_data.COLUMN_TYPES, na_values="?")
def merge_and_truncate(params, experiments, reflections):
"""Filter data, assess space group, run french wilson and Wilson stats."""
logger.info("\nMerging scaled reflection data\n")
# first filter bad reflections using dials.util.filter methods
reflections = filter_reflection_table(
reflections[0],
intensity_choice=["scale"],
d_min=params.d_min,
d_max=params.d_max,
combine_partials=params.combine_partials,
partiality_threshold=params.partiality_threshold,
)
# ^ scale factor has been applied, so now set to 1.0 - okay as not
# going to output scale factor in merged mtz.
reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0)
scaled_array = scaled_data_as_miller_array([reflections], experiments)
# Note, merge_equivalents does not raise an error if data is unique.
if params.anomalous:
anomalous_scaled = scaled_array.as_anomalous_array()
merged = scaled_array.merge_equivalents(
use_internal_variance=params.merging.use_internal_variance).array()
merged_anom = None
if params.anomalous:
merged_anom = anomalous_scaled.merge_equivalents(
use_internal_variance=params.merging.use_internal_variance).array(
)
# Before merge, do some assessment of the space_group
if params.assess_space_group:
merged_reflections = flex.reflection_table()
merged_reflections["intensity"] = merged.data()
merged_reflections["variance"] = flex.pow2(merged.sigmas())
merged_reflections["miller_index"] = merged.indices()
logger.info("Running systematic absences check")
run_systematic_absences_checks(experiments, merged_reflections)
# Run the stats on truncating on anomalous or non anomalous?
if params.anomalous:
intensities = merged_anom
else:
intensities = merged
assert intensities.is_xray_intensity_array()
amplitudes = None
anom_amplitudes = None
if params.truncate:
logger.info("\nScaling input intensities via French-Wilson Method")
out = StringIO()
if params.anomalous:
anom_amplitudes = intensities.french_wilson(params=params, log=out)
n_removed = intensities.size() - anom_amplitudes.size()
assert anom_amplitudes.is_xray_amplitude_array()
amplitudes = anom_amplitudes.as_non_anomalous_array()
amplitudes = amplitudes.merge_equivalents().array()
else:
amplitudes = intensities.french_wilson(params=params, log=out)
n_removed = intensities.size() - amplitudes.size()
logger.info("Total number of rejected intensities %s", n_removed)
logger.debug(out.getvalue())
if params.reporting.wilson_stats:
if not intensities.space_group().is_centric():
try:
wilson_scaling = data_statistics.wilson_scaling(
miller_array=intensities,
n_residues=params.n_residues) # XXX default n_residues?
except (IndexError, RuntimeError) as e:
logger.error(
"\n"
"Error encountered during Wilson statistics calculation:\n"
"Perhaps there are too few unique reflections.\n"
"%s",
e,
exc_info=True,
)
else:
# Divert output through logger - do with StringIO rather than
# info_handle else get way too much whitespace in output.
out = StringIO()
wilson_scaling.show(out=out)
logger.info(out.getvalue())
# Apply wilson B to give absolute scale?
# Show merging stats again.
if params.reporting.merging_stats:
try:
stats, anom_stats = merging_stats_from_scaled_array(
scaled_array,
params.merging.n_bins,
params.merging.use_internal_variance,
)
except DialsMergingStatisticsError as e:
logger.error(e, exc_info=True)
else:
if params.merging.anomalous and anom_stats:
logger.info(make_merging_statistics_summary(anom_stats))
else:
logger.info(make_merging_statistics_summary(stats))
logger.info(table_1_summary(stats, anom_stats))
return merged, merged_anom, amplitudes, anom_amplitudes
def _find_snippet_imports(module_data, module_path, strip_comments):
"""
Given the source of the module, convert it to a Jinja2 template to insert
module code and return whether it's a new or old style module.
"""
module_style = 'old'
if REPLACER in module_data:
module_style = 'new'
elif REPLACER_WINDOWS in module_data:
module_style = 'new'
elif 'from ansible.module_utils.' in module_data:
module_style = 'new'
elif 'WANT_JSON' in module_data:
module_style = 'non_native_want_json'
output = StringIO()
lines = module_data.split('\n')
snippet_names = []
for line in lines:
if REPLACER in line:
output.write(_slurp(os.path.join(_SNIPPET_PATH, "basic.py")))
snippet_names.append('basic')
if REPLACER_WINDOWS in line:
ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
output.write(ps_data)
snippet_names.append('powershell')
elif line.startswith('from ansible.module_utils.'):
tokens = line.split(".")
import_error = False
if len(tokens) != 3:
import_error = True
if " import *" not in line:
import_error = True
if import_error:
raise AnsibleError(
"error importing module in %s, expecting format like 'from ansible.module_utils.basic import *'"
% module_path)
snippet_name = tokens[2].split()[0]
snippet_names.append(snippet_name)
output.write(
_slurp(os.path.join(_SNIPPET_PATH, snippet_name + ".py")))
else:
if strip_comments and line.startswith("#") or line == '':
pass
output.write(line)
output.write("\n")
if not module_path.endswith(".ps1"):
# Unixy modules
if len(snippet_names) > 0 and not 'basic' in snippet_names:
raise AnsibleError(
"missing required import in %s: from ansible.module_utils.basic import *"
% module_path)
else:
# Windows modules
if len(snippet_names) > 0 and not 'powershell' in snippet_names:
raise AnsibleError(
"missing required import in %s: # POWERSHELL_COMMON" %
module_path)
return (output.getvalue(), module_style)
def test_no_args(self):
out = StringIO()
management.call_command(self.command_name,
stdout=out,
stderr=StringIO())
self.assertIn(clean_old_history.Command.COMMAND_HINT, out.getvalue())
def log():
return StringIO("""
Info: Benchmark time: 1 CPUs 1.13195 s/step 13.1013 days/ns 2727.91 MB memory
""")
def install(self):
options = self.options
if not os.path.exists(options['runzeo']):
logger.warn(no_runzeo % options['runzeo'])
run_directory = options['run-directory']
deployment = self.deployment
if deployment:
zeo_conf_path = os.path.join(options['etc-directory'],
self.name + '-zeo.conf')
zdaemon_conf_path = os.path.join(options['etc-directory'],
self.name + '-zdaemon.conf')
event_log_path = os.path.join(options['log-directory'],
self.name + '-zeo.log')
socket_path = os.path.join(run_directory,
self.name + '-zdaemon.sock')
rc = options['deployment-name'] + '-' + self.name
options.created(
zeo_conf_path,
zdaemon_conf_path,
os.path.join(options['rc-directory'], rc),
)
logrotate = options['logrotate']
if logrotate:
open(logrotate, 'w').write(logrotate_template % dict(
logfile=event_log_path,
rc=os.path.join(options['rc-directory'], rc),
conf=zdaemon_conf_path,
))
options.created(logrotate)
pack = options.get('pack')
if pack:
pack = pack.split()
if len(pack) < 5:
raise zc.buildout.UserError(
'Too few crontab fields in pack specification')
if len(pack) > 7:
raise zc.buildout.UserError(
'Too many values in pack option')
pack_path = os.path.join(
options['crontab-directory'],
"pack-%s-%s" % (options['deployment-name'], self.name),
)
if not os.path.exists(options['zeopack']):
logger.warn("Couln'e find zeopack script, %r",
options['zeopack'])
else:
zeo_conf_path = os.path.join(run_directory, 'zeo.conf')
zdaemon_conf_path = os.path.join(run_directory, 'zdaemon.conf')
event_log_path = os.path.join(run_directory, 'zeo.log')
socket_path = os.path.join(run_directory, 'zdaemon.sock')
rc = self.name
options.created(
run_directory,
os.path.join(options['rc-directory'], rc),
)
if not os.path.exists(run_directory):
os.mkdir(run_directory)
pack = pack_path = None
zeo_conf = options.get('zeo.conf', '') + '\n'
try:
zeo_conf = ZConfig.schemaless.loadConfigFile(StringIO(zeo_conf))
except ConfigurationSyntaxError as e:
raise zc.buildout.UserError('%s in:\n%s' % (e, zeo_conf))
zeo_section = [s for s in zeo_conf.sections if s.type == 'zeo']
if not zeo_section:
raise zc.buildout.UserError('No zeo section was defined.')
if len(zeo_section) > 1:
raise zc.buildout.UserError('Too many zeo sections.')
zeo_section = zeo_section[0]
if not 'address' in zeo_section:
raise zc.buildout.UserError('No ZEO address was specified.')
storages = [
s.name or '1' for s in zeo_conf.sections
if s.type not in ('zeo', 'eventlog', 'runner')
]
if not storages:
raise zc.buildout.UserError('No storages were defined.')
if not [s for s in zeo_conf.sections if s.type == 'eventlog']:
zeo_conf.sections.append(event_log('STDOUT'))
zdaemon_conf = options.get('zdaemon.conf', '') + '\n'
zdaemon_conf = ZConfig.schemaless.loadConfigFile(
StringIO(zdaemon_conf))
defaults = {
'program': "%s -C %s" % (options['runzeo'], zeo_conf_path),
'daemon': 'on',
'transcript': event_log_path,
'socket-name': socket_path,
'directory': run_directory,
}
if deployment:
defaults['user'] = options['user']
runner = [s for s in zdaemon_conf.sections if s.type == 'runner']
if runner:
runner = runner[0]
else:
runner = ZConfig.schemaless.Section('runner')
zdaemon_conf.sections.insert(0, runner)
for name, value in defaults.items():
if name not in runner:
runner[name] = [value]
if not [s for s in zdaemon_conf.sections if s.type == 'eventlog']:
zdaemon_conf.sections.append(event_log(event_log_path))
zdaemon_conf = str(zdaemon_conf)
self.egg.install()
requirements, ws = self.egg.working_set()
open(zeo_conf_path, 'w').write(str(zeo_conf))
open(zdaemon_conf_path, 'w').write(str(zdaemon_conf))
if options.get('shell-script') == 'true':
if not os.path.exists(options['zdaemon']):
logger.warn(no_zdaemon % options['zdaemon'])
contents = "%(zdaemon)s -C '%(conf)s' $*" % dict(
zdaemon=options['zdaemon'],
conf=zdaemon_conf_path,
)
if options.get('user'):
contents = 'su %(user)s -c \\\n "%(contents)s"' % dict(
user=options['user'],
contents=contents,
)
contents = "#!/bin/sh\n%s\n" % contents
dest = os.path.join(options['rc-directory'], rc)
if not (os.path.exists(dest) and open(dest).read() == contents):
open(dest, 'w').write(contents)
os.chmod(dest, 0o755)
logger.info("Generated shell script %r.", dest)
else:
self.egg.install()
requirements, ws = self.egg.working_set()
zc.buildout.easy_install.scripts(
[(rc, 'zdaemon.zdctl', 'main')],
ws,
options['executable'],
options['rc-directory'],
arguments=('['
'\n %r, %r,'
'\n ]+sys.argv[1:]'
'\n ' % (
'-C',
zdaemon_conf_path,
)),
)
if pack:
address, = zeo_section['address']
if ':' in address:
host, port = address.split(':')
address = '-h %s -p %s' % (host, port)
else:
try:
port = int(address)
except:
address = '-U ' + address
else:
address = '-p ' + address
f = open(pack_path, 'w')
if len(pack) == 7:
assert '@' in pack[6]
f.write("MAILTO=%s\n" % pack.pop())
if len(pack) == 6:
days = pack.pop()
else:
days = 1
for storage in storages:
f.write("%s %s %s %s -S %s -d %s\n" % (
' '.join(pack),
options['user'],
options['zeopack'],
address,
storage,
days,
))
f.close()
options.created(pack_path)
return options.created()
def exercise_hbond_as_cif_loop():
xs = sucrose()
for sc in xs.scatterers():
sc.flags.set_grad_site(True)
radii = [
covalent_radii.table(elt).radius()
for elt in xs.scattering_type_registry().type_index_pairs_as_dict()
]
asu_mappings = xs.asu_mappings(buffer_thickness=2 * max(radii) + 0.5)
pair_asu_table = crystal.pair_asu_table(asu_mappings)
pair_asu_table.add_covalent_pairs(xs.scattering_types(), tolerance=0.5)
hbonds = [
geometry.hbond(1, 5, sgtbx.rt_mx('-X,0.5+Y,2-Z')),
geometry.hbond(5, 14, sgtbx.rt_mx('-X,-0.5+Y,1-Z')),
geometry.hbond(7, 10, sgtbx.rt_mx('1+X,+Y,+Z')),
geometry.hbond(10, 0),
geometry.hbond(12, 14, sgtbx.rt_mx('-1-X,0.5+Y,1-Z')),
geometry.hbond(14, 12, sgtbx.rt_mx('-1-X,-0.5+Y,1-Z')),
geometry.hbond(16, 7)
]
loop = geometry.hbonds_as_cif_loop(hbonds,
pair_asu_table,
xs.scatterers().extract_labels(),
sites_frac=xs.sites_frac()).loop
s = StringIO()
print(loop, file=s)
assert not show_diff(
s.getvalue(), """\
loop_
_geom_hbond_atom_site_label_D
_geom_hbond_atom_site_label_H
_geom_hbond_atom_site_label_A
_geom_hbond_distance_DH
_geom_hbond_distance_HA
_geom_hbond_distance_DA
_geom_hbond_angle_DHA
_geom_hbond_site_symmetry_A
O2 H2 O4 0.8200 2.0636 2.8635 165.0 2_557
O4 H4 O9 0.8200 2.0559 2.8736 174.9 2_546
O5 H5 O7 0.8200 2.0496 2.8589 169.0 1_655
O7 H7 O1 0.8200 2.0573 2.8617 166.8 .
O8 H8 O9 0.8200 2.1407 2.8943 152.8 2_456
O9 H9 O8 0.8200 2.1031 2.8943 162.1 2_446
O10 H10 O5 0.8200 2.0167 2.7979 159.1 .
""")
# with a covariance matrix
flex.set_random_seed(1)
vcv_matrix = matrix.diag(
flex.random_double(size=xs.n_parameters(), factor=1e-5))\
.as_flex_double_matrix().matrix_symmetric_as_packed_u()
loop = geometry.hbonds_as_cif_loop(hbonds,
pair_asu_table,
xs.scatterers().extract_labels(),
sites_frac=xs.sites_frac(),
covariance_matrix=vcv_matrix,
parameter_map=xs.parameter_map()).loop
s = StringIO()
print(loop, file=s)
assert not show_diff(
s.getvalue(), """\
loop_
_geom_hbond_atom_site_label_D
_geom_hbond_atom_site_label_H
_geom_hbond_atom_site_label_A
_geom_hbond_distance_DH
_geom_hbond_distance_HA
_geom_hbond_distance_DA
_geom_hbond_angle_DHA
_geom_hbond_site_symmetry_A
O2 H2 O4 0.82(3) 2.06(3) 2.86(3) 165.0(18) 2_557
O4 H4 O9 0.82(4) 2.06(4) 2.87(4) 175(2) 2_546
O5 H5 O7 0.82(2) 2.05(2) 2.859(19) 169.0(18) 1_655
O7 H7 O1 0.82(2) 2.06(2) 2.86(2) 167(2) .
O8 H8 O9 0.82(3) 2.14(3) 2.89(3) 153(3) 2_456
O9 H9 O8 0.82(3) 2.10(3) 2.89(3) 162(2) 2_446
O10 H10 O5 0.82(3) 2.02(3) 2.80(3) 159(3) .
""")
cell_vcv = flex.pow2(matrix.diag(flex.random_double(size=6,factor=1e-1))\
.as_flex_double_matrix().matrix_symmetric_as_packed_u())
loop = geometry.hbonds_as_cif_loop(hbonds,
pair_asu_table,
xs.scatterers().extract_labels(),
sites_frac=xs.sites_frac(),
covariance_matrix=vcv_matrix,
cell_covariance_matrix=cell_vcv,
parameter_map=xs.parameter_map()).loop
s = StringIO()
print(loop, file=s)
assert not show_diff(
s.getvalue(), """\
loop_
_geom_hbond_atom_site_label_D
_geom_hbond_atom_site_label_H
_geom_hbond_atom_site_label_A
_geom_hbond_distance_DH
_geom_hbond_distance_HA
_geom_hbond_distance_DA
_geom_hbond_angle_DHA
_geom_hbond_site_symmetry_A
O2 H2 O4 0.82(3) 2.06(4) 2.86(4) 165.0(18) 2_557
O4 H4 O9 0.82(4) 2.06(4) 2.87(4) 175(2) 2_546
O5 H5 O7 0.82(2) 2.05(2) 2.86(2) 169.0(18) 1_655
O7 H7 O1 0.82(2) 2.06(3) 2.86(3) 167(2) .
O8 H8 O9 0.82(3) 2.14(4) 2.89(4) 153(3) 2_456
O9 H9 O8 0.82(3) 2.10(3) 2.89(4) 162(2) 2_446
O10 H10 O5 0.82(3) 2.02(3) 2.80(3) 159(3) .
""")
def empty_log():
return StringIO("""
not the log you are looking for
""")
def test_rload(self):
"""tests DLOAD, RLOAD1, RLOAD2, TABLED2 cards"""
model = BDF(debug=False)
#model.case_control_deck = CaseControlDeck(['DLOAD=2', 'BEGIN BULK'])
sid = 2
excite_id = 20
delay = 0
tid = 42
rload1 = model.add_rload1(sid,
excite_id,
delay=0,
dphase=0,
tc=0,
td=0,
Type='LOAD',
comment='rload1')
rload1 = model.add_rload1(sid,
excite_id,
delay=1.,
dphase=0,
tc=0,
td=0,
Type='DISP',
comment='rload1')
rload1 = model.add_rload1(sid,
excite_id,
delay=2,
dphase=0,
tc=0,
td=0,
Type='VELO',
comment='rload1')
rload1 = model.add_rload1(sid,
excite_id,
delay=0,
dphase=0,
tc=0,
td=0,
Type='ACC',
comment='rload1')
sid = 3
excite_id = 30
rload2 = model.add_rload2(sid,
excite_id,
delay=0,
dphase=0,
tb=0,
tp=0,
Type='LOAD',
comment='rload2')
rload2 = model.add_rload2(sid,
excite_id,
delay=1.,
dphase=0,
tb=0,
tp=0,
Type='D',
comment='rload2')
rload2 = model.add_rload2(sid,
excite_id,
delay=2,
dphase=0,
tb=0,
tp=0,
Type='V',
comment='rload2')
rload2 = model.add_rload2(sid,
excite_id,
delay=0,
dphase=0,
tb=0,
tp=0,
Type='A',
comment='rload2')
excite_id = 20
nid = 21
c = 1
scale = 1.0
model.add_darea(excite_id, nid, c, scale, comment='darea')
model.add_grid(nid)
excite_id = 30
model.add_darea(excite_id, nid, c, scale, comment='darea')
delay_id = 2
nodes = 100
components = 2
delays = 1.5
delay = model.add_delay(delay_id, nodes, components, delays)
sid = 1
scale = 1.0
scale_factors = 1.
load_ids = 2
dload = model.add_dload(sid,
scale,
scale_factors,
load_ids,
comment='dload')
x1 = 0.1
x = np.linspace(0., 1.)
y = np.sin(x)
tabled2 = model.add_tabled2(tid, x1, x, y, comment='tabled2')
model.pop_parse_errors()
delay.validate()
delay.raw_fields()
delay.write_card()
delay.write_card(size=16)
rload1.validate()
rload1.raw_fields()
rload1.write_card()
rload1.write_card(size=16)
rload2.validate()
rload2.raw_fields()
rload2.write_card()
rload2.write_card(size=16)
dload.validate()
dload.raw_fields()
dload.write_card()
dload.write_card(size=16)
tabled2.validate()
tabled2.raw_fields()
tabled2.write_card()
tabled2.write_card(size=16)
model.validate()
model.cross_reference()
model.pop_xref_errors()
#print(model.dareas)
bdf_file = StringIO()
model.write_bdf(bdf_file, close=False)
out = bdf_file.getvalue()
bdf_file.seek(0)
outs = model.get_bdf_stats(return_type='list')
outs = model.get_bdf_stats(return_type='string')
freq = 0.5
out1 = rload1.get_load_at_freq(freq, scale=1.)
#out2 = rload2.get_load_at_time(freq, scale=1.)
#print(out1)
#print(out2)
assert len(out1) == 1, out1
#assert len(out2) == 1, out2
freq = [0.5, 0.9]
out1 = rload1.get_load_at_freq(freq, scale=1.)
#out2 = rload2.get_load_at_freq(freq, scale=1.)
#print(out1)
#print(out2)
assert len(out1) == 2, out1
#assert len(out2) == 2, out2
model2 = read_bdf(bdf_file, punch=True, debug=False)
model2.uncross_reference()
model2.safe_cross_reference()
model2.uncross_reference()
def _getInputFromUser(param):
"""
this private func reads the data from the user
for the given param
"""
loop = True
userInput = None
try:
if param.USE_DEFAULT:
logging.debug("setting default value (%s) for key (%s)" %
(mask(param.DEFAULT_VALUE), param.CONF_NAME))
controller.CONF[param.CONF_NAME] = param.DEFAULT_VALUE
else:
while loop:
# If the value was not supplied by the command line flags
if param.CONF_NAME not in commandLineValues:
message = StringIO()
message.write(param.PROMPT)
val_list = param.VALIDATORS or []
if (validators.validate_regexp not in val_list
and param.OPTION_LIST):
message.write(" [%s]" % "|".join(param.OPTION_LIST))
if param.DEFAULT_VALUE:
message.write(" [%s] " % (str(param.DEFAULT_VALUE)))
message.write(": ")
message.seek(0)
# mask password or hidden fields
if (param.MASK_INPUT):
userInput = getpass.getpass("%s :" % (param.PROMPT))
else:
userInput = raw_input(message.read())
else:
userInput = commandLineValues[param.CONF_NAME]
# If DEFAULT_VALUE is set and user did not input anything
if userInput == "" and len(str(param.DEFAULT_VALUE)) > 0:
userInput = param.DEFAULT_VALUE
# Param processing
userInput = process_param_value(param, userInput)
# If param requires validation
try:
validate_param_value(param, userInput)
controller.CONF[param.CONF_NAME] = userInput
loop = False
except ParamValidationError:
if param.LOOSE_VALIDATION:
# If validation failed but LOOSE_VALIDATION is true, ask user
answer = _askYesNo("User input failed validation, "
"do you still wish to use it")
loop = not answer
if answer:
controller.CONF[param.CONF_NAME] = userInput
continue
else:
if param.CONF_NAME in commandLineValues:
del commandLineValues[param.CONF_NAME]
else:
# Delete value from commandLineValues so that we will prompt the user for input
if param.CONF_NAME in commandLineValues:
del commandLineValues[param.CONF_NAME]
loop = True
except KeyboardInterrupt:
# add the new line so messages wont be displayed in the same line as the question
print("")
raise
except:
logging.error(traceback.format_exc())
raise Exception(output_messages.ERR_EXP_READ_INPUT_PARAM %
(param.CONF_NAME))
def setup(process_name, user_log_dict):
"""Set up the weewx logging facility"""
# Create a ConfigObj from the default string. No interpolation (it interferes with the
# interpolation directives embedded in the string).
log_config = configobj.ConfigObj(StringIO(LOGGING_STR), interpolation=False, encoding='utf-8')
# Turn off interpolation in the incoming dictionary. First save the old
# value, then restore later. However, the incoming dictionary may be a simple
# Python dictionary and not have interpolation. Hence the try block.
try:
old_interpolation = user_log_dict.interpolation
user_log_dict.interpolation = False
except AttributeError:
old_interpolation = None
# Merge in the user additions / changes:
log_config.merge(user_log_dict)
# Restore the old interpolation value
if old_interpolation is not None:
user_log_dict.interpolation = old_interpolation
# Adjust the logging level in accordance with whether or not the 'debug' flag is on
log_level = 'DEBUG' if weewx.debug else 'INFO'
# Now we need to walk the structure, plugging in the values we know.
# First, we need a function to do this:
def _fix(section, key):
if isinstance(section[key], (list, tuple)):
# The value is a list or tuple
section[key] = [item.format(log_level=log_level,
address=address,
facility=facility,
process_name=process_name) for item in section[key]]
else:
# The value is a string
section[key] = section[key].format(log_level=log_level,
address=address,
facility=facility,
process_name=process_name)
# Using the function, walk the 'Logging' part of the structure
log_config['Logging'].walk(_fix)
# Extract just the part used by Python's logging facility
log_dict = log_config.dict().get('Logging', {})
# The root logger is denoted by an empty string by the logging facility. Unfortunately,
# ConfigObj does not accept an empty string as a key. So, instead, we use this hack:
try:
log_dict['loggers'][''] = log_dict['loggers']['root']
del log_dict['loggers']['root']
except KeyError:
pass
# Make sure values are of the right type
if 'version' in log_dict:
log_dict['version'] = to_int(log_dict['version'])
if 'disable_existing_loggers' in log_dict:
log_dict['disable_existing_loggers'] = to_bool(log_dict['disable_existing_loggers'])
if 'loggers' in log_dict:
for logger in log_dict['loggers']:
if 'propagate' in log_dict['loggers'][logger]:
log_dict['loggers'][logger]['propagate'] = to_bool(log_dict['loggers'][logger]['propagate'])
# Finally! The dictionary is ready. Set the defaults.
logging.config.dictConfig(log_dict)
def exercise_cif_from_cctbx():
quartz = xray.structure(crystal_symmetry=crystal.symmetry(
(5.01, 5.01, 5.47, 90, 90, 120), "P6222"),
scatterers=flex.xray_scatterer([
xray.scatterer("Si", (1 / 2., 1 / 2., 1 / 3.)),
xray.scatterer("O", (0.197, -0.197, 0.83333))
]))
for sc in quartz.scatterers():
sc.flags.set_grad_site(True)
s = StringIO()
loop = geometry.distances_as_cif_loop(
quartz.pair_asu_table(distance_cutoff=2),
site_labels=quartz.scatterers().extract_labels(),
sites_frac=quartz.sites_frac()).loop
print(loop, file=s)
assert not show_diff(
s.getvalue(), """\
loop_
_geom_bond_atom_site_label_1
_geom_bond_atom_site_label_2
_geom_bond_distance
_geom_bond_site_symmetry_2
Si O 1.6160 4_554
Si O 1.6160 2_554
Si O 1.6160 3_664
Si O 1.6160 5_664
""")
s = StringIO()
loop = geometry.angles_as_cif_loop(
quartz.pair_asu_table(distance_cutoff=2),
site_labels=quartz.scatterers().extract_labels(),
sites_frac=quartz.sites_frac()).loop
print(loop, file=s)
assert not show_diff(
s.getvalue(), """\
loop_
_geom_angle_atom_site_label_1
_geom_angle_atom_site_label_2
_geom_angle_atom_site_label_3
_geom_angle
_geom_angle_site_symmetry_1
_geom_angle_site_symmetry_3
O Si O 101.3 2_554 4_554
O Si O 111.3 3_664 4_554
O Si O 116.1 3_664 2_554
O Si O 116.1 5_664 4_554
O Si O 111.3 5_664 2_554
O Si O 101.3 5_664 3_664
Si O Si 146.9 3 5
""")
# with a covariance matrix
flex.set_random_seed(1)
vcv_matrix = matrix.diag(
flex.random_double(size=quartz.n_parameters(), factor=1e-5))\
.as_flex_double_matrix().matrix_symmetric_as_packed_u()
s = StringIO()
loop = geometry.distances_as_cif_loop(
quartz.pair_asu_table(distance_cutoff=2),
site_labels=quartz.scatterers().extract_labels(),
sites_frac=quartz.sites_frac(),
covariance_matrix=vcv_matrix,
parameter_map=quartz.parameter_map()).loop
print(loop, file=s)
assert not show_diff(
s.getvalue(), """\
loop_
_geom_bond_atom_site_label_1
_geom_bond_atom_site_label_2
_geom_bond_distance
_geom_bond_site_symmetry_2
Si O 1.616(14) 4_554
Si O 1.616(12) 2_554
Si O 1.616(14) 3_664
Si O 1.616(12) 5_664
""")
s = StringIO()
loop = geometry.angles_as_cif_loop(
quartz.pair_asu_table(distance_cutoff=2),
site_labels=quartz.scatterers().extract_labels(),
sites_frac=quartz.sites_frac(),
covariance_matrix=vcv_matrix,
parameter_map=quartz.parameter_map()).loop
print(loop, file=s)
assert not show_diff(
s.getvalue(), """\
loop_
_geom_angle_atom_site_label_1
_geom_angle_atom_site_label_2
_geom_angle_atom_site_label_3
_geom_angle
_geom_angle_site_symmetry_1
_geom_angle_site_symmetry_3
O Si O 101.3(8) 2_554 4_554
O Si O 111.3(10) 3_664 4_554
O Si O 116.1(9) 3_664 2_554
O Si O 116.1(9) 5_664 4_554
O Si O 111.3(10) 5_664 2_554
O Si O 101.3(8) 5_664 3_664
Si O Si 146.9(9) 3 5
""")
cell_vcv = flex.pow2(matrix.diag(flex.random_double(size=6,factor=1e-1))\
.as_flex_double_matrix().matrix_symmetric_as_packed_u())
s = StringIO()
loop = geometry.distances_as_cif_loop(
quartz.pair_asu_table(distance_cutoff=2),
site_labels=quartz.scatterers().extract_labels(),
sites_frac=quartz.sites_frac(),
covariance_matrix=vcv_matrix,
cell_covariance_matrix=cell_vcv,
parameter_map=quartz.parameter_map()).loop
print(loop, file=s)
assert not show_diff(
s.getvalue(), """\
loop_
_geom_bond_atom_site_label_1
_geom_bond_atom_site_label_2
_geom_bond_distance
_geom_bond_site_symmetry_2
Si O 1.616(15) 4_554
Si O 1.616(19) 2_554
Si O 1.616(15) 3_664
Si O 1.616(19) 5_664
""")
def test_section_typo(self):
text = '\n'.join(['[clue]', 'rows=1', 'columns=1'])
stream = StringIO(text)
with pytest.raises(NoSectionError, match="No section: u?'clues'"):
read_ini(stream)
def test_tload(self):
"""tests DLOAD, TLOAD1, TLOAD2, TABLED2 cards"""
model = BDF(debug=False)
sid = 2
excite_id = 20
delay = 0
tid = 42
tload1 = model.add_tload1(sid,
excite_id,
tid,
delay=0,
Type='LOAD',
us0=0.0,
vs0=0.0,
comment='tload1')
tload1 = model.add_tload1(sid,
excite_id,
tid,
delay=1.,
Type='DISP',
us0=0.0,
vs0=0.0,
comment='')
tload1 = model.add_tload1(sid,
excite_id,
tid,
delay=2,
Type='VELO',
us0=0.0,
vs0=0.0,
comment='')
tload1 = model.add_tload1(sid,
excite_id,
tid,
delay=0,
Type='ACC',
us0=0.0,
vs0=0.0,
comment='')
sid = 3
excite_id = 30
tload2 = model.add_tload2(sid,
excite_id,
delay=0,
Type='LOAD',
T1=0.,
T2=None,
frequency=0.,
phase=0.,
c=0.,
b=0.,
us0=0.,
vs0=0.,
comment='tload2')
tload2 = model.add_tload2(sid,
excite_id,
delay=1.,
Type='D',
T1=0.,
T2=None,
frequency=0.,
phase=0.,
c=0.,
b=0.,
us0=0.,
vs0=0.,
comment='')
tload2 = model.add_tload2(sid,
excite_id,
delay=2,
Type='V',
T1=0.,
T2=None,
frequency=0.,
phase=0.,
c=0.,
b=0.,
us0=0.,
vs0=0.,
comment='')
tload2 = model.add_tload2(sid,
excite_id,
delay=0,
Type='A',
T1=0.,
T2=1.,
frequency=0.,
phase=0.,
c=0.,
b=0.,
us0=0.,
vs0=0.,
comment='')
delay_id = 2
nodes = 100
components = 2
delays = 1.5
delay = model.add_delay(delay_id, nodes, components, delays)
sid = 1
scale = 1.0
scale_factors = 1.
load_ids = 2
dload = model.add_dload(sid,
scale,
scale_factors,
load_ids,
comment='dload')
x1 = 0.1
x = np.linspace(0., 1.)
y = np.sin(x)
tabled2 = model.add_tabled2(tid, x1, x, y, comment='tabled2')
model.pop_parse_errors()
delay.validate()
delay.raw_fields()
delay.write_card()
delay.write_card(size=16)
tload1.validate()
tload1.raw_fields()
tload1.write_card()
tload1.write_card(size=16)
tload2.validate()
tload2.raw_fields()
tload2.write_card()
tload2.write_card(size=16)
dload.validate()
dload.raw_fields()
dload.write_card()
dload.write_card(size=16)
tabled2.validate()
tabled2.raw_fields()
tabled2.write_card()
tabled2.write_card(size=16)
model.validate()
model.cross_reference()
model.pop_xref_errors()
bdf_file = StringIO()
model.write_bdf(bdf_file, close=False)
out = bdf_file.getvalue()
bdf_file.seek(0)
outs = model.get_bdf_stats(return_type='list')
outs = model.get_bdf_stats(return_type='string')
time = 0.5
out1 = tload1.get_load_at_time(time, scale=1.)
out2 = tload2.get_load_at_time(time, scale=1.)
#print(out1)
assert len(out1) == 1, out1
assert len(out2) == 1, out2
#print(out1)
#print(out2)
time = [0.5, 0.9]
out1 = tload1.get_load_at_time(time, scale=1.)
out2 = tload2.get_load_at_time(time, scale=1.)
assert len(out1) == 2, out1
assert len(out2) == 2, out2
#print(out1)
#print(out2)
model2 = read_bdf(bdf_file, punch=True, debug=False)
model2.uncross_reference()
model2.safe_cross_reference()
model2.uncross_reference()
def setUp(self):
super(TestUpgradeCheckCellsV2, self).setUp()
self.output = StringIO()
self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.output))
self.useFixture(nova_fixtures.Database(database='api'))
self.cmd = status.UpgradeCommands()
def __import__(self, apply_node, check=True, reason=None):
"""
Given an apply_node, recursively search from this node to know graph,
and then add all unknown variables and apply_nodes to this graph.
"""
node = apply_node
# We import the nodes in topological order. We only are interested
# in new nodes, so we use all variables we know of as if they were the input set.
# (the functions in the graph module only use the input set to
# know where to stop going down)
new_nodes = graph.io_toposort(self.variables, apply_node.outputs)
if check:
for node in new_nodes:
if hasattr(node, 'fgraph') and node.fgraph is not self:
raise Exception("%s is already owned by another fgraph" %
node)
for r in node.inputs:
if hasattr(r, 'fgraph') and r.fgraph is not self:
raise Exception(
"%s is already owned by another fgraph" % r)
if (r.owner is None and not isinstance(r, graph.Constant)
and r not in self.inputs):
# Verbose error message
# Show a complete chain of variables from the missing input to an output
if config.exception_verbosity == 'high':
def find_path_to(output_var, input_var):
"""
Returns a list of each variable on a (not
necessarily unique) path from input_var to
output_var, where each variable in the list has
the preceding variable as one of its inputs.
Returns None if no path exists.
"""
# If output and input are the same we have a singleton path
if output_var is input_var:
return [output_var]
# If output has no inputs then there is no path
owner = output_var.owner
if owner is None:
return None
# If input_var is an input to the output node, there is a
# simple two element path
inputs = owner.inputs
if input_var in inputs:
return [input_var, output_var]
# Otherwise we must recurse by searching for a path to one
# of our inputs, then appending the output to that path
for ipt in inputs:
path = find_path_to(ipt, input_var)
if path is not None:
path.append(output_var)
return path
# Since none of the above methods returned a path, there is none
return None
# Try different outputs until we find one that has a path to the missing input
for output in self.outputs:
path = find_path_to(output, r)
if path is not None:
break
# if there is no path then r isn't really a graph input so we shouldn't be running error
# handler code in the first place
assert path is not None
tr = getattr(r.tag, 'trace', [])
detailed_err_msg = ""
if len(tr) > 0:
detailed_err_msg += "\nBacktrace when the variable is created:\n"
# Print separate message for each element in
# the list of batcktraces
sio = StringIO()
for subtr in tr:
traceback.print_list(subtr, sio)
detailed_err_msg += str(sio.getvalue())
raise MissingInputError(
'A variable that is an input to the graph was '
'neither provided as an input to the function '
'nor given a value. A chain of variables '
'leading from this input to an output is %s. '
'This chain may not be unique' % str(path) +
detailed_err_msg)
# Standard error message
raise MissingInputError(
("An input of the graph, used to compute %s, "
"was not provided and not given a value."
"Use the Theano flag exception_verbosity='high',"
"for more information on this error." %
str(node)), r)
for node in new_nodes:
assert node not in self.apply_nodes
self.__setup_node__(node)
self.apply_nodes.add(node)
for output in node.outputs:
self.__setup_r__(output)
self.variables.add(output)
for i, input in enumerate(node.inputs):
if input not in self.variables:
self.__setup_r__(input)
self.variables.add(input)
self.__add_clients__(input, [(node, i)])
assert node.fgraph is self
self.execute_callbacks('on_import', node, reason)
class TestUpgradeCheckBasic(test.NoDBTestCase):
"""Tests for the nova-status upgrade check command.
The tests in this class should just test basic logic and use mock. Real
checks which require more elaborate fixtures or the database should be done
in separate test classes as they are more or less specific to a particular
release and may be removed in a later release after they are no longer
needed.
"""
def setUp(self):
super(TestUpgradeCheckBasic, self).setUp()
self.output = StringIO()
self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.output))
self.cmd = status.UpgradeCommands()
def test_check_success(self):
fake_checks = (('good',
mock.Mock(return_value=status.UpgradeCheckResult(
status.UpgradeCheckCode.SUCCESS))), )
with mock.patch.object(self.cmd, '_upgrade_checks', fake_checks):
self.assertEqual(status.UpgradeCheckCode.SUCCESS, self.cmd.check())
expected = """\
+-----------------------+
| Upgrade Check Results |
+-----------------------+
| Check: good |
| Result: Success |
| Details: None |
+-----------------------+
"""
self.assertEqual(expected, self.output.getvalue())
def test_check_warning(self):
fake_checks = (
('good',
mock.Mock(return_value=status.UpgradeCheckResult(
status.UpgradeCheckCode.SUCCESS))),
('warn',
mock.Mock(return_value=status.UpgradeCheckResult(
status.UpgradeCheckCode.WARNING, 'there might be a problem'))
),
)
with mock.patch.object(self.cmd, '_upgrade_checks', fake_checks):
self.assertEqual(status.UpgradeCheckCode.WARNING, self.cmd.check())
expected = """\
+-----------------------------------+
| Upgrade Check Results |
+-----------------------------------+
| Check: good |
| Result: Success |
| Details: None |
+-----------------------------------+
| Check: warn |
| Result: Warning |
| Details: there might be a problem |
+-----------------------------------+
"""
self.assertEqual(expected, self.output.getvalue())
def test_check_failure(self):
# make the error details over 60 characters so we test the wrapping
error_details = 'go back to bed' + '!' * 60
fake_checks = (
('good',
mock.Mock(return_value=status.UpgradeCheckResult(
status.UpgradeCheckCode.SUCCESS))),
('warn',
mock.Mock(return_value=status.UpgradeCheckResult(
status.UpgradeCheckCode.WARNING, 'there might be a problem'))
),
('fail',
mock.Mock(return_value=status.UpgradeCheckResult(
status.UpgradeCheckCode.FAILURE, error_details))),
)
with mock.patch.object(self.cmd, '_upgrade_checks', fake_checks):
self.assertEqual(status.UpgradeCheckCode.FAILURE, self.cmd.check())
expected = """\
+-----------------------------------------------------------------------+
| Upgrade Check Results |
+-----------------------------------------------------------------------+
| Check: good |
| Result: Success |
| Details: None |
+-----------------------------------------------------------------------+
| Check: warn |
| Result: Warning |
| Details: there might be a problem |
+-----------------------------------------------------------------------+
| Check: fail |
| Result: Failure |
| Details: go back to bed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! |
| !!!!!!!!!!!!!! |
+-----------------------------------------------------------------------+
"""
self.assertEqual(expected, self.output.getvalue())
def setUp(self):
super(TestNovaStatusMain, self).setUp()
self.output = StringIO()
self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.output))
def generateTransactionsSummary(files,unmatched_only=False):
try:
# Split transactions into biller codes
biller_codes = {}
biller_code_emails = {}
if unmatched_only:
for f in files:
for t in f.transactions.all():
if t.biller_code in biller_codes:
txns = list(biller_codes[t.biller_code])
txns.append(t)
biller_codes[t.biller_code] = txns
else:
biller_codes[t.biller_code] = [t]
else:
for n, f in files:
for t in f.transactions.all():
if t.biller_code in biller_codes:
txns = list(biller_codes[t.biller_code])
txns.append(t)
biller_codes[t.biller_code] = txns
else:
biller_codes[t.biller_code] = [t]
# Generate summaries per biller code
for k,v in biller_codes.items():
matched = []
unmatched = []
for t in v:
if t.matched:
matched.append(t)
else:
unmatched.append(t)
output = StringIO()
if not unmatched_only:
# Matched txns
output.write('Matched transactions:\n')
for m in matched:
output.write(' CRN: {} Amount: ${}\n'.format(m.crn,m.amount))
# Unmatched txns
output.write('\nUnmatched transactions:\n')
for u in unmatched:
output.write(' CRN: {} Amount: ${}\n'.format(u.crn,u.amount))
contents = output.getvalue()
output.close()
# Add the biller code email
biller_code_emails[k] = contents
return biller_code_emails
except Exception as e:
traceback.print_exc(e)
raise
def setUp(self):
super(TestUpgradeCheckBasic, self).setUp()
self.output = StringIO()
self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.output))
self.cmd = status.UpgradeCommands()
def generateParserSummary(files):
valid = files['valid']
other = files['other']
failed = files['failed']
processed = files['processed']
output = StringIO()
output.write('Successful Files with transactions:\n')
# Successful Files
for n,t in valid:
output.write(' File Name: {}\n'.format(n))
output.write(' Transactions:\n')
for trans in t.transactions.all():
output.write(' CRN: {}\n'.format(trans.crn))
# Successful Files without transactions
output.write('\nSuccessful Files without transactions:\n')
for n,t in other:
output.write(' File Name: {}\n'.format(n))
# Failed files
output.write('\nFailed Files:\n')
for n,r in failed:
output.write(' File Name: {}\n'.format(n))
output.write(' Reason: {}\n'.format(r))
# Already processed Files
output.write('\nFiles previously processed:\n')
for n,t in processed:
output.write(' File Name: {}\n'.format(n))
contents = output.getvalue()
output.close()
return contents
def generate_yang(test, uuid_set):
global count, level_memory
global mqtt_commands
global my_set
global device_category
my_set = uuid_set
'''
Generate a YANG-in-XML Tree
- print the YANG Tree as string with SerialIO
- build a new root-Element, called <data> with 'xmlns' Attribute
- attach the stringified CDATA to the new Element
- print the XML
'''
#python-modeled.netconf/modeled/netconf/yang/__init__.py
module1 = Statement(None, None, None, 'module', 'mqtt-led')
my_namespace = "http://ipv6lab.beuth-hochschule.de/led"
my_prefix = "led"
namespace = Statement(None, module1, None, 'namespace', my_namespace)
module1.substmts.append(namespace)
prefix = Statement(None, module1, None, 'prefix', my_prefix)
module1.substmts.append(prefix)
#http://stackoverflow.com/questions/10844064/items-in-json-object-are-out-of-order-using-json-dumps
data = json.loads(test, object_pairs_hook=OrderedDict)
count = 0
level_memory = {}
#print_dict(data, module1, count)
parse_dict(data, module1)
#revision = str(datetime.now())
#revision = Statement(None, module, None, 'revision', revision)
#module.substmts.append(revision)
#https://github.com/mbj4668/pyang/blob/master/pyang/plugin.py
#https://github.com/modeled/modeled.netconf/blob/master/modeled/netconf/yang/container.py
"""Serialize YANG container to the given output `format`.
"""
# output stream for pyang output plugin
stream = StringIO()
# gets filled with all availabe pyang output format plugins
PYANG_PLUGINS = {}
# register and initialise pyang plugin
pyang.plugin.init([])
for plugin in pyang.plugin.plugins:
plugin.add_output_format(PYANG_PLUGINS)
del plugin
#for name in PYANG_PLUGINS:
# print(name)
#...
#dsdl
#depend
#name
#omni
#yin
#tree
#jstree
#capability
#yang
#uml
#jtox
#jsonxsl
#sample-xml-skeleton
plugin = PYANG_PLUGINS['yang']
# register plugin options according to pyang script
optparser = OptionParser()
plugin.add_opts(optparser)
# pyang plugins also need a pyang.Context
ctx = pyang.Context(DummyRepository())
# which offers plugin-specific options (just take defaults)
ctx.opts = optparser.parse_args([])[0]
# ready to serialize
plugin.emit(ctx, [module1], stream)
# and return the resulting data
stream.seek(0)
yang = stream.getvalue()
print('\nAusgabe: ')
print(stream.read())
print("")
#return stream.read()
#root = etree.Element("data", xmlns="urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring")
#root.text = etree.CDATA(yang)
#print etree.tostring(root, pretty_print=True)
#return etree.tostring(root, pretty_print=True)
#returns the constructed yang from json-config, a list of mqtt-commands and a set of uuids
return (yang, mqtt_commands, my_set, device_category)
