Example #1
0
def parse_image(img, out_dir=None):
	print "parsing image %s" % img

	out_file = "%s.j3m" % img

	if out_dir is not None:
		out_file = os.path.join(out_dir, out_file.split("/")[-1])
	
	j3m_data = StringIO()
	obscura_marker_found = False

	cmd = [os.path.join(os.path.dirname(os.path.abspath(__file__)), "lib", "j3mparser.out"), img]

	p = Popen(cmd, stdout=PIPE, close_fds=True)
	data = p.stdout.readline()

	while data:
		if re.match(r'^file: .*', data):
			pass
		elif re.match(r'^Generic APPn .*', data):
			pass
		elif re.match(r'^Component.*', data):
			pass
		elif re.match(r'^Didn\'t find .*', data):
			pass
		elif re.match(r'^Got obscura marker.*', data):
			obscura_marker_found = True
		else:
			if obscura_marker_found:
				j3m_data.write(data)

		data = p.stdout.readline()

	p.stdout.close()


	try:
		j3m_data = j3m_data.getvalue()

		if j3m_data.split('\n')[:2] == gpg_sentinel:
			print "Now decrypting..."

			j3m_asc = "%s.asc" % out_file
			
			with open(j3m_asc, 'wb+') as OUT:
				OUT.write(j3m_data)

			decrypt_file(j3m_asc, out_file)

		else:
			with open(out_file, 'wb+') as OUT:
				OUT.write(j3m_data)

		return True, out_file
	except Exception as e:
		print "could not get j3m data from this image"
		print e, type(e)

	return False
def dump_threads():
    """Dump running threads

    Returns a string with the tracebacks.
    """
    res = []
    frames = sys._current_frames()
    log.debug('Number of Threads: %s' % str(len(frames) - 1))
    # if Number of Threads == 0, then the Zope instance
    # hadn't handled any request
    this_thread_id = thread.get_ident()

    for thread_id, frame in frames.iteritems():
        if thread_id == this_thread_id:
            continue
        # Find request in frame
        reqinfo = ['']
        f = frame
        while f is not None:
            co = f.f_code
            if co.co_filename.endswith('Publish.py'):
                request = f.f_locals.get('request')
                if request is not None:
                    method = request.get('REQUEST_METHOD', '')
                    path = request.get('PATH_INFO', '')
                    url = request.get('URL', '')
                    agent = request.get('HTTP_USER_AGENT', '')
                    query_string = request.get('QUERY_STRING')

                    query = 'QUERY: %s %s' % (method, path)
                    if query_string is not None:
                        query += '?%s' % query_string

                    reqinfo.append(query)
                    reqinfo.append('URL: %s' % url)
                    reqinfo.append('HTTP_USER_AGENT: %s' % agent)
                    break
            f = f.f_back

        output = StringIO()
        traceback.print_stack(frame, file=output)
        output = output.getvalue()

        lines = [line.strip() for line in output.split('\n')
                 if line.strip() != '']
        zeo_marker = os.path.join('ZEO', 'zrpc', 'connection')
        acquire_marker = 'l.acquire()'
        if len(lines) > 1 and (zeo_marker in lines[-2] or
                               acquire_marker in lines[-1]):
            output = None

        res.append((thread_id, reqinfo, output))
    return res
def dump_threads():
    """Dump running threads

    Returns a string with the tracebacks.
    """
    res = []
    frames = sys._current_frames()
    this_thread_id = thread.get_ident()

    for thread_id, frame in frames.iteritems():
        if thread_id == this_thread_id:
            continue
        # Find request in frame
        reqinfo = ['']
        f = frame
        while f is not None:
            co = f.f_code
            if co.co_filename.endswith('Publish.py'):
                request = f.f_locals.get('request')
                if request is not None:
                    method = request.get('REQUEST_METHOD', '')
                    path = request.get('PATH_INFO', '')
                    url = request.get('URL', '')
                    agent = request.get('HTTP_USER_AGENT', '')
                    query_string = request.get('QUERY_STRING')

                    query = 'QUERY: %s %s' % (method, path)
                    if query_string is not None:
                        query += '?%s' % query_string

                    reqinfo.append(query)
                    reqinfo.append('URL: %s' % url)
                    reqinfo.append('HTTP_USER_AGENT: %s' % agent)
                    break
            f = f.f_back

        output = StringIO()
        traceback.print_stack(frame, file=output)
        output = output.getvalue()

        lines = [
            line.strip() for line in output.split('\n') if line.strip() != ''
        ]
        zeo_marker = os.path.join('ZEO', 'zrpc', 'connection')
        acquire_marker = 'l.acquire()'
        if len(lines) > 1 and (zeo_marker in lines[-2]
                               or acquire_marker in lines[-1]):
            output = None

        res.append((thread_id, reqinfo, output))
    return res
def buildDeb ( module, install = False ):
    """ Create a debian package for the module
    """
    logger.info("Trying to build a debain package for module %s" % module.base)
    srcdir = os.path.join(os.getcwd(), module.base)
    if not os.path.exists(srcdir):
        exit("You need to get the src first, use -r")

    # Check if the module already has debian rules to create a package
    debiandir = os.path.join(srcdir, "debian")
    debianour = os.path.join("python-ogre", "debs", "%s-debian" % module.base)

    if not os.path.exists(debiandir):
        logger.info("Package does not come with a debian dir, seeing if we have our own..")

        # Do we have a debian rules we can copy into the package?
        if not os.path.exists(debianour):
            logger.error("We do not have a debian dir!?")
            exit("Can not build debian packages for this module.")
        else:
            # Do the copy
            ret = spawnTask("cp -rvf %s %s" % (debianour, debiandir), os.getcwd())
            if ret != 0:
                exit("Was not able to copy the debian directory over")

    # Rewrite any place holders
    for file in os.listdir(debiandir):
        if not os.path.isfile(os.path.join(debiandir, file)):
            continue
        ret = spawnTask( "sed --in-place "+os.path.join(debiandir,file)+" -e\"s|%%SHORTDATE%%|`date +%Y%m%d`|g\" -e\"s|%%LONGDATE%%|`date +'%a, %d %b %Y %H:%m:%S %z'`|g\" -e\"s|%%VERSION%%|"+module.source_version+"|g\"", srcdir)
        if ret != 0:
            exit("Was not able to update the debian %s." % file)

    # Check that all the dependencies for this package are avaliable
    c = StringIO()
    ret = spawnTask("dpkg-checkbuilddeps", srcdir, c)
    if ret != 0:
        c = c.getvalue().strip()
        c = c.split(':')[-1].strip().split()
        logger.info ("Please install the following dependencies %r " % c)
        exit("The package's dependencies have not been installed.")

    os.chmod(os.path.join(debiandir, "rules"), 0755)

    ret = spawnTask("fakeroot ./debian/rules binary", srcdir, c)
    if ret != 0:
        logger.warning("Was not able to build package")
    logger.info("Package successfully built!")
Example #5
0
def exercise_two_models_with_holes(model):
  selection_strings=["chain A", "chain B", "chain C", "chain D"]
  group = ncs.restraints.group.from_atom_selections(
    model=model,
    reference_selection_string=None,
    selection_strings=selection_strings,
    coordinate_sigma=0.05,
    b_factor_weight=0.4321,
    special_position_warnings_only=False)
  sites_cart = model.pdb_atoms.extract_xyz()
  ncs_operators = group.operators(sites_cart=sites_cart)
  out = StringIO()
  ncs_operators.show(sites_cart=sites_cart, out=out, prefix="{*")
  assert not show_diff(out.getvalue(), """\
{*NCS operator 1:
{*  Reference selection: "chain A"
{*      Other selection: "chain B"
{*  Number of atom pairs: 22
{*  Rotation={{-0.925533, 0.322815, -0.197938},
{*            {0.329616, 0.429511, -0.840758},
{*            {-0.186393, -0.843393, -0.503931}}
{*  Translation={{163.62}, {-13.0292}, {44.8533}}
{*  Histogram of differences:
{*    0.092573 - 0.238983: 1
{*    0.238983 - 0.385393: 2
{*    0.385393 - 0.531803: 7
{*    0.531803 - 0.678214: 3
{*    0.678214 - 0.824624: 4
{*    0.824624 - 0.971034: 5
{*  RMS difference with respect to the reference: 0.653687
{*NCS operator 2:
{*  Reference selection: "chain A"
{*      Other selection: "chain C"
{*  Number of atom pairs: 32
{*  Rotation={{-0.988874, -0.139883, -0.0506023},
{*            {0.0139383, -0.425808, 0.904706},
{*            {-0.1481, 0.893935, 0.423021}}
{*  Translation={{177.315}, {18.2319}, {1.24026}}
{*  Histogram of differences:
{*    0.291817 - 0.418479: 2
{*    0.418479 - 0.545141: 9
{*    0.545141 - 0.671802: 5
{*    0.671802 - 0.798464: 10
{*    0.798464 - 0.925126: 3
{*    0.925126 - 1.051787: 3
{*  RMS difference with respect to the reference: 0.677436
{*NCS operator 3:
{*  Reference selection: "chain A"
{*      Other selection: "chain D"
{*  Number of atom pairs: 24
{*  Rotation={{0.950594, -0.191857, 0.244055},
{*            {-0.192933, -0.981014, -0.0197252},
{*            {0.243205, -0.0283355, -0.969561}}
{*  Translation={{6.77797}, {56.2476}, {-5.96327}}
{*  Histogram of differences:
{*    0.270982 - 0.414517: 3
{*    0.414517 - 0.558053: 3
{*    0.558053 - 0.701588: 7
{*    0.701588 - 0.845124: 6
{*    0.845124 - 0.988659: 2
{*    0.988659 - 1.132195: 3
{*  RMS difference with respect to the reference: 0.724248
""")
  energies_sites_no_gradients = ncs_operators.energies_sites(
    sites_cart=sites_cart, compute_gradients=False)
  assert energies_sites_no_gradients.number_of_restraints == 110
  assert eps_eq(energies_sites_no_gradients.residual_sum, 7014.03969257)
  assert eps_eq(energies_sites_no_gradients.target, 7014.03969257)
  assert energies_sites_no_gradients.gradients is None
  assert eps_eq(energies_sites_no_gradients.rms_with_respect_to_average,
    [0.41226641576521778, 0.38139080907663186,
     0.39748408968570492, 0.40001937328488651])
  energies_sites = ncs_operators.energies_sites(sites_cart=sites_cart)
  assert energies_sites_no_gradients.number_of_restraints \
      == energies_sites.number_of_restraints
  assert energies_sites_no_gradients.residual_sum \
      == energies_sites.residual_sum
  assert energies_sites_no_gradients.target \
      == energies_sites.target
  assert eps_eq(energies_sites.gradients.norm(), 3349.99455344)
  assert eps_eq(energies_sites.rms_with_respect_to_average,
   energies_sites_no_gradients.rms_with_respect_to_average)
  site_labels = [
    '"'+atom.pdb_label_columns()+'"' for atom in
      model.pdb_atoms]
  out = StringIO()
  energies_sites.show_distances_to_average(
    site_labels=site_labels, out=out, prefix="#^")
  assert not show_diff(out.getvalue(), """\
#^NCS selection: "chain A"
#^                     Distance to NCS average
#^  " N   GLN A   1 ":   0.4263
#^  " CA  GLN A   1 ":   0.2141
#^  " C   GLN A   1 ":   0.4052
...
#^  " CA  THR B   6 ":   0.4001
#^  " C   THR B   6 ":   0.6281
#^NCS selection: "chain C"
#^                     Distance to NCS average
#^  " N   GLN C   1 ":   0.4135
#^  " CA  GLN C   1 ":   0.5070
...
#^  " C   SER D   4 ":   0.6943
#^  " CA BSER D   4 ":   0.4444
#^  " N   THR D   6 ":   0.3724
#^  " CA  THR D   6 ":   0.4129
#^  " C   THR D   6 ":   0.4017
""", selections=[range(5),range(56,62),range(-5,0)])
  for ag,fg in zip(energies_sites.gradients,
                   finite_difference_site_gradients(
                     ncs_operators=ncs_operators,
                     sites_cart=sites_cart,
                     sites_average=energies_sites.sites_average)):
    assert eps_eq(ag, fg)
  #
  u_isos = model.get_xray_structure().extract_u_iso_or_u_equiv()
  eng = group.energies_adp_iso(
    u_isos=u_isos, average_power=1, compute_gradients=False)
  energies_adp_iso_no_gradients = eng
  assert eng.number_of_restraints == 110
  assert eps_eq(eng.residual_sum, 1.11021057745)
  assert eps_eq(eng.target, eng.residual_sum)
  assert eng.gradients is None
  assert eps_eq(eng.rms_with_respect_to_average,
    [3.8233537528289001, 4.4894247897900934,
     3.71150443476373, 4.0839849076232442])
  energies_adp_iso = group.energies_adp_iso(u_isos=u_isos, average_power=1)
  assert energies_adp_iso.number_of_restraints == eng.number_of_restraints
  assert eps_eq(energies_adp_iso.residual_sum, eng.residual_sum)
  assert eps_eq(energies_adp_iso.target, eng.target)
  assert eps_eq(energies_adp_iso.gradients.norm(), 4.50764745473)
  assert eps_eq(energies_adp_iso.rms_with_respect_to_average,
                eng.rms_with_respect_to_average)
  out = StringIO()
  energies_adp_iso.show_differences_to_average(
    site_labels=site_labels, out=out, prefix="Y$")
  assert not show_diff(out.getvalue().replace(" 7.66 = ", " 7.65 = "), """\
Y$NCS selection: "chain A"
Y$                       B-iso   NCS ave  Difference
Y$  " N   GLN A   1 ":   11.77 -   12.08 =  -0.3133
Y$  " CA  GLN A   1 ":    9.09 -    9.28 =  -0.1933
Y$  " C   GLN A   1 ":   15.40 -   12.05 =   3.3500
...
Y$  " CA  THR A   6 ":    3.98 -    7.65 =  -3.6750
Y$  " C   THR A   6 ":   14.64 -   10.99 =   3.6475
Y$NCS selection: "chain B"
Y$                       B-iso   NCS ave  Difference
Y$  " N   GLU B   2 ":   12.87 -   10.35 =   2.5225
...
Y$  " C   SER D   4 ":    7.29 -   10.45 =  -3.1575
Y$  " CA BSER D   4 ":    5.23 -    7.00 =  -1.7667
Y$  " N   THR D   6 ":    4.55 -    8.69 =  -4.1425
Y$  " CA  THR D   6 ":    8.78 -    7.65 =   1.1250
Y$  " C   THR D   6 ":   10.80 -   10.99 =  -0.1925
""", selections=[range(5),range(32,37),range(-5,0)])
  for average_power in [1, 0.69, 0.35]:
    finite_difference_gradients = flex.double()
    eps = 1.e-6
    for i_u_iso in xrange(u_isos.size()):
      rs = []
      for signed_eps in [eps, -eps]:
        u_isos_eps = u_isos.deep_copy()
        u_isos_eps[i_u_iso] += signed_eps
        energies = group.energies_adp_iso(
          u_isos=u_isos_eps,
          average_power=average_power,
          compute_gradients=False)
        rs.append(energies.residual_sum)
      finite_difference_gradients.append((rs[0]-rs[1])/(2*eps))
    energies_adp_iso = group.energies_adp_iso(
      u_isos=u_isos, average_power=average_power)
    assert eps_eq(energies_adp_iso.gradients, finite_difference_gradients)
  #
  groups = ncs.restraints.groups()
  groups.members.append(group)
  for coordinate_sigma,b_factor_weight in [
        (None,1.234),
        (0.1,None)]:
    groups.members.append(
      ncs.restraints.group.from_atom_selections(
        model=model,
        reference_selection_string=None,
        selection_strings=selection_strings,
        coordinate_sigma=coordinate_sigma,
        b_factor_weight=b_factor_weight,
        special_position_warnings_only=False))
  energies_adp_iso = groups.energies_adp_iso(u_isos=u_isos, average_power=1)
  assert energies_adp_iso.number_of_restraints == 220
  assert eps_eq(energies_adp_iso.residual_sum, 4.2807726061)
  assert eps_eq(energies_adp_iso.target, energies_adp_iso.residual_sum)
  assert eps_eq(energies_adp_iso.gradients.norm(), 17.3806790658)
  energies_adp_iso = groups.energies_adp_iso(
    u_isos=u_isos, average_power=1, normalization=True)
  assert energies_adp_iso.number_of_restraints == 220
  assert eps_eq(energies_adp_iso.residual_sum, 4.2807726061)
  assert eps_eq(energies_adp_iso.target, energies_adp_iso.residual_sum/220)
  assert eps_eq(energies_adp_iso.gradients.norm(), 17.3806790658/220)
  for rms in energies_adp_iso.rms_with_respect_to_averages:
    if (rms is not None):
      assert eps_eq(
        rms, energies_adp_iso_no_gradients.rms_with_respect_to_average)
  assert energies_adp_iso.rms_with_respect_to_averages[2] is None
  energies_sites = groups.energies_sites(sites_cart=sites_cart)
  assert energies_sites.number_of_restraints == 220
  assert eps_eq(energies_sites.residual_sum, 8767.54961571)
  assert eps_eq(energies_sites.target, energies_sites.residual_sum)
  assert eps_eq(energies_sites.gradients.norm(), 4187.49319181)
  energies_sites = groups.energies_sites(
    sites_cart=sites_cart, normalization=True)
  assert energies_sites.number_of_restraints == 220
  assert eps_eq(energies_sites.residual_sum, 8767.54961571)
  assert eps_eq(energies_sites.target, energies_sites.residual_sum/220)
  assert eps_eq(energies_sites.gradients.norm(), 4187.49319181/220)
  for rms in energies_sites.rms_with_respect_to_averages:
    if (rms is not None):
      assert eps_eq(
        rms, energies_sites_no_gradients.rms_with_respect_to_average)
  assert energies_sites.rms_with_respect_to_averages[1] is None
  out = StringIO()
  groups.show_adp_iso_differences_to_average(
    u_isos=u_isos, site_labels=site_labels, out=out, prefix="W@")
  assert not show_diff(out.getvalue(), """\
W@NCS restraint group 1:
W@  weight: 0.4321
W@  NCS selection: "chain A"
W@                         B-iso   NCS ave  Difference
W@    " N   GLN A   1 ":   11.77 -   12.08 =  -0.3133
...
W@    " C   THR D   6 ":   10.80 -   10.99 =  -0.1925
W@NCS restraint group 3:
W@  b_factor_weight: None  =>  restraints disabled
""", selections=[range(5),range(-3,0)])
  out = StringIO()
  groups.show_operators(sites_cart=sites_cart, out=out, prefix="K&")
  assert not show_diff(out.getvalue(), """\
K&NCS restraint group 1:
K&  NCS operator 1:
K&    Reference selection: "chain A"
...
K&      0.824624 - 0.971034: 5
K&    RMS difference with respect to the reference: 0.653687
K&  NCS operator 2:
K&    Reference selection: "chain A"
K&        Other selection: "chain C"
K&    Number of atom pairs: 32
...
K&      0.845124 - 0.988659: 2
K&      0.988659 - 1.132195: 3
K&    RMS difference with respect to the reference: 0.724248
""", selections=[range(3),range(15,21),range(-3,0)])
  out = StringIO()
  groups.show_sites_distances_to_average(
    sites_cart=sites_cart, site_labels=site_labels, out=out, prefix="[")
  assert not show_diff(out.getvalue(), """\
[NCS restraint group 1:
[  coordinate_sigma: 0.05
[  weight:  400
[  NCS selection: "chain A"
[                       Distance to NCS average
[    " N   GLN A   1 ":   0.4263
...
[    " C   THR D   6 ":   0.4017
[NCS restraint group 2:
[  coordinate_sigma: None  =>  restraints disabled
[NCS restraint group 3:
[  coordinate_sigma: 0.1
[  weight:  100
[  NCS selection: "chain A"
[                       Distance to NCS average
[    " N   GLN A   1 ":   0.4263
...
[    " C   THR D   6 ":   0.4017
""", selections=[range(6),range(120,129),range(-1,0)])
  #
  selection = groups.selection_restrained()
  assert selection.size() == 132
  assert selection.count(True) == 110
  out = StringIO()
  rm = model.get_restraints_manager()
  rm.ncs_groups = groups
  out = model.restraints_as_geo()
  # print out
  for i, l in enumerate(out.split('\n')):
    if l == "Atoms without NCS restraints:":
      break
  out_res = "\n".join(out.split('\n')[i:-1])
  assert not show_diff(out_res, """\
Atoms without NCS restraints:
MODEL        1
ATOM     20  N   ALA B   3     111.517   7.175  -8.669  1.00 10.73           N
ATOM     21  CA  ALA B   3     112.152   8.103  -8.026  1.00 16.28           C
ATOM     22  C   ALA B   3     111.702   8.243  -5.903  1.00  9.19           C
ATOM     24  CA  SER B   4     111.797   9.689  -4.364  1.00  8.91           C
TER
ATOM     38  N   ALA C   3     109.043  27.391  28.663  1.00 15.05           N
ATOM     39  CA  ALA C   3     109.073  26.531  28.433  1.00  3.14           C
ATOM     40  C   ALA C   3     108.930  26.867  26.637  1.00 15.22           C
TER
ATOM     57  N   ALA D   3      65.439   9.903 -12.471  1.00  7.59           N
ATOM     58  CA  ALA D   3      65.019   9.566 -11.201  1.00 15.62           C
ATOM     59  C   ALA D   3      65.679  11.045 -11.097  1.00  2.65           C
ATOM     61  CA CSER D   4      65.657  12.870  -8.333  1.00  5.84           C
TER
ENDMDL
MODEL        2
ATOM     86  N   ALA B   3     111.984   7.364  -8.288  1.00  3.16           N
ATOM     87  CA  ALA B   3     112.389   8.456  -7.544  1.00 12.73           C
ATOM     88  C   ALA B   3     111.615   8.267  -6.238  1.00  7.17           C
ATOM     90  CA  SER B   4     111.707   9.131  -4.317  1.00 13.83           C
TER
ATOM    104  N   ALA C   3     109.237  27.879  29.334  1.00  6.45           N
ATOM    105  CA  ALA C   3     109.043  26.399  28.156  1.00 15.21           C
ATOM    106  C   ALA C   3     108.983  26.760  27.178  1.00 11.73           C
TER
ATOM    123  N   ALA D   3      65.188  10.335 -12.959  1.00  6.94           N
ATOM    124  CA  ALA D   3      65.543   9.614 -11.242  1.00  5.18           C
ATOM    125  C   ALA D   3      65.064  11.402 -10.648  1.00 16.01           C
ATOM    127  CA CSER D   4      65.127  12.969  -9.218  1.00  9.03           C
TER
ENDMDL
""")
  #
  for group in groups.members:
    assert group.registry.number_of_additional_isolated_sites == 0
  groups.register_additional_isolated_sites(number=10)
  for group in groups.members:
    assert group.registry.number_of_additional_isolated_sites == 10
  groups.register_additional_isolated_sites(number=3)
  for group in groups.members:
    assert group.registry.number_of_additional_isolated_sites == 13