def _decompress_xz(filename): """Eumlates an option function in read mode for xz. See the comment in _compress_xz for more information. This function tries to emulate the lzma module as much as possible """ if not filename.endswith('.xz'): filename = '{}.xz'.format(filename) try: with open(os.devnull, 'w') as null: string = subprocess.check_output( ['xz', '--decompress', '--stdout', filename], stderr=null) except OSError as e: if e.errno == errno.ENOENT: raise exceptions.PiglitFatalError( 'No xz binary available') raise # We need a file-like object, so the contents must be placed in # a StringIO object. io = StringIO() io.write(string) io.seek(0) yield io io.close()
class IncludedResponse(object): def __init__(self): self.headers = None self.status = None self.output = StringIO() self.str = None def close(self): self.str = self.output.getvalue() self.output.close() self.output = None def write(self, s): assert self.output is not None, ( "This response has already been closed and no further data " "can be written.") self.output.write(s) def __str__(self): return self.body def body__get(self): if self.str is None: return self.output.getvalue() else: return self.str body = property(body__get)
def picture_view(request, user_id, year=None): """Displays a view of a user's picture. Args: user_id The ID of the user whose picture is being fetched. year The user's picture from this year is fetched. If not specified, use the preferred picture. """ try: user = User.get_user(id=user_id) except User.DoesNotExist: raise Http404 default_image_path = os.path.join(settings.PROJECT_ROOT, "static/img/default_profile_pic.png") if user is None: raise Http404 else: if year is None: preferred = user.preferred_photo if preferred is not None: if preferred.endswith("Photo"): preferred = preferred[:-len("Photo")] if preferred == "AUTO": data = user.default_photo() if data is None: image_buffer = io.open(default_image_path, mode="rb") else: image_buffer = StringIO(data) # Exclude 'graduate' from names array elif preferred in Grade.names: data = user.photo_binary(preferred) if data: image_buffer = StringIO(data) else: image_buffer = io.open(default_image_path, mode="rb") else: image_buffer = io.open(default_image_path, mode="rb") else: data = user.photo_binary(year) if data: image_buffer = StringIO(data) else: image_buffer = io.open(default_image_path, mode="rb") response = HttpResponse(content_type="image/jpeg") response["Content-Disposition"] = "filename={}_{}.jpg".format(user_id, year or preferred) try: img = image_buffer.read() except UnicodeDecodeError: img = io.open(default_image_path, mode="rb").read() image_buffer.close() response.write(img) return response
def __str__(self): buffer = StringIO() if self.refTime is not None: refTimeInSecs = self.refTime.getTime() / 1000 micros = (self.refTime.getTime() % 1000) * 1000 dtObj = datetime.datetime.utcfromtimestamp(refTimeInSecs) dtObj = dtObj.replace(microsecond=micros) buffer.write(dtObj.isoformat(' ')) if "FCST_USED" in self.utilityFlags: hrs = int(self.fcstTime / 3600) mins = int((self.fcstTime - (hrs * 3600)) / 60) buffer.write(" (" + str(hrs)) if mins != 0: buffer.write(":" + str(mins)) buffer.write(")") if "PERIOD_USED" in self.utilityFlags: buffer.write("[") buffer.write(self.validPeriod.start.isoformat(' ')) buffer.write("--") buffer.write(self.validPeriod.end.isoformat(' ')) buffer.write("]") strVal = buffer.getvalue() buffer.close() return strVal
def test_write_ini_with_custom_converters(self): def dict_encoder(dict_): return ','.join('%s:%s' % (k, v) for (k, v) in dict_.items()) def dict_decoder(string): return dict(x.split(':') for x in string.split(',')) n = Namespace(doc='top') n.add_option( 'a', default={'one': 'One'}, doc='the doc string', to_string_converter=dict_encoder, from_string_converter=dict_decoder, ) c = ConfigurationManager([n], use_admin_controls=True, use_auto_help=False, argv_source=[]) expected = "# the doc string\n#a=one:One\n" out = StringIO() c.write_conf(for_configobj, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() self.assertEqual(expected.strip(), received.strip())
def test_write_json(self): n = Namespace(doc='top') n.add_option( 'aaa', '2011-05-04T15:10:00', 'the a', short_form='a', from_string_converter=datetime_from_ISO_string ) c = ConfigurationManager( [n], use_admin_controls=True, use_auto_help=False, argv_source=[] ) out = StringIO() c.write_conf(for_json, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() jrec = json.loads(received) expect_to_find = { "short_form": "a", "default": "2011-05-04T15:10:00", "doc": "the a", "value": "2011-05-04T15:10:00", "from_string_converter": "configman.datetime_util.datetime_from_ISO_string", "name": "aaa" } for key, value in expect_to_find.items(): self.assertEqual(jrec['aaa'][key], value)
def test_write_json(self): n = Namespace(doc='top') n.add_option('aaa', '2011-05-04T15:10:00', 'the a', short_form='a', from_string_converter=datetime_from_ISO_string) c = ConfigurationManager([n], use_admin_controls=True, use_auto_help=False, argv_source=[]) out = StringIO() c.write_conf(for_json, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() jrec = json.loads(received) expect_to_find = { "short_form": "a", "default": "2011-05-04T15:10:00", "doc": "the a", "value": "2011-05-04T15:10:00", "from_string_converter": "configman.datetime_util.datetime_from_ISO_string", "name": "aaa" } for key, value in expect_to_find.items(): self.assertEqual(jrec['aaa'][key], value)
def test_write_ini_with_custom_converters(self): def dict_encoder(dict_): return ','.join('%s:%s' % (k, v) for (k, v) in dict_.items()) def dict_decoder(string): return dict(x.split(':') for x in string.split(',')) n = Namespace(doc='top') n.add_option( 'a', default={'one': 'One'}, doc='the doc string', to_string_converter=dict_encoder, from_string_converter=dict_decoder, ) c = ConfigurationManager( [n], use_admin_controls=True, use_auto_help=False, argv_source=[] ) expected = "# the doc string\n#a=one:One\n" out = StringIO() c.write_conf(for_configobj, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() self.assertEqual(expected.strip(), received.strip())
def test_for_mapping_long_doc_in_write_conf(self): n = self._some_namespaces() n = Namespace(doc='top') n.add_option( 'aaa', 'Default Value Goes In Here', 'This time the documentation string is really long. So long ' 'that we have to write it on multiple lines.', ) cm = ConfigurationManager( n, values_source_list=[], ) out = StringIO() cm.write_conf(for_mapping, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() for line in received.splitlines(): self.assertTrue(len(line) < 80, line) expected = """ # This time the documentation string is really long. So long that we have to # write it on multiple lines. (default: 'Default Value Goes In Here') aaa='Default Value Goes In Here' """.strip() if six.PY3: expected = expected.replace("='", "=b'") self.assertEqual(received.strip(), expected)
def __str__(self): buffer = StringIO() if self.refTime is not None: refTimeInSecs = self.refTime.getTime() / 1000 micros = (self.refTime.getTime() % 1000) * 1000 dtObj = datetime.datetime.utcfromtimestamp(refTimeInSecs) dtObj = dtObj.replace(microsecond=micros) # This won't be compatible with java or string from java since its to microsecond buffer.write(dtObj.isoformat(' ')) if "FCST_USED" in self.utilityFlags: hrs = int(self.fcstTime / 3600) mins = int((self.fcstTime - (hrs * 3600)) / 60) buffer.write(" (" + str(hrs)) if mins != 0: buffer.write(":" + str(mins)) buffer.write(")") if "PERIOD_USED" in self.utilityFlags: buffer.write("[") buffer.write(self.validPeriod.start.isoformat(' ')) buffer.write("--") buffer.write(self.validPeriod.end.isoformat(' ')) buffer.write("]") strVal = buffer.getvalue() buffer.close() return strVal
def scan(path): """ Performs an in-process binary module scan. That means the module is loaded (imported) into the current Python interpreter. "path" - a path to a binary module to scan Returns a CIX 2.0 XML string. """ from gencix.python import gencixcore as gencix name,_ = os.path.splitext(os.path.basename(path)) dir = os.path.dirname(path) root = gencix.Element('codeintel', version='2.0', name=name) gencix.docmodule(name, root, usefile=True, dir=dir) gencix.perform_smart_analysis(root) gencix.prettify(root) tree = gencix.ElementTree(root) stream = StringIO() try: stream.write('<?xml version="1.0" encoding="UTF-8"?>\n') tree.write(stream) return stream.getvalue() finally: stream.close()
def scan(path): """ Performs an in-process binary module scan. That means the module is loaded (imported) into the current Python interpreter. "path" - a path to a binary module to scan Returns a CIX 2.0 XML string. """ from gencix.python import gencixcore as gencix name, _ = os.path.splitext(os.path.basename(path)) dir = os.path.dirname(path) root = gencix.Element('codeintel', version='2.0', name=name) gencix.docmodule(name, root, usefile=True, dir=dir) gencix.perform_smart_analysis(root) gencix.prettify(root) tree = gencix.ElementTree(root) stream = StringIO() try: stream.write('<?xml version="1.0" encoding="UTF-8"?>\n') tree.write(stream) return stream.getvalue() finally: stream.close()
def run_pylint(): buff = StringIO() reporter = text.ParseableTextReporter(output=buff) args = ["--include-ids=y", "--errors-only", "rbd_iscsi_client"] lint.Run(args, reporter=reporter, exit=False) val = buff.getvalue() buff.close() return val
def run_pylint(): buff = StringIO() reporter = text.ParseableTextReporter(output=buff) args = ["--include-ids=y", "-E", "sahara"] lint.Run(args, reporter=reporter, exit=False) val = buff.getvalue() buff.close() return val
def run_pylint(): buff = StringIO() reporter = text.ParseableTextReporter(output=buff) args = ["-rn", "--disable=all", "--enable=" + enabled_codes ,"murano"] lint.Run(args, reporter=reporter, exit=False) val = buff.getvalue() buff.close() return val
def run_pylint(): buff = StringIO() reporter = text.ParseableTextReporter(output=buff) args = ["--include-ids=y", "-E", "savanna"] lint.Run(args, reporter=reporter, exit=False) val = buff.getvalue() buff.close() return val
def write_default_ansible_cfg(work_dir, remote_user, ssh_private_key=None, transport=None, base_ansible_cfg='/etc/ansible/ansible.cfg', override_ansible_cfg=None): ansible_config_path = os.path.join(work_dir, 'ansible.cfg') shutil.copy(base_ansible_cfg, ansible_config_path) config = configparser.ConfigParser() config.read(ansible_config_path) config.set('defaults', 'retry_files_enabled', 'False') log_path = os.path.join(work_dir, 'ansible.log') config.set('defaults', 'log_path', log_path) if os.path.exists(log_path): new_path = (log_path + '-' + datetime.now().strftime("%Y-%m-%dT%H:%M:%S")) os.rename(log_path, new_path) config.set('defaults', 'forks', '25') config.set('defaults', 'timeout', '30') config.set('defaults', 'gather_timeout', '30') # mistral user has no home dir set, so no place to save a known hosts file config.set( 'ssh_connection', 'ssh_args', '-o UserKnownHostsFile=/dev/null ' '-o StrictHostKeyChecking=no ' '-o ControlMaster=auto ' '-o ControlPersist=30m ' '-o ServerAliveInterval=5 ' '-o ServerAliveCountMax=5') config.set('ssh_connection', 'control_path_dir', os.path.join(work_dir, 'ansible-ssh')) config.set('ssh_connection', 'retries', '8') config.set('ssh_connection', 'pipelining', 'True') # Set connection info in config file so that subsequent/nested ansible # calls can re-use it if remote_user: config.set('defaults', 'remote_user', remote_user) if ssh_private_key: config.set('defaults', 'private_key_file', ssh_private_key) if transport: config.set('defaults', 'transport', transport) if override_ansible_cfg: sio_cfg = StringIO() sio_cfg.write(override_ansible_cfg) sio_cfg.seek(0) config.readfp(sio_cfg) sio_cfg.close() with open(ansible_config_path, 'w') as configfile: config.write(configfile) return ansible_config_path
def run_pylint(): buff = StringIO() reporter = text.ParseableTextReporter(output=buff) args = ["-rn", "--disable=all", "--enable=" + ",".join(ENABLED_CODES), "murano"] lint.Run(args, reporter=reporter, exit=False) val = buff.getvalue() buff.close() return val
def run_pylint(): buff = StringIO() args = ["--msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg}", "-E", "ceilometer"] lint.Run(args, exit=False) val = buff.getvalue() buff.close() return val
def run_pylint(): buff = StringIO() reporter = text.TextReporter(output=buff) args = [ "--msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}'", "-E", "cinderclient"] lint.Run(args, reporter=reporter, exit=False) val = buff.getvalue() buff.close() return val
def get_cix_string(cix, prettyFormat=True): # Get the CIX. if prettyFormat: prettify(cix) cixstream = StringIO() cixtree = ElementTree(cix) cixstream.write('<?xml version="1.0" encoding="UTF-8"?>\n') cixtree.write(cixstream) cixcontent = cixstream.getvalue() cixstream.close() return cixcontent
def load_conf(xfile, options=None, parse_conf_func=None, load_creds=False, envvar=None, custom_file=None): signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) conf = {'_config_directory': None} if os.path.exists(xfile): with open(xfile, 'r') as f: conf = helpers.load_yaml(f) config_directory = os.path.dirname(os.path.abspath(xfile)) conf['_config_directory'] = config_directory if custom_file: conf = helpers.merge( helpers.load_conf_yaml_file(custom_file, config_directory), conf) conf['_config_directory'] = config_directory elif envvar and os.environ.get(envvar): c = StringIO(os.environ[envvar]) conf = helpers.load_yaml(c.getvalue()) c.close() conf['_config_directory'] = None if parse_conf_func: conf = parse_conf_func(conf) else: conf = parse_conf(conf, load_creds) for x in ('modules', 'plugins'): conf = import_conf_files(x, conf) init_modules(conf) init_plugins(conf) if _INOTIFY: init_inotify(conf) if not options or not isinstance(options, object): return conf for def_option in iterkeys(get_default_options()): if getattr(options, def_option, None) is None \ and def_option in conf['general']: setattr(options, def_option, conf['general'][def_option]) setattr(options, 'configuration', conf) return options
def _format_stack(frame): """ Pretty-print the stack of `frame` like logging would. """ sio = StringIO() sio.write("Stack (most recent call last):\n") traceback.print_stack(frame, file=sio) sinfo = sio.getvalue() if sinfo[-1] == "\n": sinfo = sinfo[:-1] sio.close() return sinfo
def _format_exception(exc_info): """ Prettyprint an `exc_info` tuple. Shamelessly stolen from stdlib's logging module. """ sio = StringIO() traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], None, sio) s = sio.getvalue() sio.close() if s[-1:] == "\n": s = s[:-1] return s
def vmd_visualize_script(pdb_filename, trajectory_filename, topology, cplx_filename, coloring, step): """Function that creates the string in a syntax, that vmd can interpret. This string can be saved to a text/data file. Includes a comment-header. The very last printed lines prevent vmd from automatically creating an additional Representation. Parameters: ----------- pdb_filename: String Containing name of pdb-file (parsed argument) trajectory_filename: String Containing name of trajectory file (parsed argument) topology: dictionary yaml-dictionary from cplx cplx_filename: String Containing name of cplx-file (parsed argument) coloring: String (parsed argument) for choosing coloring scheme Returns: -------- String the complete string to be written to data-file """ trajectory_type = os.path.splitext(trajectory_filename)[-1][1:] out = StringIO() out.write( HEADER_FORMAT.format( generation_time=datetime.datetime.time(datetime.datetime.now()), topology_inputfile=cplx_filename, pdb_inputfile=pdb_filename, trajectory_inputfile=trajectory_filename, )) out.write(FUNCTIONS) out.write(LOAD_VMD_TOP_FORMAT.format(vmd_top=pdb_filename)) out.write( LOADTRAJ_FORMAT.format(traj_file=trajectory_filename, traj_type=trajectory_type, step=step)) out.write( "#adding representation to molecule 0 \n #################################### \n" ) out.write(vmd_rep_gen(topology, coloring)) out.write(SHOW_SIM_BOX) rep = out.getvalue() out.close() return rep
def dumps(root): """Convert obj into an xml string. """ if not isinstance(root, ET.Element): raise ValueError document = ET.ElementTree(root) flow = StringIO() document.write(flow, encoding='utf-8', xml_declaration=True) contents = flow.getvalue() flow.close() return minidom.parseString(contents).toprettyxml(indent=" ")
def vmd_rep_gen(topology, coloring): """Goes through the given topology and creates representations from that. The domains will be in a certain coloring scheme (specified by COLORING input-param). Different domain types will be shown in different rep_types by vmd. Parameter: ---------- topology : dictionary Dictionary from the cplx read by yaml coloring : string random or domain. Parsed as input argument for choosing color-scheme Returns: -------- string The string containing all the interpretable stuff for vmd to create the representations (also HEADER) """ out = StringIO() bead_counter = 1 rep_counter = 0 name2type = get_type_translater(topology["definitions"]) for j, top in enumerate(topology["topologies"]): domains = top["domains"] for i, dom in six.iteritems(domains): chainids = dom["chain-ids"] rep_number = rep_counter first_bead = bead_counter last_bead = first_bead + dom["nbeads"] bead_counter = last_bead + 1 dom_rep, rep_counter = vmd_domain_rep( name2type[dom["type"]], chainids, coloring, rep_counter, rep_number, i, dom, ) out.write(dom_rep) # The Following has to be the LAST included instruction in the file !! # Because otherwise vmd will automatically generate an additional # representation including all atoms, displaying them as lines. out.write(DEL_REP_FORMAT.format(rep_counter=rep_counter)) concatenated_string = out.getvalue() out.close() return concatenated_string
class LoggingFramework: def __init__(self): self.k = StringIO() self.current_out = sys.stdout self.current_err = sys.stderr sys.stdout = self.k sys.stderr = self.k def __del__(self): sys.stdout = self.current_out sys.stderr = self.current_err self.k.flush() self.k.close() def getvalue(self): return self.k.getvalue()
class _FileWrapper(object): def set_file(self, filename): self.f = open(filename, 'r') def set_stdin(self): self.f = sys.stdin def set_text(self, text): from six.moves import cStringIO as StringIO self.f = StringIO(text) def readline(self): return self.f.readline() def close(self): self.f.close()
def old(self, no_reduce_db): try: if no_reduce_db: touch('./dummydb') fileobj = StringIO() self.write_pdb(fileobj) fileobj.seek(0) reduce = os.path.join(os.getenv('LIBTBX_BUILD'), 'reduce', 'exe', 'reduce') if not os.path.exists(reduce): reduce = 'phenix.reduce' cmd = [reduce, '-BUILD', '-NUC', '-NOFLIP', '-DB ./dummydb', '-'] if no_reduce_db: process = subprocess.Popen([ reduce, '-BUILD', '-NUC', '-NOFLIP', '-DB ./dummydb', '-' ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: process = subprocess.Popen( [reduce, '-BUILD', '-NUC', '-NOFLIP', '-'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate(str.encode(fileobj.read())) out = out.decode() err = err.decode() if process.wait(): logger.error("REDUCE returned non-zero exit status: " "See reduce_info.log for more details") # print out the reduce log even if it worked with open('reduce_info.log', 'w') as fh: fh.write(err) pdbh = StringIO(out) # not using load_file since it does not read StringIO print('-' * 80) print(pdbh) print('-' * 80) self.parm = parmed.read_PDB(pdbh) finally: fileobj.close() if no_reduce_db: os.unlink('./dummydb') return self
def _get_config(self, section, progpath, parse_vers, check_version=True): my_vers = False if check_version: my_vers = parse_vers( subprocess.check_output((progpath, '--version')).strip()) myconf = StringIO("%s\n%s" % (self._read_file( self._default_file), self._read_file(self._custom_file))) self._myconf.readfp(myconf) myconf.close() if my_vers: self._check_conf_versions(self._myconf, section, my_vers.group('version'), 'ver-') self._check_conf_versions(self._myconf, section, my_vers.group('distrib'), 'dist-') return self._myconf
def test_write_ini(self): n = self._some_namespaces() c = ConfigurationManager( [n], [{ "c.fred": "just like George Jetson", }], use_admin_controls=True, #use_config_files=False, use_auto_help=False, argv_source=[] ) expected = """# the a #aaa=2011-05-04T15:10:00 [c] # husband from Flintstones fred=just like George Jetson # wife from Flintstones #wilma=waspish's [d] # female neighbor from I Love Lucy #ethel=silly # male neighbor from I Love Lucy #fred=crabby [x] # the password #password=secret "message" # how big in tons #size=100 """ out = StringIO() c.write_conf(for_configobj, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() self.assertEqual(expected.strip(), received.strip())
def test_write_ini(self): n = self._some_namespaces() c = ConfigurationManager( [n], [{ "c.fred": "just like George Jetson", }], use_admin_controls=True, #use_config_files=False, use_auto_help=False, argv_source=[]) expected = """# the a #aaa=2011-05-04T15:10:00 [c] # husband from Flintstones fred=just like George Jetson # wife from Flintstones #wilma=waspish's [d] # female neighbor from I Love Lucy #ethel=silly # male neighbor from I Love Lucy #fred=crabby [x] # the password #password=secret "message" # how big in tons #size=100 """ out = StringIO() c.write_conf(for_configobj, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() self.assertEqual(expected.strip(), received.strip())
def formatException(self, exc_info, record=None): """Format exception output with CONF.logging_exception_prefix.""" if not record: return logging.Formatter.formatException(self, exc_info) stringbuffer = StringIO() traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], None, stringbuffer) lines = stringbuffer.getvalue().split('\n') stringbuffer.close() if CONF.logging_exception_prefix.find('%(asctime)') != -1: record.asctime = self.formatTime(record, self.datefmt) formatted_lines = [] for line in lines: pl = CONF.logging_exception_prefix % record.__dict__ fl = '%s%s' % (pl, line) formatted_lines.append(fl) return '\n'.join(formatted_lines)
def test_for_mapping_nested_namespaces(self): n = self._some_namespaces() cm = ConfigurationManager( n, values_source_list=[], ) out = StringIO() cm.write_conf(for_mapping, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() expected = """ # the a (default: '2011-05-04T15:10:00') aaa='2011-05-04T15:10:00' # your uncle (default: 98) c__dwight='98' # husband from Flintstones (default: 'stupid') c__fred='stupid' # wife from Flintstones (default: 'waspish') c__wilma='waspish' # my uncle (default: 97) c__e__dwight='97' # female neighbor from I Love Lucy (default: 'silly') d__ethel='silly' # male neighbor from I Love Lucy (default: 'crabby') d__fred='crabby' # the password (default: 'secret') x__password='******' # how big in tons (default: 100) x__size='100' """.strip() if six.PY3: expected = expected.replace("='", "=b'") self.assertEqual(received.strip(), expected)
def load_conf(xfile, options=None, envvar=None): signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) conf = {'_config_directory': None} if os.path.exists(xfile): with open(xfile, 'r') as f: conf = parse_conf(load_yaml(f)) conf['_config_directory'] = os.path.dirname(os.path.abspath(xfile)) elif envvar and os.environ.get(envvar): c = StringIO(os.environ[envvar]) conf = parse_conf(load_yaml(c.getvalue())) c.close() conf['_config_directory'] = None for x in ('modules', 'plugins'): conf = import_conf_files(x, conf) init_modules(conf) init_plugins(conf) if not conf.get('dns'): conf['dns'] = {} if conf['dns'] and conf['dns'].get('domains'): for name, domain_cfg in viewitems(conf['dns']['domains']): cfg = {'rrsets': [], 'vars': {}} for x in ('vars', 'rrsets'): if isinstance(cfg[x], list): append_func = getattr(cfg[x], 'extend') else: append_func = getattr(cfg[x], 'update') if domain_cfg.get("import_%s" % x): append_func( import_file(domain_cfg["import_%s" % x], conf['_config_directory'], cfg)) if x in domain_cfg: append_func(domain_cfg[x]) if not cfg['rrsets']: cfg = None if cfg: conf['dns']['domains'][name] = cfg if not options or not isinstance(options, object): return conf for def_option in iterkeys(get_default_options()): if getattr(options, def_option, None) is None \ and def_option in conf['general']: setattr(options, def_option, conf['general'][def_option]) setattr(options, 'configuration', conf) return options
class TextDumper(Communicator): """Class to dump data. """ def __init__(self, fname=None): Communicator.__init__(self, self) self._fname = fname self._file = None self.open(fname) def open(self, fname=None): """Open a new file for writing. """ self.close() self._fname = fname if fname is None: self._file = StringIO() else: self._file = open(fname, "w") def close(self): """Transmit the objects. """ val = None if self._file is not None: Communicator.close(self) if self._fname is None: val = self._file.getvalue() self._file.close() self._file = None self._fname = None return val def info(self, obj): """Write the object info""" kind = xc_type(obj) self._file.write("%c " % kind) if kind in ['C', 'I', 'N', 'R']: self._file.write('%d ' % ctypes.sizeof(c_type(obj))) elif type(obj) is XCStruct: self._file.write('%s ' % obj.name()) else: self._file.write('%s ' % type(obj).__name__) def data(self, obj, total=None): """Write the objects data. """ if total is not None: tmp = '%s ' % CHAR_MAP.get(xc_type(obj[0]), None) if type(obj[0]) in XC_TYPES: for i in xrange(total): self._file.write(tmp % obj[i].value) elif type(obj[0]) in CTYPES: for i in xrange(total): self._file.write(tmp % obj[i]) else: for i in xrange(total): data(self, obj[i], "") return tmp = '%s ' % CHAR_MAP[xc_type(obj)] if type(obj) not in XC_TYPES: self._file.write(tmp % obj) else: self._file.write(tmp % obj.value) def trans_type(self, obj): """Transfer a character. """ self._file.write("%c " % obj) def trans_byte(self, obj): """Transfer a byte. """ self._file.write("%d " % obj) def trans_num(self, num): """Transfer an integer. """ self._file.write("%d " % num) def trans_varname(self, varname): """Transfer a variable name. """ self._file.write("%s " % varname) def trans_name(self, name): """Transfer a name. """ self._file.write("%s " % name) def trans_num_objects(self): """Transfer the number of objects. """ self._file.write("%d\n" % len(self.obj)) def trans_num_classes(self): """Tranfer the number of objects in cls_obj. """ if len(self.cls_obj) > 0: self._file.write("%d\n" % len(self.cls_obj)) else: self._file.write("%d " % len(self.cls_obj)) def trans_close(self): """Print character to denote when transfer of obj is done. """ self._file.write("\n")
class PythonWidget(HistoryConsoleWidget): """ A basic in-process Python interpreter. """ # Emitted when a command has been executed in the interpeter. executed = QtCore.Signal() #-------------------------------------------------------------------------- # 'object' interface #-------------------------------------------------------------------------- def __init__(self, parent=None): super(PythonWidget, self).__init__(parent) # PythonWidget attributes. self.locals = dict(__name__='__console__', __doc__=None) self.interpreter = InteractiveInterpreter(self.locals) # PythonWidget protected attributes. self._buffer = StringIO() self._bracket_matcher = BracketMatcher(self._control) self._call_tip_widget = CallTipWidget(self._control) self._completion_lexer = CompletionLexer(PythonLexer()) self._hidden = False self._highlighter = PythonWidgetHighlighter(self) self._last_refresh_time = 0 # file-like object attributes. self.encoding = sys.stdin.encoding # Configure the ConsoleWidget. self.tab_width = 4 self._set_continuation_prompt('... ') # Configure the CallTipWidget. self._call_tip_widget.setFont(self.font) self.font_changed.connect(self._call_tip_widget.setFont) # Connect signal handlers. document = self._control.document() document.contentsChange.connect(self._document_contents_change) # Display the banner and initial prompt. self.reset() #-------------------------------------------------------------------------- # file-like object interface #-------------------------------------------------------------------------- def flush(self): """ Flush the buffer by writing its contents to the screen. """ self._buffer.seek(0) text = self._buffer.getvalue() self._buffer.close() self._buffer = StringIO() self._append_plain_text(text) self._control.moveCursor(QtGui.QTextCursor.End) def readline(self, prompt=None): """ Read and return one line of input from the user. """ return self._readline(prompt) def write(self, text, refresh=True): """ Write text to the buffer, possibly flushing it if 'refresh' is set. """ if not self._hidden: self._buffer.write(text) if refresh: current_time = time() if current_time - self._last_refresh_time > 0.05: self.flush() self._last_refresh_time = current_time def writelines(self, lines, refresh=True): """ Write a list of lines to the buffer. """ for line in lines: self.write(line, refresh=refresh) #--------------------------------------------------------------------------- # 'ConsoleWidget' abstract interface #--------------------------------------------------------------------------- def _is_complete(self, source, interactive): """ Returns whether 'source' can be completely processed and a new prompt created. When triggered by an Enter/Return key press, 'interactive' is True; otherwise, it is False. """ if interactive: lines = source.splitlines() if len(lines) == 1: try: return compile_command(source) is not None except: # We'll let the interpeter handle the error. return True else: return lines[-1].strip() == '' else: return True def _execute(self, source, hidden): """ Execute 'source'. If 'hidden', do not show any output. See parent class :meth:`execute` docstring for full details. """ # Save the current std* and point them here old_stdin = sys.stdin old_stdout = sys.stdout old_stderr = sys.stderr sys.stdin = sys.stdout = sys.stderr = self # Run the source code in the interpeter self._hidden = hidden try: more = self.interpreter.runsource(source) finally: self._hidden = False # Restore std* unless the executed changed them if sys.stdin is self: sys.stdin = old_stdin if sys.stdout is self: sys.stdout = old_stdout if sys.stderr is self: sys.stderr = old_stderr self.executed.emit() self._show_interpreter_prompt() def _prompt_started_hook(self): """ Called immediately after a new prompt is displayed. """ if not self._reading: self._highlighter.highlighting_on = True def _prompt_finished_hook(self): """ Called immediately after a prompt is finished, i.e. when some input will be processed and a new prompt displayed. """ if not self._reading: self._highlighter.highlighting_on = False def _tab_pressed(self): """ Called when the tab key is pressed. Returns whether to continue processing the event. """ # Perform tab completion if: # 1) The cursor is in the input buffer. # 2) There is a non-whitespace character before the cursor. text = self._get_input_buffer_cursor_line() if text is None: return False complete = bool(text[:self._get_input_buffer_cursor_column()].strip()) if complete: self._complete() return not complete #--------------------------------------------------------------------------- # 'ConsoleWidget' protected interface #--------------------------------------------------------------------------- def _event_filter_console_keypress(self, event): """ Reimplemented for smart backspace. """ if event.key() == QtCore.Qt.Key_Backspace and \ not event.modifiers() & QtCore.Qt.AltModifier: # Smart backspace: remove four characters in one backspace if: # 1) everything left of the cursor is whitespace # 2) the four characters immediately left of the cursor are spaces col = self._get_input_buffer_cursor_column() cursor = self._control.textCursor() if col > 3 and not cursor.hasSelection(): text = self._get_input_buffer_cursor_line()[:col] if text.endswith(' ') and not text.strip(): cursor.movePosition(QtGui.QTextCursor.Left, QtGui.QTextCursor.KeepAnchor, 4) cursor.removeSelectedText() return True return super(PythonWidget, self)._event_filter_console_keypress(event) def _insert_continuation_prompt(self, cursor): """ Reimplemented for auto-indentation. """ super(PythonWidget, self)._insert_continuation_prompt(cursor) source = self.input_buffer space = 0 for c in source.splitlines()[-1]: if c == '\t': space += 4 elif c == ' ': space += 1 else: break if source.rstrip().endswith(':'): space += 4 cursor.insertText(' ' * space) #--------------------------------------------------------------------------- # 'PythonWidget' public interface #--------------------------------------------------------------------------- def execute_file(self, path, hidden=False): """ Attempts to execute file with 'path'. If 'hidden', no output is shown. """ self.execute("exec(open(%s).read())" % repr(path), hidden=hidden) def reset(self): """ Resets the widget to its initial state. Similar to ``clear``, but also re-writes the banner. """ self._reading = False self._highlighter.highlighting_on = False self._control.clear() self._append_plain_text(self._get_banner()) self._show_interpreter_prompt() #--------------------------------------------------------------------------- # 'PythonWidget' protected interface #--------------------------------------------------------------------------- def _call_tip(self): """ Shows a call tip, if appropriate, at the current cursor location. """ # Decide if it makes sense to show a call tip cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.Left) if cursor.document().characterAt(cursor.position()) != '(': return False context = self._get_context(cursor) if not context: return False # Look up the context and show a tip for it symbol, leftover = self._get_symbol_from_context(context) doc = getattr(symbol, '__doc__', None) if doc is not None and not leftover: self._call_tip_widget.show_call_info(doc=doc) return True return False def _complete(self): """ Performs completion at the current cursor location. """ context = self._get_context() if context: symbol, leftover = self._get_symbol_from_context(context) if len(leftover) == 1: leftover = leftover[0] if symbol is None: names = list(self.interpreter.locals.keys()) names += list(six.moves.builtins.__dict__.keys()) else: names = dir(symbol) completions = [ n for n in names if n.startswith(leftover) ] if completions: cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.Left, n=len(context[-1])) self._complete_with_items(cursor, completions) def _get_banner(self): """ Gets a banner to display at the beginning of a session. """ banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \ '"license" for more information.' return banner % (sys.version, sys.platform) def _get_context(self, cursor=None): """ Gets the context for the specified cursor (or the current cursor if none is specified). """ if cursor is None: cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.StartOfBlock, QtGui.QTextCursor.KeepAnchor) text = cursor.selection().toPlainText() return self._completion_lexer.get_context(text) def _get_symbol_from_context(self, context): """ Find a python object in the interpeter namespace from a context (a list of names). """ context = map(str, context) if len(context) == 0: return None, context base_symbol_string = context[0] symbol = self.interpreter.locals.get(base_symbol_string, None) if symbol is None: symbol = six.moves.builtins.__dict__.get(base_symbol_string, None) if symbol is None: return None, context context = context[1:] for i, name in enumerate(context): new_symbol = getattr(symbol, name, None) if new_symbol is None: return symbol, context[i:] else: symbol = new_symbol return symbol, [] def _show_interpreter_prompt(self): """ Shows a prompt for the interpreter. """ self.flush() self._show_prompt('>>> ') #------ Signal handlers ---------------------------------------------------- def _document_contents_change(self, position, removed, added): """ Called whenever the document's content changes. Display a call tip if appropriate. """ # Calculate where the cursor should be *after* the change: position += added document = self._control.document() if position == self._get_cursor().position(): self._call_tip()
def test_write_ini_with_reference_value_froms( self ): n = self._some_namespaces() n.namespace('x1') n.x1.add_option( 'password', default='secret "message"', doc='the password', likely_to_be_changed=True, reference_value_from='xxx.yyy' ) n.namespace('x2') n.x2.add_option( 'password', default='secret "message"', doc='the password', reference_value_from='xxx.yyy' ) external_values = { 'xxx': { 'yyy': { 'password': '******' } }, } c = ConfigurationManager( [n], values_source_list=[external_values], use_admin_controls=True, use_auto_help=False, argv_source=[] ) expected = ("""# the a #aaa=2011-05-04T15:10:00 [xxx] #+include ./common_xxx.ini [[yyy]] #+include ./common_yyy.ini # the password password=dwight and wilma [c] # husband from Flintstones #fred="stupid, deadly" # wife from Flintstones #wilma=waspish's [d] # female neighbor from I Love Lucy #ethel=silly # male neighbor from I Love Lucy #fred=crabby [x] # the password #password=secret "message" # how big in tons #size=100 [x1] # the password # see "xxx.yyy.password" for the default or override it here password=dwight and wilma [x2] # the password # see "xxx.yyy.password" for the default or override it here password=dwight and wilma """) out = StringIO() c.write_conf(for_configobj, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() self.assertEqual(expected.strip(), received.strip())
class PythonWidget(HistoryConsoleWidget): """ A basic in-process Python interpreter. """ # Emitted when a command has been executed in the interpeter. executed = QtCore.Signal() #-------------------------------------------------------------------------- # 'object' interface #-------------------------------------------------------------------------- def __init__(self, parent=None): super(PythonWidget, self).__init__(parent) # PythonWidget attributes. self.locals = dict(__name__='__console__', __doc__=None) self.interpreter = InteractiveInterpreter(self.locals) # PythonWidget protected attributes. self._buffer = StringIO() self._bracket_matcher = BracketMatcher(self._control) self._call_tip_widget = CallTipWidget(self._control) self._completion_lexer = CompletionLexer(PythonLexer()) self._hidden = False self._highlighter = PythonWidgetHighlighter(self) self._last_refresh_time = 0 # file-like object attributes. self.encoding = sys.stdin.encoding # Configure the ConsoleWidget. self.tab_width = 4 self._set_continuation_prompt('... ') # Configure the CallTipWidget. self._call_tip_widget.setFont(self.font) self.font_changed.connect(self._call_tip_widget.setFont) # Connect signal handlers. document = self._control.document() document.contentsChange.connect(self._document_contents_change) # Display the banner and initial prompt. self.reset() #-------------------------------------------------------------------------- # file-like object interface #-------------------------------------------------------------------------- def flush(self): """ Flush the buffer by writing its contents to the screen. """ self._buffer.seek(0) text = self._buffer.getvalue() self._buffer.close() self._buffer = StringIO() self._append_plain_text(text) self._control.moveCursor(QtGui.QTextCursor.End) def readline(self, prompt=None): """ Read and return one line of input from the user. """ return self._readline(prompt) def write(self, text, refresh=True): """ Write text to the buffer, possibly flushing it if 'refresh' is set. """ if not self._hidden: self._buffer.write(text) if refresh: current_time = time() if current_time - self._last_refresh_time > 0.05: self.flush() self._last_refresh_time = current_time def writelines(self, lines, refresh=True): """ Write a list of lines to the buffer. """ for line in lines: self.write(line, refresh=refresh) #--------------------------------------------------------------------------- # 'ConsoleWidget' abstract interface #--------------------------------------------------------------------------- def _is_complete(self, source, interactive): """ Returns whether 'source' can be completely processed and a new prompt created. When triggered by an Enter/Return key press, 'interactive' is True; otherwise, it is False. """ if interactive: lines = source.splitlines() if len(lines) == 1: try: return compile_command(source) is not None except: # We'll let the interpeter handle the error. return True else: return lines[-1].strip() == '' else: return True def _execute(self, source, hidden): """ Execute 'source'. If 'hidden', do not show any output. See parent class :meth:`execute` docstring for full details. """ # Save the current std* and point them here old_stdin = sys.stdin old_stdout = sys.stdout old_stderr = sys.stderr sys.stdin = sys.stdout = sys.stderr = self # Run the source code in the interpeter self._hidden = hidden try: more = self.interpreter.runsource(source) finally: self._hidden = False # Restore std* unless the executed changed them if sys.stdin is self: sys.stdin = old_stdin if sys.stdout is self: sys.stdout = old_stdout if sys.stderr is self: sys.stderr = old_stderr self.executed.emit() self._show_interpreter_prompt() def _prompt_started_hook(self): """ Called immediately after a new prompt is displayed. """ if not self._reading: self._highlighter.highlighting_on = True def _prompt_finished_hook(self): """ Called immediately after a prompt is finished, i.e. when some input will be processed and a new prompt displayed. """ if not self._reading: self._highlighter.highlighting_on = False def _tab_pressed(self): """ Called when the tab key is pressed. Returns whether to continue processing the event. """ # Perform tab completion if: # 1) The cursor is in the input buffer. # 2) There is a non-whitespace character before the cursor. text = self._get_input_buffer_cursor_line() if text is None: return False complete = bool(text[:self._get_input_buffer_cursor_column()].strip()) if complete: self._complete() return not complete #--------------------------------------------------------------------------- # 'ConsoleWidget' protected interface #--------------------------------------------------------------------------- def _event_filter_console_keypress(self, event): """ Reimplemented for smart backspace. """ if event.key() == QtCore.Qt.Key_Backspace and \ not event.modifiers() & QtCore.Qt.AltModifier: # Smart backspace: remove four characters in one backspace if: # 1) everything left of the cursor is whitespace # 2) the four characters immediately left of the cursor are spaces col = self._get_input_buffer_cursor_column() cursor = self._control.textCursor() if col > 3 and not cursor.hasSelection(): text = self._get_input_buffer_cursor_line()[:col] if text.endswith(' ') and not text.strip(): cursor.movePosition(QtGui.QTextCursor.Left, QtGui.QTextCursor.KeepAnchor, 4) cursor.removeSelectedText() return True return super(PythonWidget, self)._event_filter_console_keypress(event) def _insert_continuation_prompt(self, cursor): """ Reimplemented for auto-indentation. """ super(PythonWidget, self)._insert_continuation_prompt(cursor) source = self.input_buffer space = 0 for c in source.splitlines()[-1]: if c == '\t': space += 4 elif c == ' ': space += 1 else: break if source.rstrip().endswith(':'): space += 4 cursor.insertText(' ' * space) #--------------------------------------------------------------------------- # 'PythonWidget' public interface #--------------------------------------------------------------------------- def execute_file(self, path, hidden=False): """ Attempts to execute file with 'path'. If 'hidden', no output is shown. """ self.execute("exec(open(%s).read())" % repr(path), hidden=hidden) def reset(self): """ Resets the widget to its initial state. Similar to ``clear``, but also re-writes the banner. """ self._reading = False self._highlighter.highlighting_on = False self._control.clear() self._append_plain_text(self._get_banner()) self._show_interpreter_prompt() #--------------------------------------------------------------------------- # 'PythonWidget' protected interface #--------------------------------------------------------------------------- def _call_tip(self): """ Shows a call tip, if appropriate, at the current cursor location. """ # Decide if it makes sense to show a call tip cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.Left) if cursor.document().characterAt(cursor.position()) != '(': return False context = self._get_context(cursor) if not context: return False # Look up the context and show a tip for it symbol, leftover = self._get_symbol_from_context(context) doc = getattr(symbol, '__doc__', None) if doc is not None and not leftover: self._call_tip_widget.show_call_info(doc=doc) return True return False def _complete(self): """ Performs completion at the current cursor location. """ context = self._get_context() if context: symbol, leftover = self._get_symbol_from_context(context) if len(leftover) == 1: leftover = leftover[0] if symbol is None: names = list(self.interpreter.locals.keys()) names += list(six.moves.builtins.__dict__.keys()) else: names = dir(symbol) completions = [n for n in names if n.startswith(leftover)] if completions: cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.Left, n=len(context[-1])) self._complete_with_items(cursor, completions) def _get_banner(self): """ Gets a banner to display at the beginning of a session. """ banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \ '"license" for more information.' return banner % (sys.version, sys.platform) def _get_context(self, cursor=None): """ Gets the context for the specified cursor (or the current cursor if none is specified). """ if cursor is None: cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.StartOfBlock, QtGui.QTextCursor.KeepAnchor) text = cursor.selection().toPlainText() return self._completion_lexer.get_context(text) def _get_symbol_from_context(self, context): """ Find a python object in the interpeter namespace from a context (a list of names). """ context = map(str, context) if len(context) == 0: return None, context base_symbol_string = context[0] symbol = self.interpreter.locals.get(base_symbol_string, None) if symbol is None: symbol = six.moves.builtins.__dict__.get(base_symbol_string, None) if symbol is None: return None, context context = context[1:] for i, name in enumerate(context): new_symbol = getattr(symbol, name, None) if new_symbol is None: return symbol, context[i:] else: symbol = new_symbol return symbol, [] def _show_interpreter_prompt(self): """ Shows a prompt for the interpreter. """ self.flush() self._show_prompt('>>> ') #------ Signal handlers ---------------------------------------------------- def _document_contents_change(self, position, removed, added): """ Called whenever the document's content changes. Display a call tip if appropriate. """ # Calculate where the cursor should be *after* the change: position += added document = self._control.document() if position == self._get_cursor().position(): self._call_tip()
def test_write_ini_with_reference_value_froms(self): n = self._some_namespaces() n.namespace('x1') n.x1.add_option('password', default='secret "message"', doc='the password', likely_to_be_changed=True, reference_value_from='xxx.yyy') n.namespace('x2') n.x2.add_option('password', default='secret "message"', doc='the password', reference_value_from='xxx.yyy') external_values = { 'xxx': { 'yyy': { 'password': '******' } }, } c = ConfigurationManager([n], values_source_list=[external_values], use_admin_controls=True, use_auto_help=False, argv_source=[]) expected = ("""# the a #aaa=2011-05-04T15:10:00 [xxx] #+include ./common_xxx.ini [[yyy]] #+include ./common_yyy.ini # the password password=dwight and wilma [c] # husband from Flintstones #fred="stupid, deadly" # wife from Flintstones #wilma=waspish's [d] # female neighbor from I Love Lucy #ethel=silly # male neighbor from I Love Lucy #fred=crabby [x] # the password #password=secret "message" # how big in tons #size=100 [x1] # the password # see "xxx.yyy.password" for the default or override it here password=dwight and wilma [x2] # the password # see "xxx.yyy.password" for the default or override it here password=dwight and wilma """) out = StringIO() c.write_conf(for_configobj, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() self.assertEqual(expected.strip(), received.strip())
def donttest_for_conf_nested_namespaces(self): n = self._some_namespaces() cm = ConfigurationManager( n, values_source_list=[], ) out = StringIO() cm.write_conf(for_conf, opener=stringIO_context_wrapper(out)) received = out.getvalue() out.close() expected = """# name: aaa # doc: the a aaa=2011-05-04T15:10:00 #------------------------------------------------------------------------------ # c - c space # name: c.dwight # doc: your uncle c.dwight=98 # name: c.fred # doc: husband from Flintstones c.fred=stupid # name: c.wilma # doc: wife from Flintstones c.wilma=waspish #------------------------------------------------------------------------------ # e - e space # name: c.e.dwight # doc: my uncle c.e.dwight=97 #------------------------------------------------------------------------------ # d - d space # name: d.ethel # doc: female neighbor from I Love Lucy d.ethel=silly # name: d.fred # doc: male neighbor from I Love Lucy d.fred=crabby #------------------------------------------------------------------------------ # x - x space # name: x.password # doc: the password x.password=secret # name: x.size # doc: how big in tons x.size=100""" self.assertEqual(received.strip(), expected) strio = StringIO(expected) n.c.dwight.default = 3823 n.c.e.dwight = 'fred' cm2 = ConfigurationManager( n, [stringIO_context_wrapper(strio)], use_admin_controls=False, use_auto_help=False ) result = cm2.get_config() self.assertEqual(len(result), 4) self.assertEqual(sorted(result.keys()), ['aaa', 'c', 'd', 'x']) self.assertEqual(len(result.c), 4) self.assertEqual( sorted(result.c.keys()), ['dwight', 'e', 'fred', 'wilma'] ) self.assertEqual(result.c.dwight, 98) self.assertEqual(len(result.c.e), 1) self.assertEqual(result.c.e.dwight, '97')
def run( params=None, # params for running from command line map_data=None, # map_data, as_double() pdb_inp=None, pdb_hierarchy=None, crystal_symmetry=None, resolution=None, scattering_table='n_gaussian', smoothing_window=5, crossover_atom='CA', minimum_matching_atoms=3, minimum_length=2, dist_max=1.0, minimum_improvement=0.01, max_regions_to_test=10, max_ends_per_region=5, maximum_fraction=0.5, max_keep=10, map_coeffs_file=None,map_coeffs_labels=None, pdb_in_file=None, pdb_out=None, verbose=None, out=sys.stdout): if out is None: out=sys.stdout # explode and refine calls it this way # get info from params if present if params: verbose=params.control.verbose map_coeffs_file=params.input_files.map_coeffs_file map_coeffs_labels=params.input_files.map_coeffs_labels pdb_in_file=params.input_files.pdb_in_file resolution=params.crystal_info.resolution scattering_table=params.crystal_info.scattering_table smoothing_window=params.crossover.smoothing_window crossover_atom=params.crossover.crossover_atom minimum_matching_atoms=params.crossover.minimum_matching_atoms minimum_length=params.crossover.minimum_length dist_max=params.crossover.dist_max minimum_improvement=params.crossover.minimum_improvement max_regions_to_test=params.crossover.max_regions_to_test max_ends_per_region=params.crossover.max_ends_per_region maximum_fraction=params.crossover.maximum_fraction max_keep=params.crossover.max_keep pdb_out=params.output_files.pdb_out # Consistency checks if(pdb_hierarchy is not None): assert pdb_in_file is None assert pdb_inp is None assert crystal_symmetry is not None # XXX more checks here! # Get map_data if not present if not map_data: if not map_coeffs_file or not os.path.isfile(map_coeffs_file): raise Sorry("Cannot find the map_coeffs_file '%s'" %( str(map_coeffs_file))) from mmtbx.building.minimize_chain import get_map_coeffs map_coeffs=get_map_coeffs(map_coeffs_file, map_coeffs_labels=map_coeffs_labels) fft_map = map_coeffs.fft_map(resolution_factor = 0.25) fft_map.apply_sigma_scaling() map_data = fft_map.real_map_unpadded() map_data=map_data.as_double() if map_coeffs and not crystal_symmetry: crystal_symmetry=map_coeffs.crystal_symmetry() if map_coeffs and not resolution: resolution=map_coeffs.d_min() # Get the starting model if(pdb_hierarchy is None): if pdb_inp is None: if not pdb_in_file or not os.path.isfile(pdb_in_file): raise Sorry("Cannot read input PDB file '%s'" %( str(pdb_in_file))) else: print("Taking models from %s" %(pdb_in_file), file=out) pdb_string=open(pdb_in_file).read() pdb_inp=iotbx.pdb.input(source_info=None, lines = pdb_string) if pdb_inp is None: raise Sorry("Need a model or models") if not crystal_symmetry: crystal_symmetry=pdb_inp.crystal_symmetry() assert crystal_symmetry is not None hierarchy = pdb_inp.construct_hierarchy() else: hierarchy = pdb_hierarchy # XXX FIXME n_models=0 for model in hierarchy.models(): n_models+=1 if n_models==1: # nothing to do return hierarchy #xrs = pdb_inp.xray_structure_simple(crystal_symmetry=crystal_symmetry) xrs = hierarchy.extract_xray_structure(crystal_symmetry=crystal_symmetry) xrs.scattering_type_registry(table=scattering_table) if not resolution: from cctbx import maptbx resolution=maptbx.resolution_from_map_and_model.run( map_data=map_data, xray_structure=xrs).d_min if(resolution is None): raise Sorry("Resolution is required") print("\nResolution limit: %7.2f" %(resolution), file=out) print("\nSummary of input models", file=out) xrs.show_summary(f=out, prefix=" ") print("\nReady with %d models and map" %(n_models), file=out) # Get CC by residue for each model and map chain_id_and_resseq_list=[] # Instead set up chain_id and resseq (range) from mmtbx.secondary_structure.find_ss_from_ca import \ split_model,get_first_resno, get_last_resno,get_chain_id model_list=split_model(hierarchy=hierarchy,only_first_model=True) for m in model_list: h=m.hierarchy first_resno=get_first_resno(h) last_resno=get_last_resno(h) chain_id=get_chain_id(h) residue_range=[first_resno,last_resno] chain_id_and_resseq=[chain_id,residue_range] if not chain_id_and_resseq in chain_id_and_resseq_list: chain_id_and_resseq_list.append(chain_id_and_resseq) # Run through chains separately # NOTE: All models of each chain must match exactly # Save composite model, chain by chain composite_model_stream=StringIO() for chain_id_and_resseq in chain_id_and_resseq_list: f=StringIO() chain_id,[start_resno,end_resno]=chain_id_and_resseq atom_selection=get_atom_selection(chain_id=chain_id, start_resno=start_resno,end_resno=end_resno) asc=hierarchy.atom_selection_cache() sel=asc.selection(string = atom_selection) sel_hierarchy=hierarchy.select(sel) pdb_inp=sel_hierarchy.as_pdb_input(crystal_symmetry=crystal_symmetry) ph=pdb_inp.construct_hierarchy() print("\nWorking on chain_id='%s' resseq %d:%d\n" %( chain_id_and_resseq[0],chain_id_and_resseq[1][0],chain_id_and_resseq[1][1]), file=out) # get CC values for all residues cc_dict=get_cc_dict(hierarchy=ph,map_data=map_data,d_min=resolution, crystal_symmetry=crystal_symmetry, table=scattering_table,out=out) # smooth CC values with window of smoothing_window smoothed_cc_dict=smooth_cc_values(cc_dict=cc_dict, smoothing_window=smoothing_window, verbose=verbose,out=out) # figure out all the places where crossover can occur. # FIXME: order of keys changes in py2/3 vthis could be bad n_residues=cc_dict[list(cc_dict.keys())[0]].size() crossover_dict=get_crossover_dict( n_residues=n_residues, hierarchy=ph, crossover_atom=crossover_atom, dist_max=dist_max, minimum_matching_atoms=minimum_matching_atoms, verbose=verbose,out=out) # Now we are ready to identify the best composite model... # A composite has reside 0 from model x, residue 1 from model y etc. # Each change from model a to model b between residues i and i+1 must have # a crossover between a and b at either residue i or i+1 keys=list(cc_dict.keys()) keys.sort() sorted_working_model_list=[] for key in keys: working_model=model_object(source_id=key, cc_dict=cc_dict, smoothed_cc_dict=smoothed_cc_dict, crossover_dict=crossover_dict, minimum_length=minimum_length, minimum_improvement=minimum_improvement, max_regions_to_test=max_regions_to_test, max_ends_per_region=max_ends_per_region, maximum_fraction=maximum_fraction) if verbose: working_model.show_summary(out=out) sorted_working_model_list.append( [working_model.get_score(),working_model]) sorted_working_model_list.sort() sorted_working_model_list.reverse() sorted_working_model_list=\ sorted_working_model_list[:max_keep] working_model_list=[] for s,m in sorted_working_model_list: working_model_list.append(m) # Go through all the working models and cross them with other models to # optimize...Then take all the best and cross... best_score,best_model=sorted_working_model_list[0] found=True cycle=0 while found: cycle+=1 print("\nCYCLE %d current best is %7.3f\n" %( cycle,best_model.get_score()), file=out) found=False sorted_working_model_list=[] new_best=best_model id=0 for working_model in working_model_list: id+=1 others=[] for m in working_model_list: if not working_model==m: others.append(m) new_working_model=working_model.optimize_with_others(others=others) if not new_working_model: print() continue aa=[new_working_model.get_score(),new_working_model] if not aa in sorted_working_model_list: sorted_working_model_list.append(aa) if not sorted_working_model_list: break # nothing to do sorted_working_model_list.sort() sorted_working_model_list.reverse() sorted_working_model_list=sorted_working_model_list[:max_keep] new_working_score,new_working_model=sorted_working_model_list[0] if new_working_score>best_model.get_score(): best_model=new_working_model found=True if verbose: print("NEW BEST SCORE: %7.2f" %(best_model.get_score()), file=out) best_model.show_summary(out=out) print("\nDONE... best is %7.3f\n" %( best_model.get_score()), file=out) # Create composite of this chain # Note residue values. We are going to pick each residue from one of # the models for model in ph.models(): for chain in model.chains(): if chain.id != chain_id: continue residue_list=[] for rg in chain.residue_groups(): residue_list.append(rg.resseq) residue_list.sort() assert len(best_model.source_list)==len(residue_list) for i in range(len(residue_list)): atom_selection=get_atom_selection(model_id=best_model.source_list[i], resseq_sel=residue_list[i]) asc=ph.atom_selection_cache() sel=asc.selection(string = atom_selection) sel_hierarchy=ph.select(sel) print(remove_ter(sel_hierarchy.as_pdb_string()), file=composite_model_stream) # All done, make a new pdb_hierarchy pdb_string=composite_model_stream.getvalue() pdb_inp=iotbx.pdb.input(source_info=None, lines = pdb_string) pdb_hierarchy=pdb_inp.construct_hierarchy() if pdb_out: f=open(pdb_out,'w') print(pdb_hierarchy.as_pdb_string(crystal_symmetry=crystal_symmetry), file=f) print("Final model is in: %s\n" %(f.name)) f.close() return pdb_hierarchy
def main(): parser = argparse.ArgumentParser(description='Print tables pretty') parser.add_argument('infile', type=str, nargs='?', help='input file, default is stdin') parser.add_argument( '-d', dest='delimiter', type=str, required=False, help='delimiter of fields of input. Default is white space.') parser.add_argument( '-j', dest='justify', type=str, required=False, default='left', choices=['left', 'right', 'center'], help='justification, either left, right or center. Default is left') parser.add_argument('-s', dest='separator', type=str, required=False, default=' ', help='separator of fields in output') args = parser.parse_args() table = [] maxwidth = [] # default is to read from stdin fin = sys.stdin if args.infile: try: fin = open(args.infile, 'rt') except IOError as e: print('Error: %s: %s\n' % (e.strerror, args.infile), file=sys.stderr) sys.exit(e.errno) try: for line in fin: fields = None # split line by delimiter if args.delimiter: fields = line.strip().split(args.delimiter) else: fields = line.strip().split() for i in xrange(len(fields)): width = len(fields[i]) if (i + 1) > len(maxwidth): maxwidth.append(width) else: if width > maxwidth[i]: maxwidth[i] = width table.append(fields) fin.close() for fields in table: line = StringIO() for i in xrange(len(fields)): # format field with different justification nSpace = maxwidth[i] - len(fields[i]) if args.justify == 'left': line.write(fields[i]) for j in xrange(nSpace): line.write(' ') elif args.justify == 'right': for j in xrange(nSpace): line.write(' ') line.write(fields[i]) elif args.justify == 'center': for j in xrange(nSpace / 2): line.write(' ') line.write(fields[i]) for j in xrange(nSpace - nSpace / 2): line.write(' ') line.write(args.separator) print(line.getvalue()) line.close() except IOError as e: if e.errno == errno.EPIPE: sys.exit(-e.errno) else: raise e except KeyboardInterrupt: pass
def vmd_visualize_unified(pdb_filename, trajectory_filename, cplx_filename, step): """Function that creates a string in a syntax that can be interpreted by vmd. This string can be saved to a text/data file. Includes a comment-header. The very last printed lines prevent vmd from automatically creating an additional Representation. This Function creates a very basic representation of the given topology: All simulation beads are displayed as vdw,particles of defined radius (default = 1.0) Parameters ---------- pdb_filename: String Containing name of pdb-file (parsed argument) trajectory_filename: String Containing name of trajectory file (parsed argument) cplx_filename: String Containing name of cplx-file (parsed argument) Returns ------- String the complete string to be written to data-file """ vdw_radius = 1.0 trajectory_type = os.path.splitext(trajectory_filename)[-1][1:] out = StringIO() out.write( HEADER_FORMAT.format( generation_time=datetime.datetime.time(datetime.datetime.now()), topology_inputfile=cplx_filename, pdb_inputfile=pdb_filename, trajectory_inputfile=trajectory_filename, )) out.write(LOAD_VMD_TOP_FORMAT.format(vmd_top=pdb_filename)) out.write( LOADTRAJ_FORMAT.format(traj_file=trajectory_filename, traj_type=trajectory_type, step=step)) out.write( "#adding representation to molecule 0 \n #################################### \n" ) colorid = 1 rep_type = "VDW" rep_param = "{} 12.000000".format(vdw_radius) rep_number = 0 out.write( ADDREP_FORMAT.format( colorid=colorid, rep_number=rep_number, mol_number=0, resid_string="all", rep_type=rep_type, rep_param=rep_param, )) rep_counter = 1 out.write(DEL_REP_FORMAT.format(rep_counter=rep_counter)) out.write(SHOW_SIM_BOX) rep = out.getvalue() out.close() return rep
def write_default_ansible_cfg(work_dir, remote_user, ssh_private_key=None, transport=None, base_ansible_cfg='/etc/ansible/ansible.cfg', override_ansible_cfg=None): ansible_config_path = os.path.join(work_dir, 'ansible.cfg') shutil.copy(base_ansible_cfg, ansible_config_path) modules_path = ('/root/.ansible/plugins/modules:' '/usr/share/ansible/plugins/modules:' '%s/library' % constants.DEFAULT_VALIDATIONS_BASEDIR) lookups_path = ( 'root/.ansible/plugins/lookup:' '/usr/share/ansible/plugins/lookup:' '%s/lookup_plugins' % constants.DEFAULT_VALIDATIONS_BASEDIR) callbacks_path = ( '~/.ansible/plugins/callback:' '/usr/share/ansible/plugins/callback:' '%s/callback_plugins' % constants.DEFAULT_VALIDATIONS_BASEDIR) roles_path = ('%(work_dir)s/roles:' '/root/.ansible/roles:' '/usr/share/ansible/roles:' '/etc/ansible/roles:' '%(ooo_val_path)s/roles:' '%(work_dir)s' % { 'work_dir': work_dir, 'ooo_val_path': constants.DEFAULT_VALIDATIONS_BASEDIR }) config = configparser.ConfigParser() config.read(ansible_config_path) config.set('defaults', 'retry_files_enabled', 'False') config.set('defaults', 'roles_path', roles_path) config.set('defaults', 'library', modules_path) config.set('defaults', 'callback_plugins', callbacks_path) config.set('defaults', 'lookup_plugins', lookups_path) log_path = os.path.join(work_dir, 'ansible.log') config.set('defaults', 'log_path', log_path) if os.path.exists(log_path): new_path = (log_path + '-' + datetime.now().strftime("%Y-%m-%dT%H:%M:%S")) os.rename(log_path, new_path) config.set('defaults', 'forks', '25') config.set('defaults', 'timeout', '30') config.set('defaults', 'gather_timeout', '30') # mistral user has no home dir set, so no place to save a known hosts file config.set('ssh_connection', 'ssh_args', '-o UserKnownHostsFile=/dev/null ' '-o StrictHostKeyChecking=no ' '-o ControlMaster=auto ' '-o ControlPersist=30m ' '-o ServerAliveInterval=5 ' '-o ServerAliveCountMax=5') config.set('ssh_connection', 'control_path_dir', os.path.join(work_dir, 'ansible-ssh')) config.set('ssh_connection', 'retries', '8') config.set('ssh_connection', 'pipelining', 'True') # Set connection info in config file so that subsequent/nested ansible # calls can re-use it if remote_user: config.set('defaults', 'remote_user', remote_user) if ssh_private_key: config.set('defaults', 'private_key_file', ssh_private_key) if transport: config.set('defaults', 'transport', transport) if override_ansible_cfg: sio_cfg = StringIO() sio_cfg.write(override_ansible_cfg) sio_cfg.seek(0) config.readfp(sio_cfg) sio_cfg.close() with open(ansible_config_path, 'w') as configfile: config.write(configfile) return ansible_config_path