def post(self):
if 'attachmentName' in request.files:
file = request.files['attachmentName']
if file:
filename = secure_filename(file.name)
file.save(filename)
book = xlrd.open_workbook(filename)
conn = psycopg2.connect()
cur = conn.cursor()
input_buffer = StringIO()
n = 0
for sh in book.sheets():
for i in range(2, sh.nrows):
line = '\t'.join([str(x) for x in sh.row_values(i)])+'\n'
line = self.block(sh.name, line)
if line:
input_buffer.write(line)
if n % 50 == 0:
input_buffer.seek(0)
cur.copy_from(input_buffer, table[sh.name])
conn.commit()
input_buffer = StringIO()
n += 1
cur.copy_from(input_buffer, table[sh.name])
input_buffer = StringIO()
conn.commit()
cur.execute("select * from ms_infor")
print(cur.fetchall())
return '导入成功'
return '导入失败'
def marshal( self ):
oStreamString = StringIO()
oStreamString.write(unicode('<?xml version="1.0" ?>\n'))
self.export( oStreamString, 0, name_="XSDataResultWaitMultiFile" )
oStringXML = oStreamString.getvalue()
oStreamString.close()
return oStringXML
def contentxml(self):
"""
Generates the content.xml file
@return a bytestream in UTF-8 encoding
"""
xml=StringIO()
xml.write(_XMLPROLOGUE)
x = DocumentContent()
x.write_open_tag(0, xml)
if self.scripts.hasChildNodes():
self.scripts.toXml(1, xml)
if self.fontfacedecls.hasChildNodes():
self.fontfacedecls.toXml(1, xml)
a = AutomaticStyles()
stylelist = self._used_auto_styles([self.styles, self.automaticstyles, self.body])
if len(stylelist) > 0:
a.write_open_tag(1, xml)
for s in stylelist:
s.toXml(2, xml)
a.write_close_tag(1, xml)
else:
a.toXml(1, xml)
self.body.toXml(1, xml)
x.write_close_tag(0, xml)
return xml.getvalue().encode("utf-8")
def save(self):
"""
Create the writer & save
"""
# GH21227 internal compression is not used when file-like passed.
if self.compression and hasattr(self.path_or_buf, 'write'):
msg = ("compression has no effect when passing file-like "
"object as input.")
warnings.warn(msg, RuntimeWarning, stacklevel=2)
# when zip compression is called.
is_zip = isinstance(self.path_or_buf, ZipFile) or (
not hasattr(self.path_or_buf, 'write')
and self.compression == 'zip')
if is_zip:
# zipfile doesn't support writing string to archive. uses string
# buffer to receive csv writing and dump into zip compression
# file handle. GH21241, GH21118
f = StringIO()
close = False
elif hasattr(self.path_or_buf, 'write'):
f = self.path_or_buf
close = False
else:
f, handles = _get_handle(self.path_or_buf, self.mode,
encoding=self.encoding,
compression=self.compression)
close = True
try:
writer_kwargs = dict(lineterminator=self.line_terminator,
delimiter=self.sep, quoting=self.quoting,
doublequote=self.doublequote,
escapechar=self.escapechar,
quotechar=self.quotechar)
if self.encoding == 'ascii':
self.writer = csvlib.writer(f, **writer_kwargs)
else:
writer_kwargs['encoding'] = self.encoding
self.writer = UnicodeWriter(f, **writer_kwargs)
self._save()
finally:
if is_zip:
# GH17778 handles zip compression separately.
buf = f.getvalue()
if hasattr(self.path_or_buf, 'write'):
self.path_or_buf.write(buf)
else:
f, handles = _get_handle(self.path_or_buf, self.mode,
encoding=self.encoding,
compression=self.compression)
f.write(buf)
close = True
if close:
f.close()
for _fh in handles:
_fh.close()
def stylesxml(self):
"""
Generates the styles.xml file
@return valid XML code as a unicode string
"""
xml=StringIO()
xml.write(_XMLPROLOGUE)
x = DocumentStyles()
x.write_open_tag(0, xml)
if self.fontfacedecls.hasChildNodes():
self.fontfacedecls.toXml(1, xml)
self.styles.toXml(1, xml)
a = AutomaticStyles()
a.write_open_tag(1, xml)
for s in self._used_auto_styles([self.masterstyles]):
s.toXml(2, xml)
a.write_close_tag(1, xml)
if self.masterstyles.hasChildNodes():
self.masterstyles.toXml(1, xml)
x.write_close_tag(0, xml)
result = xml.getvalue()
assert(type(result)==type(u""))
return result
def run(self):
try:
# Try parsing as XML
root = etree.parse(self.fn)
ns = "{%s}" % schemas['corpus']
out = StringIO()
for i in root.getiterator(ns + "entry"):
out.write(i.text + "\n")
self.corpus = out.getvalue()
del out
except:
# Turns out it's not XML
self.corpus = open(self.fn, 'r')
try:
open(self.dct) # test existence
except:
raise # TODO: wrap error for output
if not self.result:
delim = re.compile(r"\$[^^]*\^")
f = open(self.fn, 'r')
data = f.read()
f.close()
output = destxt(data).encode('utf-8')
timing_begin = time.time()
proc = Popen([self.app] + self.app_args + [self.dct], stdin=PIPE, stdout=PIPE, close_fds=True)
output = str(proc.communicate(output)[0].decode('utf-8'))
self.timer = time.time() - timing_begin
output = retxt(output)
output = delim.sub("$\n^", output)
self.result = output.split('\n')
return 0
def toXml(self, filename=u''):
"""
converts the document to a valid Xml format.
@param filename unicode string: the name of a file, defaults to
an empty string.
@return if filename is not empty, the XML code will be written into it
and the method returns None; otherwise the method returns a StringIO
containing valid XML.
Then a ".getvalue()" should return a unicode string.
"""
assert(type(filename)==type(u""))
result=None
xml=StringIO()
if sys.version_info[0]==2:
xml.write(_XMLPROLOGUE)
else:
xml.write(_XMLPROLOGUE)
self.body.toXml(0, xml)
if not filename:
result=xml.getvalue()
else:
f=codecs.open(filename,'w', encoding='utf-8')
f.write(xml.getvalue())
f.close()
return result
def test_opened_file(self):
sio = StringIO()
sio.write('test_data')
sio.seek(0)
file, close = open_if_filename(sio)
assert not close
eq_('test_data', file.read())
def marshal( self ):
oStreamString = StringIO()
oStreamString.write(unicode('<?xml version="1.0" ?>\n'))
self.export( oStreamString, 0, name_="XSDataOutputXdsBurnStrategy" )
oStringXML = oStreamString.getvalue()
oStreamString.close()
return oStringXML
def cli_success(resp):
data = resp["result"]["random"]["data"]
msg = StringIO()
msg.write("\n".join(map(str, data)))
output = msg.getvalue()
msg.close()
return output
class ContentState( State ):
""" This state records every line it meets as content until it reaches a line notifying a deposit reference or a publication reference.
Can return ``self`` or :class:`.PublicationState` or :class:`.DepositState`.
"""
def __init__( self, store ):
State.__init__( self, store )
self.textIO = StringIO()
def process_line( self, line ):
if publication_line_re.match( line ):
self.store_content()
return PublicationState( self.store ).process_line( line )
elif deposit_line_re.match( line ):
self.store_content()
return DepositState( self.store ).process_line( line )
else:
self.add_content_line( line )
globalOutputter.writeTagLine( 'TXT', line )
return self
def add_content_line( self, line ):
self.textIO.write( line )
self.textIO.write( '\n' )
def store_content( self ):
self.store['current_article']['content'] = self.textIO.getvalue()
class TFramedTransport(TTransportBase, CReadableTransport):
"""Class that wraps another transport and frames its I/O when writing."""
def __init__(self, trans,):
self.__trans = trans
self.__rbuf = StringIO()
self.__wbuf = StringIO()
def isOpen(self):
return self.__trans.isOpen()
def open(self):
return self.__trans.open()
def close(self):
return self.__trans.close()
def read(self, sz):
ret = self.__rbuf.read(sz)
if len(ret) != 0:
return ret
self.readFrame()
return self.__rbuf.read(sz)
def readFrame(self):
buff = self.__trans.readAll(4)
sz, = unpack('!i', buff)
self.__rbuf = StringIO(self.__trans.readAll(sz))
def write(self, buf):
self.__wbuf.write(buf)
def flush(self):
wout = self.__wbuf.getvalue()
wsz = len(wout)
# reset wbuf before write/flush to preserve state on underlying failure
self.__wbuf = StringIO()
# N.B.: Doing this string concatenation is WAY cheaper than making
# two separate calls to the underlying socket object. Socket writes in
# Python turn out to be REALLY expensive, but it seems to do a pretty
# good job of managing string buffer operations without excessive copies
buf = pack("!i", wsz) + wout
self.__trans.write(buf)
self.__trans.flush()
# Implement the CReadableTransport interface.
@property
def cstringio_buf(self):
return self.__rbuf
def cstringio_refill(self, prefix, reqlen):
# self.__rbuf will already be empty here because fastbinary doesn't
# ask for a refill until the previous buffer is empty. Therefore,
# we can start reading new frames immediately.
while len(prefix) < reqlen:
self.readFrame()
prefix += self.__rbuf.getvalue()
self.__rbuf = StringIO(prefix)
return self.__rbuf
def extract_diff_header(status, deleted,
with_diff_header, suppress_header, diffoutput):
headers = []
if diffoutput.startswith('Submodule'):
if with_diff_header:
return ('', diffoutput)
else:
return diffoutput
start = False
del_tag = 'deleted file mode '
output = StringIO()
diff = diffoutput.split('\n')
for line in diff:
if not start and '@@' == line[:2] and '@@' in line[2:]:
start = True
if start or (deleted and del_tag in line):
output.write(line + '\n')
else:
if with_diff_header:
headers.append(line)
elif not suppress_header:
output.write(line + '\n')
result = output.getvalue()
output.close()
if with_diff_header:
return('\n'.join(headers), result)
else:
return result
def dump(self):
output = StringIO()
for n in self._nodes:
output.write(self._dot_node(n))
output.write(u"\n")
for r in self._relations:
output.write(self._dot_relation(r))
output.write(u"\n")
if len(self._families) > 1:
for f in self._families:
output.write(self._dot_sub_graph(f))
template = u"""
digraph %s
{
\trankdir = "LR";
\tranksep = 0.5;
\tlabel = "%s";
\tlabelloc = "t";
\tfontsize = "24";
\tfontname = "SimHei";
\tgraph [style="filled", color="lightgrey"];
\tnode [fontname="SimSun"];
\tedge [fontname="SimSun"];
%s
}
"""
return template % (self._name, self._name, output.getvalue())
def csvpp(csv_input):
max_widths = []
max_indent = 0
for line in csv.reader(StringIO(csv_input)):
widths = [len(s.strip()) for s in line]
max_widths = list(map(max, izip_longest(max_widths, widths, fillvalue=0)))
indent = len(line[0]) - len(line[0].lstrip())
max_indent = max(max_indent, indent)
result = StringIO()
for line in csv.reader(StringIO(csv_input)):
result.write(u' ' * max_indent)
last_column = len(line) - 1
for value, max_width, column in zip(line, max_widths, count()):
value = value.strip()
result.write(u"" + value)
if column != last_column:
result.write(u", ")
result.write(u" " * (max_width - len(value)))
result.write(u'\n')
return result.getvalue()
def marshal( self ):
oStreamString = StringIO()
oStreamString.write(unicode('<?xml version="1.0" ?>\n'))
self.export( oStreamString, 0, name_="XSDataInputControlBackground3D" )
oStringXML = oStreamString.getvalue()
oStreamString.close()
return oStringXML
def marshal( self ):
oStreamString = StringIO()
oStreamString.write(unicode('<?xml version="1.0" ?>\n'))
self.export( oStreamString, 0, name_="XSDataNexusArrayGroup" )
oStringXML = oStreamString.getvalue()
oStreamString.close()
return oStringXML
def __str__(self):
output = StringIO()
for y in range(self._height):
for x in range(self._width):
output.write(self.get_symbol(x, y))
output.write('\n')
return output.getvalue()
def marshal( self ):
oStreamString = StringIO()
oStreamString.write(unicode('<?xml version="1.0" ?>\n'))
self.export( oStreamString, 0, name_="XSDataInputInterfacev2_2" )
oStringXML = oStreamString.getvalue()
oStreamString.close()
return oStringXML
def check_cpaste(code, should_fail=False):
"""Execute code via 'cpaste' and ensure it was executed, unless
should_fail is set.
"""
ip.user_ns['code_ran'] = False
src = StringIO()
if not hasattr(src, 'encoding'):
# IPython expects stdin to have an encoding attribute
src.encoding = None
src.write(code)
src.write('\n--\n')
src.seek(0)
stdin_save = sys.stdin
sys.stdin = src
try:
context = tt.AssertPrints if should_fail else tt.AssertNotPrints
with context("Traceback (most recent call last)"):
ip.magic('cpaste')
if not should_fail:
assert ip.user_ns['code_ran'], "%r failed" % code
finally:
sys.stdin = stdin_save
def generate_includefile(self):
# add the swig includes
includeFile = StringIO()
for f in options.swig_includes:
includeFile.write("%%include %s\n" % f)
includeFile.write('\n\n')
return includeFile
def test_reading_successful(self):
"""
Test the successful reading of CSV Line and have the record passed through to the Data Controller.
The Data Controller is this TestCsvImporter class acting as a stub - it implements necessary 'public' method.
Expected:
- One record will be passed through to process_record method
- self.process_record_call is True
- self.start_import_called is True
"""
input_text = StringIO()
input_text.write(
u'date,category,employee name,employee address,expense description,pre-tax amount,tax name,tax amount\n')
input_text.write(
u'12/1/2013,Travel,Don Draper,"783 Park Ave, New York, NY 10021",Taxi ride, 350.00 ,NY Sales tax, 31.06\n')
input_text.seek(0)
dialect = csv.Sniffer().sniff(sample=input_text.getvalue())
dialect.delimiter = ','
dialect.quotechar = '"'
csv_importer = CsvImporter(data_controller=self)
actual_return = csv_importer.import_data(input_text)
self.assertEqual(self.upload_id, actual_return)
self.assertTrue(self.process_record_call)
self.assertTrue(self.start_import_called)
def redraw(self):
"""Create the pulldown menu."""
text = StringIO()
text.write(TOP)
self.undisplay()
actions = []
count = 0
if len(self.bookmarks.get()) > 0:
text.write('<placeholder name="GoToBook">')
for item in self.bookmarks.get():
try:
label, obj = self.make_label(item)
func = self.callback(item)
action_id = "BM:%s" % item
actions.append((action_id, None, label, None, None, func))
text.write('<menuitem action="%s"/>' % action_id)
count += 1
except AttributeError:
pass
text.write('</placeholder>')
text.write(BTM)
self.action_group.add_actions(actions)
self.uistate.uimanager.insert_action_group(self.action_group, 1)
self.active = self.uistate.uimanager.add_ui_from_string(text.getvalue())
self.uistate.uimanager.ensure_update()
text.close()
def procgpgkey(rawkey):
'''Convert ASCII armoured GPG key to binary
'''
# TODO: CRC checking? (will RPM do this anyway?)
# Normalise newlines
rawkey = re.sub('\r\n?', '\n', rawkey)
# Extract block
block = StringIO()
inblock = 0
pastheaders = 0
for line in rawkey.split('\n'):
if line.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----'):
inblock = 1
elif inblock and line.strip() == '':
pastheaders = 1
elif inblock and line.startswith('-----END PGP PUBLIC KEY BLOCK-----'):
# Hit the end of the block, get out
break
elif pastheaders and line.startswith('='):
# Hit the CRC line, don't include this and stop
break
elif pastheaders:
block.write(line+'\n')
# Decode and return
return base64.decodestring(block.getvalue().encode())
def get_text(node: etree.Element, stream: io.StringIO=None):
"""
Extract text from this node and all children until an <ol> occurs
"""
if stream is None:
start = True
stream = io.StringIO()
else:
start = False
def to_xml(s: str):
s = "" if s is None else s
return s.encode('ascii', 'xmlcharrefreplace').decode()
stream.write(to_xml(node.text))
for child in node:
if child.tag == "ol":
break
get_text(child, stream=stream)
if start:
# we are done, return the buffered string
return stream.getvalue()
else:
# we are in a child, append our tail to the total string
stream.write(to_xml(node.tail))
def _get_env_variable(self, var_name, default=False):
"""
Get the environment variable or return exception
:param var_name: Environment Variable to lookup
"""
try:
return os.environ[var_name]
except KeyError:
from io import StringIO
from configparser import ConfigParser
env_file = os.environ.get('PROJECT_ENV_FILE', self._root_folder_path + "/.env")
try:
config = StringIO()
config.write("[DATA]\n")
config.write(open(env_file).read())
config.seek(0, os.SEEK_SET)
cp = ConfigParser()
cp.read_file(config)
value = dict(cp.items('DATA'))[var_name.lower()]
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
elif value.startswith("'") and value.endswith("'"):
value = value[1:-1]
os.environ.setdefault(var_name, value)
return value
except (KeyError, IOError):
if default is not False:
return default
error_msg = "Either set the env variable '{var}' or place it in your " \
"{env_file} file as '{var} = VALUE'"
raise ConfigurationError(error_msg.format(var=var_name, env_file=env_file))
def _repr(self):
buf = StringIO()
space = 2 + max(len(x) for x in self.names)
for name, tipo in zip(self.names, self.types):
buf.write('\n{0}{1}'.format(name.ljust(space), str(tipo)))
return "ibis.Schema {{{0}\n}}".format(util.indent(buf.getvalue(), 2))
class LinkParser(HTMLParser):
def reset(self):
HTMLParser.reset(self)
self.data = StringIO()
def handle_data(self, data):
self.data.write('%s\n' % decode(data))
def __str__(self):
"""Returns string representation of all stored entities."""
buf = StringIO()
for entity in self.entities.values():
buf.write(u'Entity {0}:\n'.format(entity.key.flat_path))
buf.write(u' {0}\n'.format(dict(entity)))
return buf.getvalue()
def get_version_info(url):
out_stream = StringIO()
pkg_maxlen = 0
version_maxlen = 0
version_sections = extract_versions_section(url)
for pkg_name, version_infos in version_sections.items():
pkg_maxlen = max(len(pkg_name), pkg_maxlen)
for version_info in version_infos:
version_maxlen = max(len(version_info.version), version_maxlen)
outfmt = ('{{pkg_name:{pkg_maxlen}}} = {{color}}{{version_info.version:{version_maxlen}}}{reset} {{index}} {{version_info.origin}}\n' # NOQA
.format(pkg_maxlen=pkg_maxlen, version_maxlen=version_maxlen,
reset=colorama.Fore.RESET))
for pkg_name, version_infos in sorted(version_sections.items()):
if (max((parse_version(version_info.version)
for version_info in version_infos)) !=
parse_version(version_infos[0].version)):
color = colorama.Fore.RED
else:
color = colorama.Fore.RESET
for index, version_info in enumerate(version_infos):
pkg_name_to_show = pkg_name if not index else ''
out_stream.write(outfmt.format(pkg_name=pkg_name_to_show,
version_info=version_info,
color=color,
index=index))
out_stream.seek(0)
return out_stream
def format_linting_result_header(verbose=0):
"""Format the header of a linting result output."""
text_buffer = StringIO()
if verbose >= 1:
text_buffer.write("==== readout ====\n")
return text_buffer.getvalue()
def stringify(self, ident=0, tabsize=4, code_only=False):
"""Use indentation to render this segment and its children as a string."""
buff = StringIO()
preface = self._preface(ident=ident, tabsize=tabsize)
buff.write(preface + "\n")
if not code_only and self.comment_seperate and len(self._comments) > 0:
if self._comments:
buff.write((" " * ((ident + 1) * tabsize)) + "Comments:" + "\n")
for seg in self._comments:
buff.write(
seg.stringify(
ident=ident + 2,
tabsize=tabsize,
code_only=code_only,
)
)
if self._non_comments:
buff.write((" " * ((ident + 1) * tabsize)) + "Code:" + "\n")
for seg in self._non_comments:
buff.write(
seg.stringify(
ident=ident + 2,
tabsize=tabsize,
code_only=code_only,
)
)
else:
for seg in self.segments:
# If we're in code_only, only show the code segments, otherwise always true
if not code_only or seg.is_code:
buff.write(
seg.stringify(
ident=ident + 1,
tabsize=tabsize,
code_only=code_only,
)
)
return buff.getvalue()
def write(self, s):
StringIO.write(self, s)
for writer in self.writers:
writer.write(s)
class TFramedTransport(TTransportBase, CReadableTransport):
"""Class that wraps another transport and frames its I/O when writing."""
def __init__(
self,
trans,
):
self.__trans = trans
self.__rbuf = StringIO()
self.__wbuf = StringIO()
def isOpen(self):
return self.__trans.isOpen()
def open(self):
return self.__trans.open()
def close(self):
return self.__trans.close()
def read(self, sz):
ret = self.__rbuf.read(sz)
if len(ret) != 0:
return ret
self.readFrame()
return self.__rbuf.read(sz)
def readFrame(self):
buff = self.__trans.readAll(4)
sz, = unpack('!i', buff)
self.__rbuf = StringIO(self.__trans.readAll(sz))
def write(self, buf):
self.__wbuf.write(buf)
def flush(self):
wout = self.__wbuf.getvalue()
wsz = len(wout)
# reset wbuf before write/flush to preserve state on underlying failure
self.__wbuf = StringIO()
# N.B.: Doing this string concatenation is WAY cheaper than making
# two separate calls to the underlying socket object. Socket writes in
# Python turn out to be REALLY expensive, but it seems to do a pretty
# good job of managing string buffer operations without excessive copies
buf = pack("!i", wsz) + wout
self.__trans.write(buf)
self.__trans.flush()
# Implement the CReadableTransport interface.
@property
def cstringio_buf(self):
return self.__rbuf
def cstringio_refill(self, prefix, reqlen):
# self.__rbuf will already be empty here because fastbinary doesn't
# ask for a refill until the previous buffer is empty. Therefore,
# we can start reading new frames immediately.
while len(prefix) < reqlen:
self.readFrame()
prefix += self.__rbuf.getvalue()
self.__rbuf = StringIO(prefix)
return self.__rbuf
def bill_of_materials(self) -> str:
"""Generate a bill of materials from the project.
See https://spdx.org/specifications.
"""
out = StringIO()
# Write mandatory tags
out.write("SPDXVersion: SPDX-2.1\n")
out.write("DataLicense: CC0-1.0\n")
out.write("SPDXID: SPDXRef-DOCUMENT\n")
out.write(f"DocumentName: {Path(self.path).resolve().name}\n")
# TODO: Generate UUID from git revision maybe
# TODO: Fix the URL
out.write(
f"DocumentNamespace:"
f" http://spdx.org/spdxdocs/spdx-v2.1-{uuid4()}\n"
)
# Author
# TODO: Fix Person and Organization
out.write("Creator: Person: Anonymous ()\n")
out.write("Creator: Organization: Anonymous ()\n")
out.write(f"Creator: Tool: reuse-{__version__}\n")
now = datetime.datetime.utcnow()
now = now.replace(microsecond=0)
out.write(f"Created: {now.isoformat()}Z\n")
out.write(
"CreatorComment: <text>This document was created automatically"
" using available reuse information consistent with"
" REUSE.</text>\n"
)
reports = sorted(self.file_reports, key=lambda x: x.spdxfile.name)
for report in reports:
out.write(
f"Relationship: SPDXRef-DOCUMENT describes"
f" {report.spdxfile.spdx_id}\n"
)
for report in reports:
out.write("\n")
out.write(f"FileName: {report.spdxfile.name}\n")
out.write(f"SPDXID: {report.spdxfile.spdx_id}\n")
out.write(f"FileChecksum: SHA1: {report.spdxfile.chk_sum}\n")
# IMPORTANT: Make no assertion about concluded license. This tool
# cannot, with full certainty, determine the license of a file.
out.write("LicenseConcluded: NOASSERTION\n")
for lic in sorted(report.spdxfile.licenses_in_file):
out.write(f"LicenseInfoInFile: {lic}\n")
if report.spdxfile.copyright:
out.write(
f"FileCopyrightText:"
f" <text>{report.spdxfile.copyright}</text>\n"
)
else:
out.write("FileCopyrightText: NONE\n")
# Licenses
for lic, path in sorted(self.licenses.items()):
if lic.startswith("LicenseRef-"):
out.write("\n")
out.write(f"LicenseID: {lic}\n")
out.write("LicenseName: NOASSERTION\n")
with (Path(self.path) / path).open() as fp:
out.write(f"ExtractedText: <text>{fp.read()}</text>\n")
return out.getvalue()
def format_config(linter, verbose=0):
"""Format the config of a `Linter`."""
text_buffer = StringIO()
# Only show version information if verbosity is high enough
if verbose > 0:
text_buffer.write("==== sqlfluff ====\n")
config_content = [
('sqlfluff', get_package_version()),
('python', get_python_version()),
('dialect', linter.dialect.name),
('verbosity', verbose)
]
text_buffer.write(cli_table(config_content, col_width=25))
text_buffer.write("\n")
if linter.config.get('rule_whitelist'):
text_buffer.write(cli_table([('rules', ', '.join(linter.config.get('rule_whitelist')))], col_width=41))
if verbose > 1:
text_buffer.write("== Raw Config:\n")
text_buffer.write(format_config_vals(linter.config.iter_vals()))
return text_buffer.getvalue()
def format_linting_result_footer(result, verbose=0):
"""Format the footer of a linting result output given a `LintingResult`."""
text_buffer = StringIO()
text_buffer.write('\n')
text_buffer.write(format_linting_stats(result, verbose=verbose))
return text_buffer.getvalue()
""" StrinIO -> utilizado para criar arquivos na memoria"""
from io import StringIO
arquivo = StringIO("Texto de exemplo")
print(arquivo.read())
arquivo.write(" Adicionando linhas")
arquivo.seek(0)
print(arquivo.read())
def main():
parser = argparse.ArgumentParser(
description=
'Convert all the groups in a STIG file into a set of lua hardening scripts'
)
parser.add_argument(
'-d',
'--dest',
help='The destination directory (default: %(default)s)',
default='.')
parser.add_argument('file', help='The STIG file to convert into lua files')
args = parser.parse_args()
tree = ET.parse(args.file)
ns = {'ns': 'http://checklists.nist.gov/xccdf/1.1'}
for group in tree.findall('ns:Group', ns):
id = group.attrib['id']
for rule in group.findall('ns:Rule', ns):
buffer = StringIO()
def tryget(attribute):
try:
return rule.find('ns:{}'.format(attribute), ns).text
except:
return ''
severity = rule.get('severity', '')
weight = rule.get('weight', '')
title = tryget('title')
try:
descriptiontext = rule.find('ns:description', ns).text
descriptiontree = ET.fromstring(
'<root>{}</root>'.format(descriptiontext))
description = descriptiontree.find('VulnDiscussion').text
except:
description = ''
fixtext = tryget('fixtext')
try:
checktext = rule.find('ns:check',
ns).find('ns:check-content',
ns).text.strip()
except:
checktext = ''
buffer.write('id = {}\n'.format(luastring(id)))
buffer.write('severity = {}\n'.format(luastring(severity)))
buffer.write('weight = {}\n'.format(weight))
buffer.write('title = {}\n'.format(luastring(title)))
buffer.write('description = {}\n'.format(luastring(description)))
buffer.write('fixtext = {}\n'.format(luastring(fixtext)))
buffer.write('checktext = {}\n'.format(luastring(checktext)))
buffer.write('\n')
buffer.write('function test()\nend\n')
buffer.write('\n')
buffer.write('function fix()\nend\n')
filename = os.path.join(args.dest, '{}.lua'.format(id))
print('writing {}'.format(filename))
with open(filename, 'w') as file:
file.write(buffer.getvalue())
def format_linting_path(p, verbose=0):
"""Format a linting path."""
text_buffer = StringIO()
if verbose > 0:
text_buffer.write(format_path(p))
return text_buffer.getvalue()
def readme_contents(user_name: str, user_email: str) -> StringIO:
"""Create the Hangar README.txt contents used to fill out file on repo initialization
Parameters
----------
user_name : str
name of the user initializing the repository on the machine.
user_email : str
email of the user initializing the repository on the machine.
Returns
-------
StringIO
Buffered string text ready to be sent to a file writer.
"""
buf = StringIO()
buf.write(
f'This directory has been used to initialize a Hangar Repository\n')
buf.write(f'\n')
buf.write(f'This repository was initialized by:\n')
buf.write(f' User Name: {user_name}\n')
buf.write(f' User Email: {user_email}\n')
buf.write(f' Creation Time: {time.asctime(time.gmtime())} UTC\n')
buf.write(f' Software Version: {__version__}\n')
buf.write(f'\n')
buf.write(
f'NOTE: The repository may have been updated to work with newer Hangar versions\n'
)
buf.write(f'since initialization.\n')
buf.write(f'\n')
buf.write(
f'Do not modify the contents of this `{DIR_HANGAR}` folder under any circumstances.\n'
)
buf.write(
f'The contents are not meant to be understood by humans. Doing so will result\n'
)
buf.write(f'in data loss / corruption.\n')
buf.write(f'\n')
buf.write(
f'The project homepage can be found at: https://github.com/tensorwerk/hangar-py/ \n'
)
buf.write(
f'Documention is available at: https://hangar-py.readthedocs.io/en/latest/ \n'
)
buf.write(f'\n')
buf.write(
f'NOTE: If this Repository has been initialized in a directory under traditional\n'
)
buf.write(
f'version control systems, please add `{DIR_HANGAR}` as an ignored directory path.\n'
)
buf.write(
f'Failure to do so will result in undesireable performance of version control\n'
)
buf.write(
f'systems meant for text/code such as Git, Mercurial, Subversion, etc.\n'
)
return buf
class lbfgs(base_optimizer):
"""the class of lbfgs method"""
def __init__(self, options):
super(lbfgs, self).__init__(options)
self.k = 0
self.end = 0
def optimize(
self,
molecule,
refE=0.,
opt_type='UNCONSTRAINED',
opt_steps=20,
maxcor=10,
ictan=None,
xyzframerate=4,
verbose=False,
):
# stash/initialize some useful attributes
print(" initial E %5.4f" % (molecule.energy - refE))
geoms = []
energies = []
geoms.append(molecule.geometry)
energies.append(molecule.energy - refE)
self.check_inputs(molecule, opt_type, ictan)
nconstraints = self.get_nconstraints(opt_type)
self.buf = StringIO()
# form initial coord basis
constraints = self.get_constraint_vectors(molecule, opt_type, ictan)
molecule.update_coordinate_basis(constraints=constraints)
if opt_type == 'SEAM' or opt_type == 'MECI':
self.opt_cross = True
# get coordinates
x = np.copy(molecule.coordinates)
xyz = np.copy(molecule.xyz)
x_prim = molecule.primitive_internal_values
num_coords = molecule.num_coordinates - nconstraints
# Evaluate the function value and its gradient.
fx = molecule.energy
g = molecule.gradient.copy()
# project out the constraint
gc = g.copy()
for c in molecule.constraints.T:
gc -= np.dot(gc.T, c[:, np.newaxis]) * c[:, np.newaxis]
g_prim = block_matrix.dot(molecule.coord_basis, gc)
molecule.gradrms = np.sqrt(np.dot(gc.T, gc) / num_coords)
# primitive constraint step
self.cstep_prim = np.zeros_like(g_prim)
dE = molecule.difference_energy
# Not checking convergence here anymore ...
#if molecule.PES.__class__.__name__!="PES" and self.opt_cross:
# if molecule.gradrms < self.conv_grms and abs(dE)<1.0:
# print(" converged")
# return geoms,energies
#elif molecule.gradrms < self.conv_grms:
# print(" converged")
# return geoms,energies
## reset k in principle k does not have to reset but . . .
# TRY Turning off Feb 2020
if opt_type != 'CLIMB':
self.k = 0
self.end = 0
# initialize the iteration data list
if self.k == 0:
self.lm = []
for i in range(0, maxcor):
s_prim = np.zeros_like(g_prim)
y_prim = np.zeros_like(g_prim)
self.lm.append(
iterationData(0.0, s_prim.flatten(), y_prim.flatten()))
for ostep in range(opt_steps):
print(" On opt step {} ".format(ostep + 1))
SCALE = self.options['SCALEQN']
if molecule.newHess > 0:
SCALE = self.options['SCALEQN'] * molecule.newHess
if self.k != 0:
# update vectors s and y:
# TODO this doesn't work exactly with constraint steps
self.lm[self.end].s_prim = molecule.coord_obj.Prims.calcDiff(
xyz, self.xyzp) - self.cstep_prim.flatten()
self.lm[self.end].y_prim = g_prim - self.gp_prim
self.end = (self.end + 1) % maxcor
#j = self.end
bound = min(self.k, maxcor)
s_prim = np.array(
[self.lm[i].s_prim.flatten() for i in range(maxcor)])
y_prim = np.array(
[self.lm[i].y_prim.flatten() for i in range(maxcor)])
hess_inv = LbfgsInvHessProduct(s_prim[:bound], y_prim[:bound])
# compute the negative gradients
d_prim = -g_prim
# perform matrix product
d_prim = hess_inv._matvec(d_prim)
d_prim = np.reshape(d_prim, (-1, 1)) / SCALE
else:
# d: store the negative gradient of the object function on point x.
d_prim = -g_prim / SCALE
self.k = self.k + 1
# form in DLC basis (does nothing if cartesian)
d = block_matrix.dot(block_matrix.transpose(molecule.coord_basis),
d_prim)
# normalize the direction
actual_step = np.linalg.norm(d)
print(" actual_step= %1.5f" % actual_step)
d = d / actual_step #normalize
if actual_step > self.DMAX:
step = self.DMAX
print(" reducing step, new step = %1.5f" % step)
else:
step = actual_step
# store
xp = x.copy()
self.xyzp = xyz.copy()
gp = g.copy()
self.gp_prim = block_matrix.dot(molecule.coord_basis, gc)
fxp = fx
pgradrms = molecule.gradrms
# => calculate constraint step <= #
constraint_steps = self.get_constraint_steps(molecule, opt_type, g)
self.cstep_prim = block_matrix.dot(molecule.coord_basis,
constraint_steps)
# line search
ls = self.Linesearch(nconstraints, x, fx, gc, d, step, xp,
constraint_steps, self.linesearch_parameters,
molecule, verbose)
# save new values from linesearch
molecule = ls['molecule']
step = ls['step']
x = ls['x']
fx = ls['fx']
g = ls['g']
dEstep = fx - fxp
dq = x - xp
# TODO dEpre is missing second order effects or is it?
dEpre = np.dot(gc.T, dq) * units.KCAL_MOL_PER_AU
constraint_energy = np.dot(
gp.T, constraint_steps) * units.KCAL_MOL_PER_AU
if opt_type not in ['UNCONSTRAINED', 'ICTAN']:
print("constraint_energy: %1.4f" % constraint_energy)
dEpre += constraint_energy
#if abs(dEpre)<0.05:
# dEpre = np.sign(dEpre)*0.05
ratio = dEstep / dEpre
print(" dEstep=%5.4f" % dEstep)
print(" dEpre=%5.4f" % dEpre)
print(" ratio=%5.4f" % ratio)
# revert to the privious point
if ls['status'] < 0 or (ratio < 0. and opt_type != 'CLIMB'):
if ratio < 0.:
ls['status'] = -1
x = xp.copy()
molecule.xyz = self.xyzp.copy()
g = gp.copy()
fx = fxp
#fx = molecule.energy
ratio = 0.
dEstep = 0.
print('[ERROR] the point return to the previous point')
self.lm = []
for i in range(0, maxcor):
s_prim = np.zeros_like(g_prim)
y_prim = np.zeros_like(g_prim)
self.lm.append(
iterationData(0.0, s_prim.flatten(), y_prim.flatten()))
self.k = 0
self.end = 0
molecule.newHess = 5
self.DMAX = ls['step'] / 2
if self.DMAX < self.DMIN:
self.DMAX = self.DMIN
else:
# update molecule xyz
xyz = molecule.update_xyz(x - xp)
# if ratio is less than 0.3 than reduce DMAX
flag = True
if ratio < 0.3 and ls['status'] == 0: #and abs(dEpre)>0.05:
print(" Reducing DMAX")
self.DMAX /= 1.5
if self.DMAX < self.DMIN:
self.DMAX = self.DMIN
if molecule.newHess < 5:
molecule.newHess += 1
flag = False
if opt_type == "CLIMB":
if self.SCALE_CLIMB < 10. and opt_type == 'CLIMB':
self.SCALE_CLIMB += 1.
print('SCALING CLIMB BY {}'.format(self.SCALE_CLIMB))
elif ratio > 0.3:
molecule.newHess -= 1
if opt_type == "CLIMB":
if self.SCALE_CLIMB > 1.:
self.SCALE_CLIMB -= 1.
# project out the constraints
gc = g.copy()
for c in molecule.constraints.T:
gc -= np.dot(gc.T, c[:, np.newaxis]) * c[:, np.newaxis]
g_prim = block_matrix.dot(molecule.coord_basis, gc)
dE = molecule.difference_energy
if dE < 100.:
print(" difference energy is %5.4f" % dE)
molecule.gradrms = np.sqrt(np.dot(gc.T, gc) / num_coords)
# control step size NEW FEB 2020
if flag:
if ls['status'] == 0: # passed
dgradrms = molecule.gradrms - pgradrms
print("dgradrms ", dgradrms)
if ls['step'] > self.DMAX:
if ls['step'] <= self.options[
'abs_max_step']: # absolute max
print(" Increasing DMAX to {}".format(ls['step']))
self.DMAX = ls['step']
else:
self.DMAX = self.options['abs_max_step']
elif ls['step'] < self.DMAX:
if ls['step'] >= self.DMIN: # absolute min
print(" Decreasing DMAX to {}".format(ls['step']))
self.DMAX = ls['step']
elif ls['step'] <= self.DMIN:
self.DMAX = self.DMIN
print(" Decreasing DMAX to {}".format(self.DMIN))
elif ratio > 0.85 and ratio < 1.1 and actual_step > self.DMAX and dgradrms < -0.00005:
print(" HERE increasing DMAX")
self.DMAX *= 1.1
if self.DMAX > self.options['abs_max_step']:
self.DMAX = self.options['abs_max_step']
else:
print("status not zero")
if ostep % xyzframerate == 0:
geoms.append(molecule.geometry)
energies.append(molecule.energy - refE)
manage_xyz.write_xyzs_w_comments('opt_{}.xyz'.format(
molecule.node_id),
geoms,
energies,
scale=1.)
if self.options['print_level'] > 0:
print(
" Node: %d Opt step: %d E: %5.4f predE: %5.4f ratio: %1.3f gradrms: %1.5f ss: %1.3f DMAX: %1.3f"
% (molecule.node_id, ostep + 1, fx - refE, dEpre, ratio,
molecule.gradrms, step, self.DMAX))
self.buf.write(
u' Node: %d Opt step: %d E: %5.4f predE: %5.4f ratio: %1.3f gradrms: %1.5f ss: %1.3f DMAX: %1.3f\n'
% (molecule.node_id, ostep + 1, fx - refE, dEpre, ratio,
molecule.gradrms, step, self.DMAX))
gmax = float(np.max(np.absolute(gc)))
disp = float(np.linalg.norm((xyz - self.xyzp).flatten()))
print(" gmax %5.4f disp %5.4f dEstep %5.4f gradrms %5.4f\n" %
(gmax, disp, dEstep, molecule.gradrms))
self.converged = False
if self.opt_cross and abs(
dE
) < self.conv_dE and molecule.gradrms < self.conv_grms and abs(
gmax) < self.conv_gmax and abs(
dEstep) < self.conv_Ediff and abs(
disp) < self.conv_disp and ls['status'] == 0:
self.converged = True
elif not self.opt_cross and molecule.gradrms < self.conv_grms and abs(
gmax) < self.conv_gmax and abs(
dEstep) < self.conv_Ediff and abs(
disp) < self.conv_disp:
if self.opt_climb and opt_type == "CLIMB":
gts = np.dot(g.T, molecule.constraints[:, 0])
if abs(gts) < self.conv_grms:
self.converged = True
else:
self.converged = True
if self.converged:
print(" converged")
if ostep % xyzframerate != 0:
geoms.append(molecule.geometry)
energies.append(molecule.energy - refE)
manage_xyz.write_xyzs_w_comments('opt_{}.xyz'.format(
molecule.node_id),
geoms,
energies,
scale=1.)
break
#print " ########## DONE WITH TOTAL STEP #########"
#update DLC --> this changes q, g, Hint
if not molecule.coord_obj.__class__.__name__ == 'CartesianCoordinates':
if opt_type == 'SEAM' or opt_type == "MECI":
constraints = self.get_constraint_vectors(
molecule, opt_type, ictan)
molecule.update_coordinate_basis(constraints=constraints)
x = np.copy(molecule.coordinates)
g = molecule.gradient.copy()
# project out the constraint
gc = g.copy()
for c in molecule.constraints.T:
gc -= np.dot(gc.T, c[:, np.newaxis]) * c[:, np.newaxis]
g_prim = block_matrix.dot(molecule.coord_basis, gc)
print(" opt-summary")
print(self.buf.getvalue())
return geoms, energies
def write(configuration_space):
if not isinstance(configuration_space, ConfigurationSpace):
raise TypeError("pcs_parser.write expects an instance of %s, "
"you provided '%s'" %
(ConfigurationSpace, type(configuration_space)))
param_lines = StringIO()
condition_lines = StringIO()
forbidden_lines = []
for hyperparameter in configuration_space.get_hyperparameters():
# Check if the hyperparameter names are valid SMAC names!
try:
pp_param_name.parseString(hyperparameter.name)
except pyparsing.ParseException:
raise ValueError("Illegal hyperparameter name for SMAC: %s" %
hyperparameter.name)
# First build params
if param_lines.tell() > 0:
param_lines.write("\n")
if isinstance(hyperparameter, NumericalHyperparameter):
param_lines.write(build_continuous(hyperparameter))
elif isinstance(hyperparameter, CategoricalHyperparameter):
param_lines.write(build_categorical(hyperparameter))
elif isinstance(hyperparameter, Constant):
param_lines.write(build_constant(hyperparameter))
else:
raise TypeError("Unknown type: %s (%s)" %
(type(hyperparameter), hyperparameter))
for condition in configuration_space.get_conditions():
if condition_lines.tell() > 0:
condition_lines.write("\n")
condition_lines.write(build_condition(condition))
for forbidden_clause in configuration_space.get_forbiddens():
# Convert in-statement into two or more equals statements
dlcs = forbidden_clause.get_descendant_literal_clauses()
# First, get all in statements and convert them to equal statements
in_statements = []
other_statements = []
for dlc in dlcs:
if isinstance(dlc, MultipleValueForbiddenClause):
if not isinstance(dlc, ForbiddenInClause):
raise ValueError("SMAC cannot handle this forbidden "
"clause: %s" % dlc)
in_statements.append([
ForbiddenEqualsClause(dlc.hyperparameter, value)
for value in dlc.values
])
else:
other_statements.append(dlc)
# Second, create the product of all elements in the IN statements,
# create a ForbiddenAnd and add all ForbiddenEquals
if len(in_statements) > 0:
for i, p in enumerate(product(*in_statements)):
all_forbidden_clauses = list(p) + other_statements
f = ForbiddenAndConjunction(*all_forbidden_clauses)
forbidden_lines.append(build_forbidden(f))
else:
forbidden_lines.append(build_forbidden(forbidden_clause))
if condition_lines.tell() > 0:
condition_lines.seek(0)
param_lines.write("\n\n")
for line in condition_lines:
param_lines.write(line)
if len(forbidden_lines) > 0:
forbidden_lines.sort()
param_lines.write("\n\n")
for line in forbidden_lines:
param_lines.write(line)
param_lines.write("\n")
# Check if the default configuration is a valid configuration!
param_lines.seek(0)
return param_lines.getvalue()
def pdf_translate(pdf_path, fgmt, make_marian_conf=None, logger=None):
page_split_tag = '\n\n<<PAGE_SPLIT_TAG>>\n\n'
output_string = StringIO()
with open(pdf_path, 'rb') as in_file:
parser = PDFParser(in_file)
doc = PDFDocument(parser)
rsrcmgr = PDFResourceManager()
device = TextConverter(rsrcmgr,
output_string,
laparams=LAParams(boxes_flow=0.3,
line_margin=1.0))
interpreter = PDFPageInterpreter(rsrcmgr, device)
for idx, page in enumerate(PDFPage.create_pages(doc)):
interpreter.process_page(page)
output_string.write(page_split_tag)
pdf_text = output_string.getvalue()
pdf_pages = pdf_text.split(page_split_tag)
marian_processes = []
if make_marian_conf:
marian_processes = make_marian_process(
make_marian_conf["marian_command"],
make_marian_conf["marian_args_pdf_translator"],
make_marian_conf["pdf_ports"])
ret = []
for pdf_idx, pdf_page in enumerate(pdf_pages[:-1]):
retry_max = 3
translated = None
for i in range(retry_max):
if logger:
logger.info("translate page={}".format(pdf_idx))
to_translate = pre_proc_text(pdf_page)
translated = fgmt.translate_text(to_translate)
if not fgmt.detected_marian_err:
ret.append(translated)
break
else:
translated = None
close_marian_process(marian_processes)
marian_processes = make_marian_process(
make_marian_conf["marian_command"],
make_marian_conf["marian_args_pdf_translator"],
make_marian_conf["pdf_ports"])
fgmt.detected_marian_err = False
if logger:
logger.info(fgmt.get_and_clear_logs())
logger.warning("recovery marian processes {}/{}".format(
i, retry_max - 1))
if translated is None:
ret.append(get_err_translated())
marian_processes = ckeck_restart_marian_process(
marian_processes,
make_marian_conf["max_marian_memory"],
make_marian_conf["marian_command"],
make_marian_conf["marian_args_pdf_translator"],
make_marian_conf["pdf_ports"],
logger=logger)
if logger:
logger.info(fgmt.get_and_clear_logs())
if make_marian_conf:
close_marian_process(marian_processes)
return ret
class CravatConverter(BaseConverter):
def __init__(self):
self.format_name = 'vcf'
self._in_header = True
self._first_variant = True
self._buffer = StringIO()
self._reader = None
self.addl_cols = [
{'name':'phred','title':'Phred','type':'string'},
{'name':'filter','title':'VCF filter','type':'string'},
{'name':'zygosity','title':'Zygosity','type':'string'},
{'name':'alt_reads','title':'Alternate reads','type':'int'},
{'name':'tot_reads','title':'Total reads','type':'int'},
{'name':'af','title':'Variant allele frequency','type':'float'},
{'name':'hap_block','title':'Haplotype block ID','type':'int'},
{'name':'hap_strand','title':'Haplotype strand ID','type':'int'},
]
self.ex_info_writer = None
self.curvar = None
self.csq_fields = None
def check_format(self, f):
if f.name.endswith('.vcf'):
return True
if f.name.endswith('.vcf.gz'):
return True
first_line = f.readline()
if first_line.startswith('##fileformat=VCF'):
return True
def setup(self, f):
if hasattr(self, 'conf') == False:
self.conf = {}
if type(self.conf.get('exclude_info')) == str:
self.exclude_info = set(self.conf['exclude_info'].split(','))
else:
self.exclude_info = set()
if type(self.conf.get('include_info')) == str:
self.include_info = set(self.conf['include_info'].split(','))
else:
self.include_info = set()
import vcf
reader = vcf.Reader(f, compressed=False)
self.fix_formats(reader)
self.open_extra_info(reader)
def fix_formats(self, reader):
return
# A user had AD number=1
if 'AD' in reader.formats:
reader.formats['AD']._replace(num = -3)
def open_extra_info(self, reader):
#if not reader.infos:
# return
writer_path = Path(self.output_dir)/(self.run_name+'.extra_vcf_info.var')
self.ex_info_writer = CravatWriter(str(writer_path))
info_cols = [{'name':'uid','title':'UID','type':'int'}]
info_cols.append({
'name': 'pos',
'title': 'VCF Position',
'desc': '',
'type': 'int',
'width': 60,
})
info_cols.append({
'name': 'ref',
'title': 'VCF Ref Allele',
'desc': '',
'type': 'string',
'width': 60,
})
info_cols.append({
'name': 'alt',
'title': 'VCF Alt Allele',
'desc': '',
'type': 'string',
'width': 60,
})
typemap = {'Integer':'int','Float':'float'}
if reader.infos:
for info in reader.infos.values():
info_cols.append({
'name': info.id,
'title': info.id,
'desc': info.desc,
'type': typemap.get(info.type,'string'),
'hidden': True,
})
if 'CSQ' in reader.infos:
csq_info = reader.infos['CSQ']
fields_match = re.search(r'Format: ([^\s]+)', csq_info.desc)
if fields_match:
self.csq_fields = ['CSQ_'+x for x in fields_match.group(1).split('|')]
for cname in self.csq_fields:
info_cols.append({
'name': cname,
'title': cname.replace('_',' '),
'type': 'string',
'hidden': True,
})
if self.include_info:
self.include_info.update([c['name'] for c in info_cols[:3]])
temp = info_cols
info_cols = []
for col in temp:
if col['name'] in self.include_info and col['name'] not in self.exclude_info:
col['hidden'] = False
info_cols.append(col)
del temp
else:
temp = info_cols
info_cols = []
for col in temp:
if col['name'] not in self.exclude_info:
col['hidden'] = False
info_cols.append(col)
del temp
self.info_cols = [v['name'] for v in info_cols]
self.ex_info_writer.add_columns(info_cols)
self.ex_info_writer.write_definition()
self.ex_info_writer.write_meta_line('name', 'extra_vcf_info')
self.ex_info_writer.write_meta_line('displayname', 'Extra VCF INFO Annotations')
def convert_line(self, l):
import vcf
if l.startswith('#'):
if self._in_header:
self._buffer.write(l)
return self.IGNORE
if self._first_variant:
self._first_variant = False
self._in_header = False
self._buffer.seek(0)
self._reader = vcf.Reader(self._buffer)
self._buffer.seek(0)
self._buffer.truncate()
self._buffer.write(l)
self._buffer.seek(0)
try:
variant = next(self._reader)
except StopIteration:
return self.IGNORE
wdict_blanks = {}
for alt_index, alt in enumerate(variant.ALT):
if alt is None:
alt_base = variant.REF
elif alt.type == 'NON_REF':
alt_base = None
else:
alt_base = alt.sequence
new_pos, new_ref, new_alt = self.trim_variant(variant.POS, variant.REF, alt_base)
#new_pos, new_ref, new_alt = variant.POS, variant.REF, alt_base
if variant.FILTER is None:
filter_val = None
elif len(variant.FILTER) == 0:
filter_val = 'PASS'
else:
filter_val = ';'.join(variant.FILTER)
wdict_blanks[alt_index+1] = {
'chrom': variant.CHROM,
'pos': new_pos,
'ref_base': new_ref,
'alt_base': new_alt,
'tags': variant.ID,
'phred': variant.QUAL,
'filter': filter_val,
}
wdicts = []
self.gt_occur = []
if len(variant.samples) > 0:
all_gt_zero = True
for call in variant.samples:
# Dedup gt but maintain order
for gt in list(OrderedDict.fromkeys(call.gt_alleles)):
if gt == '0' or gt is None:
continue
all_gt_zero = False
gt = int(gt)
wdict = copy.copy(wdict_blanks[gt])
if wdict['alt_base'] == '*':
continue
wdict['sample_id'] = call.sample
if call.is_het == True:
wdict['zygosity'] = 'het'
elif call.is_het == False:
wdict['zygosity'] = 'hom'
elif call.is_het is None:
wdict['zygosity'] = None
else:
wdict['zygosity'] = None
wdict['tot_reads'], wdict['alt_reads'], wdict['af'] = self.extract_read_info(call, gt)
wdict['hap_block'] = None #TODO
wdict['hap_strand'] = None #TODO
wdicts.append(wdict)
self.gt_occur.append(gt)
if all_gt_zero:
raise BadFormatError('All samples have the reference genotype.')
else:
for gt in wdict_blanks:
wdict = copy.copy(wdict_blanks[gt])
if wdict['alt_base'] == '*':
continue
wdicts.append(wdict)
self.gt_occur.append(gt)
self.curvar = variant
self.cur_csq = {}
if self.csq_fields and 'CSQ' in variant.INFO:
csq_entries = defaultdict(list)
for gt_csq in variant.INFO['CSQ']:
l = gt_csq.split('|')
csq_entries[l[0]].append(l)
for allele, entries in csq_entries.items():
transpose = zip(*entries)
self.cur_csq[allele] = dict([(cname, self.csq_format(value)) for cname, value in zip(self.csq_fields, transpose)])
return wdicts
@staticmethod
def extract_read_info(call, gt):
tot_reads = None
alt_reads = None
# AD is depth for each allele
if hasattr(call.data,'AD'):
# tot_reads
if hasattr(call.data.AD, '__iter__'):
tot_reads = sum([0 if x is None else int(x) for x in call.data.AD])
elif call.data.AD is None:
tot_reads = 0
else:
tot_reads = int(call.data.AD)
# alt_reads
try:
alt_reads = int(call.data.AD[gt])
except IndexError: # Wrong length
alt_reads = None
except TypeError: # Not indexable
alt_reads = int(call.data.AD)
# DP is total depth
if hasattr(call.data,'DP'):
tot_reads = int(call.data.DP)
if tot_reads is not None and alt_reads is not None:
try:
alt_freq = alt_reads/tot_reads
except ZeroDivisionError:
alt_freq = None
else:
alt_freq = None
return tot_reads, alt_reads, alt_freq
@staticmethod
def csq_format(l):
# Format a list of CSQ values into it's representation in OC
# Each value comes from a VEP transcript mapping
if all([x=='' for x in l]):
return None
else:
return ';'.join(l)
def trim_variant(self, pos, ref, alt):
if alt is None:
return pos, ref, alt
if len(ref) == 1 and len(alt) == 1:
return pos, ref, alt
ref = list(ref)
alt = list(alt)
adj = 0
while ref and alt and ref[0]==alt[0]:
adj += 1
ref.pop(0)
alt.pop(0)
while ref and alt and ref[-1]==alt[-1]:
ref.pop()
alt.pop()
ref = ''.join(ref) if ref else '-'
alt = ''.join(alt) if alt else '-'
return pos+adj, ref, alt
@staticmethod
def oc_info_val(info_type, val, force_str=False):
if val is None or val=='.':
oc_val = None
if info_type in ('Integer','Float'):
if isnan(val):
oc_val = None
else:
oc_val = val
else:
oc_val = val
if force_str and oc_val is None:
return '.'
elif force_str:
return str(oc_val)
else:
return oc_val
def addl_operation_for_unique_variant (self, wdict, wdict_no):
if self.ex_info_writer is None:
return
gt = self.gt_occur[wdict_no]
alt_index = gt-1
row_data = {'uid':wdict['uid']}
for info_name, info_val in self.curvar.INFO.items():
if info_name not in self.info_cols:
continue
info_desc = self._reader.infos[info_name]
if info_desc.num == 0:
oc_val = self.oc_info_val(info_desc.type, info_val)
elif info_desc.num == -1: # Number=A
oc_val = self.oc_info_val(info_desc.type, info_val[alt_index])
elif info_desc.num == -2: # Number=G
oc_val = None #TODO handle Number=G
elif info_desc.num == -3: # Number=R
val = info_val[gt] #TODO find an example and make sure this is right
oc_val = self.oc_info_val(info_desc.type, val)
elif info_desc.num is None: # Number=.
tmp = lambda val: self.oc_info_val(info_desc.type, val, force_str=True)
oc_val = ','.join(map(tmp, info_val))
elif info_desc.num == 1:
oc_val = self.oc_info_val(info_desc.type, info_val)
else: # Number>1
tmp = lambda val: self.oc_info_val(info_desc.type, val, force_str=True)
oc_val = ','.join(map(tmp, info_val))
row_data[info_name] = oc_val
alt = self.curvar.ALT[gt-1].sequence
row_data['pos'] = self.curvar.POS
row_data['ref'] = self.curvar.REF
row_data['alt'] = alt
if self.cur_csq:
#TODO csq alts and multiallelic sites
if len(self.curvar.ALT) == 1:
if self.cur_csq:
row_data.update(next(iter(self.cur_csq.values())))
else:
row_data.update(self.cur_csq.get(alt,{}))
self.ex_info_writer.write_data(row_data)
def write(self, s):
self._out.write(s)
StringIO.write(self, s)
def _write_summary_log(self, upgrade_id: str, requirements: UpgradeRequirements):
try:
Path(self.LOGS_DIR).mkdir(parents=True, exist_ok=True)
summary_text = StringIO()
summary_text.write('Upgrade summary ( id: {} )'.format(upgrade_id))
if requirements.cannot_upgrade:
summary_text.write('\nCannot upgrade:')
for dep in requirements.cannot_upgrade:
type_label = self.i18n.get('gem.{}.type.{}.label'.format(dep.pkg.gem_name, dep.pkg.get_type().lower()), dep.pkg.get_type().capitalize())
summary_text.write('\n * Type:{}\tName: {}\tVersion: {}\tReason: {}'.format(type_label, dep.pkg.name, dep.pkg.version if dep.pkg.version else '?', dep.reason if dep.reason else '?'))
summary_text.write('\n')
if requirements.to_remove:
summary_text.write('\nMust be removed:')
for dep in requirements.to_remove:
type_label = self.i18n.get('gem.{}.type.{}.label'.format(dep.pkg.gem_name, dep.pkg.get_type().lower()), dep.pkg.get_type().capitalize())
summary_text.write('\n * Type:{}\tName: {}\tVersion: {}\tReason: {}'.format(type_label, dep.pkg.name, dep.pkg.version if dep.pkg.version else '?', dep.reason if dep.reason else '?'))
summary_text.write('\n')
if requirements.to_install:
summary_text.write('\nMust be installed:')
for dep in requirements.to_install:
type_label = self.i18n.get(
'gem.{}.type.{}.label'.format(dep.pkg.gem_name, dep.pkg.get_type().lower()),
dep.pkg.get_type().capitalize())
summary_text.write('\n * Type:{}\tName: {}\tVersion: {}\tReason: {}'.format(type_label,
dep.pkg.name,
dep.pkg.version if dep.pkg.version else '?',
dep.reason if dep.reason else '?'))
summary_text.write('\n')
if requirements.to_upgrade:
summary_text.write('\nWill be upgraded:')
for dep in requirements.to_upgrade:
type_label = self.i18n.get('gem.{}.type.{}.label'.format(dep.pkg.gem_name, dep.pkg.get_type().lower()), dep.pkg.get_type().capitalize())
summary_text.write(
'\n * Type:{}\tName: {}\tVersion:{} \tNew version: {}'.format(type_label,
dep.pkg.name,
dep.pkg.version if dep.pkg.version else '?',
dep.pkg.latest_version if dep.pkg.latest_version else '?'))
summary_text.write('\n')
summary_text.seek(0)
with open(self.SUMMARY_FILE.format(upgrade_id), 'w+') as f:
f.write(summary_text.read())
except:
traceback.print_exc()
# -*- coding:utf-8 -*-
# Product Name: PyCharm
# Project Name: proj
# File Name: stringio.py
# Date: 2022/02/15 00:01:53
# Author: wangyuankun
# Function: This file is to achieve xxx function!
from io import StringIO
f = StringIO()
f.write('hello')
f.write(' ')
f.write('world')
print(f.getvalue())
def to_python(s):
L = LOLLexer()
L.input(s)
header = StringIO()
header.write(HEADER)
header_output = IndentWriter(header)
body = StringIO()
body.write(BODY)
body_output = IndentWriter(body)
write = body_output.write
output = body_output
for t in iter(L.token_stream):
if t.type == "NAME":
# Need to escape names which are Python variables Do that
# by appending an "_". But then I also need to make sure
# that "yield_" does not collide with "yield". And you
# thought you were being clever trying to use a Python
# variable. :)
name = t.value.rstrip("_")
if name in keyword.kwlist:
write(t.value + "_ ")
else:
write(t.value + " ")
elif t.type in ("RESERVED", "OP", "NUMBER", "CLOSE"):
# While not pretty, I'll put a space after each
# term because it's the simplest solution. Otherwise
# I'll need to track the amount of whitespace between
# the tokens in the original text.
write(t.value + " ")
# XXX escape names which are special in Python!
elif t.type == "STRING":
write(repr(t.value) + " ")
elif t.type == "COMMENT":
# Not enough information to keep comments on the correct
# indentation level. This is good enough. Ugly though.
# Maybe I need to fix the tokenizer.
write("#" + t.value[3:] + "\n")
output.at_first_column = True
elif t.type == "COLON":
write(":")
elif t.type == "INDENT":
output.indent += 1
pass
elif t.type == "DEDENT":
output.indent -= 1
pass
elif t.type == "NEWLINE":
write(t.value)
output.at_first_column = True
output = body_output
write = output.write
elif t.type == "PRINT":
if t.value == "stdout":
t.lexer.paren_stack.append(')')
write("print( ")
elif t.value == "stderr":
#write("print >>_lol_sys.stderr, ")
t.lexer.paren_stack.append(')')
write("sys.stderr.write( ")
else:
raise AssertionError(t.value)
elif t.type == "AUTOCALL":
write(t.value + "(")
elif t.type == "INLINE":
write(t.value)
elif t.type == "ENDMARKER":
write("\n# The end.\n")
elif t.type == "WS":
output.leading_ws = t.value
elif t.type == "FUTURE":
# Write to the header. This is a hack. Err, a hairball.
output = header_output
write = output.write
write("from __future__ import ")
else:
raise AssertionError(t.type)
return header.getvalue() + body.getvalue()
def run_cmd(self,
cmd,
username=None,
raise_if_error=True,
continuous_output=False,
silent=False,
timeout=None,
input_data=None,
success_exit_code=0,
retry=0,
retry_interval=5,
keep_retry_history=False):
""" Run command on the remote host and return result locally
:param cmd: command to execute on remote host
cmd can be a str or a list of str
:param username: user used to execute the command (sudo privilege needed)
:param raise_if_error:
if True, raise SSHException when exit code of the command is different from 0
else just return exit code and command output
:param continuous_output: if True, print output all along the command is running
:param silent:
if True, does not log the command run (useful if sensitive information are used in command)
if parameter is a list, all strings of the command matching an item of the list will be concealed
in logs (regexp supported)
:param timeout: length in seconds after what a TimeoutError exception is raised
:param input_data:
key/value dictionary used when remote command expects input from user
when key is matching command output, value is sent
:param success_exit_code: integer or list of integer considered as a success exit code for command run
:param retry: number of retry until exit code is part of successful exit code list (-1 for infinite retry) or
RunCmdError exception is raised
:param retry_interval: number of seconds between each retry
:param keep_retry_history: if True, all retries results are kept and accessible in return result
default is False as we don't want to save by default all output for all retries especially for big output
:raises TimeoutError: if command run longer than the specified timeout
:raises TypeError: if `cmd` parameter is neither a string neither a list of string
:raises SSHException: if current SSHSession is already closed
:raises RunCmdError: if exit code of the command is different from 0 and raise_if_error is True
:return: a class inheriting from collections.namedtuple containing mainly `exit_code` and `output`
of the remotely executed command
:rtype: RunCmdResult
Usage::
>>> from jumpssh import SSHSession
>>> with SSHSession('gateway.example.com', 'my_user', password='******') as ssh_session:
>>> ... ssh_session.run_cmd('hostname')
RunSSHCmdResult(exit_code=0, output=u'gateway.example.com')
"""
user = self.username
# check type of command parameter is valid
try:
string_type = basestring
except NameError:
string_type = str
if isinstance(cmd, list):
cmd = " && ".join(cmd)
elif not isinstance(cmd, string_type):
raise TypeError("Invalid type for cmd argument '%s'" % type(cmd))
# success_exit_code must be int or list of int
if isinstance(success_exit_code, int):
success_exit_code = [success_exit_code]
elif not isinstance(success_exit_code, list):
raise TypeError(
"Invalid type for success_exit_code argument '%s'" %
type(success_exit_code))
my_cmd = cmd
if username:
user = username
# need to run full command with shell to support shell builtins commands (source, ...)
my_cmd = 'sudo su - %s -c "%s"' % (user, cmd.replace('"', '\\"'))
# check session is still active before running a command, else try to open it
if not self.is_active():
self.open()
# conceal text from command to be logged if requested with silent parameter
cmd_for_log = cmd
if isinstance(silent, list):
for pattern in silent:
cmd_for_log = re.sub(pattern=pattern,
repl='XXXXXXX',
string=cmd_for_log)
if silent is not True:
logger.debug("Running command '%s' on '%s' as %s..." %
(cmd_for_log, self.host, user))
# keep track of all results for each run to make them available in response object
result_list = []
# retry command until exit_code in success code list or max retry nb reached
retry_nb = 0
while True:
channel = self.ssh_transport.open_session()
# raise error rather than blocking the call
channel.setblocking(0)
# Forward local agent
paramiko.agent.AgentRequestHandler(channel)
# Commands executed after this point will see the forwarded agent on the remote end.
channel.set_combine_stderr(True)
channel.get_pty()
channel.exec_command(my_cmd)
# prepare timer for timeout
start = datetime.datetime.now()
start_secs = time.mktime(start.timetuple())
output = StringIO()
try:
# wait until command finished running or timeout is reached
while True:
got_chunk = False
readq, _, _ = select.select([channel], [], [], timeout)
for c in readq:
if c.recv_ready():
data = channel.recv(len(
c.in_buffer)).decode('utf-8')
output.write(data)
got_chunk = True
# print output all along the command is running
if not silent and continuous_output and len(
data) > 0:
print(data)
if input_data and channel.send_ready():
# We received a potential prompt.
for pattern in input_data.keys():
# pattern text matching current output => send input data
if re.search(pattern, data):
channel.send(input_data[pattern] +
'\n')
# remote process has exited and returned an exit status
if not got_chunk and channel.exit_status_ready(
) and not channel.recv_ready():
channel.shutdown_read(
) # indicate that we're not going to read from this channel anymore
channel.close()
break # exit as remote side is finished and our buffers are empty
# Timeout check
if timeout:
now = datetime.datetime.now()
now_secs = time.mktime(now.timetuple())
et_secs = now_secs - start_secs
if et_secs > timeout:
raise exception.TimeoutError(
"Timeout of %ds reached when calling command '%s'. "
"Increase timeout if you think the command was still running successfully."
% (timeout, cmd_for_log))
except KeyboardInterrupt:
# if channel still active, forward Ctrl-C to remote host if requested by user
if self.is_active() and util.yes_no_query(
"Terminate remote command '%s'?" % cmd_for_log,
default=True,
interrupt=False):
# channel has been closed pending response from user, we don't have access to remote server anymore
if channel.closed:
exit_code = channel.recv_exit_status()
if exit_code == -1:
logger.warning(
"Unable to terminate remote command because channel is closed."
)
else:
logger.info(
"Remote command execution already finished with exit code %s"
% exit_code)
else:
# forward Ctrl-C to remote host
channel.send('\x03')
channel.close()
raise
exit_code = channel.recv_exit_status()
output_value = output.getvalue().strip()
# keep result of all runs is result, not only the last one
if keep_retry_history:
result_list.append(
RunSSHCmdResult(exit_code=exit_code, output=output_value))
if exit_code in success_exit_code:
# command ran successfully, no retry needed
break
else:
if retry < 0 or retry_nb < retry:
retry_nb += 1
time.sleep(retry_interval)
continue
# max retry reached and exception must be raised
elif raise_if_error:
raise exception.RunCmdError(
exit_code=exit_code,
success_exit_code=success_exit_code,
command=cmd_for_log,
error=output_value,
runs_nb=retry_nb + 1)
else:
# command ended in error but max retry already reached and no exception must be raised
break
# return result of run command + all information used to run the command
return RunCmdResult(exit_code=exit_code,
output=output_value,
result_list=result_list,
command=cmd_for_log,
success_exit_code=success_exit_code,
runs_nb=retry_nb + 1)
def make_gdre_icons_action(target, source, env):
dst = target[0]
svg_icons = source
icons_string = StringIO()
for f in svg_icons:
fname = str(f)
icons_string.write('\t"')
with open(fname, "rb") as svgf:
b = svgf.read(1)
while len(b) == 1:
icons_string.write("\\" + str(hex(ord(b)))[1:])
b = svgf.read(1)
icons_string.write('"')
if fname != svg_icons[-1]:
icons_string.write(",")
icons_string.write("\n")
s = StringIO()
s.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n\n")
s.write('#include "modules/modules_enabled.gen.h"\n\n')
s.write("#ifndef _GDRE_ICONS_H\n")
s.write("#define _GDRE_ICONS_H\n")
s.write("static const int gdre_icons_count = {};\n\n".format(
len(svg_icons)))
s.write("#ifdef MODULE_SVG_ENABLED\n")
s.write("static const char *gdre_icons_sources[] = {\n")
s.write(icons_string.getvalue())
s.write("};\n")
s.write("#endif // MODULE_SVG_ENABLED\n\n")
s.write("static const char *gdre_icons_names[] = {\n")
for f in svg_icons:
fname = str(f)
# Trim the `.svg` extension from the string.
icon_name = os.path.basename(fname)[:-4]
s.write('\t"{0}"'.format(icon_name))
if fname != svg_icons[-1]:
s.write(",")
s.write("\n")
s.write("};\n")
s.write("#endif\n")
with open(dst, "w") as f:
f.write(s.getvalue())
s.close()
icons_string.close()
def saveAs(self, newfile, options=None):
"""Saves Jpeg with IPTC data to a given file name."""
assert self._filename is not None
print("saveAs: self = %s" % self, file=sys.stderr)
# Open file and snarf data from it.
fh = self._getfh()
if not self.fileIsJpeg(fh):
self.log(
"Source file is not a Jpeg; I can only save Jpegs. Sorry.")
return None
ret = self.jpegCollectFileParts(fh, options)
self._closefh(fh)
if ret is None:
self.log("collectfileparts failed")
print(self.error, file=sys.stderr)
raise Exception('collectfileparts failed')
(start, end, adobe) = ret
debug(2, 'start: %d, end: %d, adobe:%d' % tuple(map(len, ret)))
self.hexDump(start), len(end)
debug(3, 'adobe1', adobe)
if options is not None and 'discardAdobeParts' in options:
adobe = None
debug(3, 'adobe2', adobe)
debug(1, 'writing...')
# fh = os.tmpfile() ## 20051011 - Windows doesn't like tmpfile ##
# Open dest file and stuff data there
# fh.truncate()
# fh.seek(0, 0)
# debug(2, self._filepos(fh))
fh = StringIO()
if not fh:
self.log("Can't open output file")
return None
debug(3, len(start), len(end))
fh.write(start)
# character set
ch = self.c_charset_r.get(
(self.out_charset is None and [self.inp_charset]
or [self.out_charset])[0], None)
# writing the character set is not the best practice - couldn't find the needed place (record) for it yet!
if SURELY_WRITE_CHARSET_INFO and ch is not None:
fh.write(pack("!BBBHH", 0x1c, 1, 90, 4, ch))
debug(2, self._filepos(fh))
#$self->PhotoshopIIMBlock($adobe, $self->PackedIIMData());
data = self.photoshopIIMBlock(adobe, self.packedIIMData())
debug(3, len(data), self.hexDump(data))
fh.write(data)
debug(2, self._filepos(fh))
fh.flush()
fh.write(end)
debug(2, self._filepos(fh))
fh.flush()
#copy the successfully written file back to the given file
fh2 = file(newfile, 'wb')
fh2.truncate()
fh2.seek(0, 0)
fh.seek(0, 0)
while 1:
buf = fh.read(8192)
if buf is None or len(buf) == 0: break
fh2.write(buf)
self._closefh(fh)
fh2.flush()
fh2.close()
return True
ATENÇÃO: Para ler ou escrever dados em arquivos do sistema operacional
o software precisa ter permissão:
- Permissão de leitura -> Para ler o arquivo.
- Permissão de escrita -> Para escrever no arquino
StringIO -> Utilizado para ler e criar arquivos em memoria.
ele não vai gravar no disco vai ficar apenas na memoria
Para utilizar o stringIO primeiro fazemos o import
"""
# -*- coding: UTF-8 -*-
# Import
from io import StringIO
mensagem = 'Esta é apenas uma string normal'
# Podemos criar um arquivo em memoria ja com uma string inserida ou
# mesmo vazio para inserirmos texto depois
arquivo = StringIO(mensagem)
# Agora tendo o arquivo podemos utilizar tudo que ja sabemos
print(arquivo.read())
# Escrevendo outros textos
arquivo.write('Outro texto')
# Podemos inclusive movimentar o cursor
arquivo.seek(0)
print(arquivo.read())
def changelog(ctx, check, version, old_version, initial, quiet, dry_run,
output_file, tag_prefix):
"""Perform the operations needed to update the changelog.
This method is supposed to be used by other tasks and not directly.
"""
if check and check not in get_valid_checks():
abort(f'Check `{check}` is not an Agent-based Integration')
# sanity check on the version provided
cur_version = old_version or get_version_string(check,
tag_prefix=tag_prefix)
if parse_version_info(version.replace(
tag_prefix, '', 1)) <= parse_version_info(
cur_version.replace(tag_prefix, '', 1)):
abort(f'Current version is {cur_version}, cannot bump to {version}')
if not quiet:
echo_info(
f'Current version of check {check}: {cur_version}, bumping to: {version}'
)
# get the name of the current release tag
target_tag = get_release_tag_string(check, cur_version)
# get the diff from HEAD
diff_lines = get_commits_since(check, None if initial else target_tag)
# for each PR get the title, we'll use it to populate the changelog
pr_numbers = parse_pr_numbers(diff_lines)
if not quiet:
echo_info(
f'Found {len(pr_numbers)} PRs merged since tag: {target_tag}')
if initial:
# Only use the first one
del pr_numbers[:-1]
user_config = ctx.obj
entries = defaultdict(list)
for pr_num in pr_numbers:
try:
payload = get_pr(pr_num, user_config)
except Exception as e:
echo_failure(f'Unable to fetch info for PR #{pr_num}: {e}')
continue
changelog_labels = get_changelog_types(payload)
if not changelog_labels:
abort(
f'No valid changelog labels found attached to PR #{pr_num}, please add one!'
)
elif len(changelog_labels) > 1:
abort(
f'Multiple changelog labels found attached to PR #{pr_num}, please only use one!'
)
changelog_type = changelog_labels[0]
if changelog_type == CHANGELOG_TYPE_NONE:
if not quiet:
# No changelog entry for this PR
echo_info(f'Skipping PR #{pr_num} from changelog due to label')
continue
author = payload.get('user', {}).get('login')
author_url = payload.get('user', {}).get('html_url')
title = f"[{changelog_type}] {payload.get('title')}"
entry = ChangelogEntry(pr_num, title, payload.get('html_url'), author,
author_url, from_contributor(payload))
entries[changelog_type].append(entry)
# store the new changelog in memory
new_entry = StringIO()
# the header contains version and date
header = f"## {version} / {datetime.utcnow().strftime('%Y-%m-%d')}\n"
new_entry.write(header)
# one bullet point for each PR
new_entry.write('\n')
for changelog_type in CHANGELOG_TYPES_ORDERED:
for entry in entries[changelog_type]:
thanks_note = ''
if entry.from_contributor:
thanks_note = f' Thanks [{entry.author}]({entry.author_url}).'
new_entry.write(
f'* {entry.title}. See [#{entry.number}]({entry.url}).{thanks_note}\n'
)
new_entry.write('\n')
# read the old contents
if check:
changelog_path = os.path.join(get_root(), check, output_file)
else:
changelog_path = os.path.join(get_root(), output_file)
old = list(stream_file_lines(changelog_path))
# write the new changelog in memory
changelog_buffer = StringIO()
# preserve the title
changelog_buffer.write(''.join(old[:2]))
# prepend the new changelog to the old contents
# make the command idempotent
if header not in old:
changelog_buffer.write(new_entry.getvalue())
# append the rest of the old changelog
changelog_buffer.write(''.join(old[2:]))
# print on the standard out in case of a dry run
if dry_run:
echo_info(changelog_buffer.getvalue())
else:
# overwrite the old changelog
write_file(changelog_path, changelog_buffer.getvalue())
def gen_mobile_code(length=6):
"""生成指定长度的手机验证码"""
code = StringIO()
for _ in range(length):
code.write(str(random.randint(0, 9)))
return code.getvalue()
def launch_ec2(params_list, exp_prefix, docker_image, code_full_path,
python_command="python",
script='scripts/run_experiment.py',
aws_config=None, dry=False, terminate_machine=True, use_gpu=False, sync_s3_pkl=False,
sync_log_on_termination=True,
periodic_sync=True, periodic_sync_interval=15):
if len(params_list) == 0:
return
default_config = dict(
image_id=config.AWS_IMAGE_ID,
instance_type=config.AWS_INSTANCE_TYPE,
key_name=config.AWS_KEY_NAME,
spot=config.AWS_SPOT,
spot_price=config.AWS_SPOT_PRICE,
iam_instance_profile_name=config.AWS_IAM_INSTANCE_PROFILE_NAME,
security_groups=config.AWS_SECURITY_GROUPS,
security_group_ids=config.AWS_SECURITY_GROUP_IDS,
network_interfaces=config.AWS_NETWORK_INTERFACES,
)
if aws_config is None:
aws_config = dict()
aws_config = dict(default_config, **aws_config)
sio = StringIO()
sio.write("#!/bin/bash\n")
sio.write("{\n")
sio.write("""
die() { status=$1; shift; echo "FATAL: $*"; exit $status; }
""")
sio.write("""
EC2_INSTANCE_ID="`wget -q -O - http://169.254.169.254/latest/meta-data/instance-id`"
""")
sio.write("""
aws ec2 create-tags --resources $EC2_INSTANCE_ID --tags Key=Name,Value={exp_name} --region {aws_region}
""".format(exp_name=params_list[0].get("exp_name"), aws_region=config.AWS_REGION_NAME))
if config.LABEL:
sio.write("""
aws ec2 create-tags --resources $EC2_INSTANCE_ID --tags Key=owner,Value={label} --region {aws_region}
""".format(label=config.LABEL, aws_region=config.AWS_REGION_NAME))
sio.write("""
aws ec2 create-tags --resources $EC2_INSTANCE_ID --tags Key=exp_prefix,Value={exp_prefix} --region {aws_region}
""".format(exp_prefix=exp_prefix, aws_region=config.AWS_REGION_NAME))
sio.write("""
service docker start
""")
sio.write("""
docker --config /home/ubuntu/.docker pull {docker_image}
""".format(docker_image=docker_image))
if config.FAST_CODE_SYNC:
sio.write("""
aws s3 cp {code_full_path} /tmp/rllab_code.tar.gz --region {aws_region}
""".format(code_full_path=code_full_path, local_code_path=config.DOCKER_CODE_DIR,
aws_region=config.AWS_REGION_NAME))
sio.write("""
mkdir -p {local_code_path}
""".format(code_full_path=code_full_path, local_code_path=config.DOCKER_CODE_DIR,
aws_region=config.AWS_REGION_NAME))
sio.write("""
tar -zxvf /tmp/rllab_code.tar.gz -C {local_code_path}
""".format(code_full_path=code_full_path, local_code_path=config.DOCKER_CODE_DIR,
aws_region=config.AWS_REGION_NAME))
else:
sio.write("""
aws s3 cp --recursive {code_full_path} {local_code_path} --region {aws_region}
""".format(code_full_path=code_full_path, local_code_path=config.DOCKER_CODE_DIR,
aws_region=config.AWS_REGION_NAME))
s3_mujoco_key_path = config.AWS_CODE_SYNC_S3_PATH + '/.mujoco/'
sio.write("""
aws s3 cp --recursive {} {} --region {}
""".format(s3_mujoco_key_path, config.MUJOCO_KEY_PATH, config.AWS_REGION_NAME))
sio.write("""
cd {local_code_path}
""".format(local_code_path=config.DOCKER_CODE_DIR))
for params in params_list:
log_dir = params.get("log_dir")
remote_log_dir = params.pop("remote_log_dir")
env = params.pop("env", None)
sio.write("""
aws ec2 create-tags --resources $EC2_INSTANCE_ID --tags Key=Name,Value={exp_name} --region {aws_region}
""".format(exp_name=params.get("exp_name"), aws_region=config.AWS_REGION_NAME))
sio.write("""
mkdir -p {log_dir}
""".format(log_dir=log_dir))
if periodic_sync:
if sync_s3_pkl:
sio.write("""
while /bin/true; do
aws s3 sync --exclude '*' --include '*.csv' --include '*.json' --include '*.pkl' {log_dir} {remote_log_dir} --region {aws_region}
sleep {periodic_sync_interval}
done & echo sync initiated""".format(log_dir=log_dir, remote_log_dir=remote_log_dir,
aws_region=config.AWS_REGION_NAME,
periodic_sync_interval=periodic_sync_interval))
else:
sio.write("""
while /bin/true; do
aws s3 sync --exclude '*' --include '*.csv' --include '*.json' {log_dir} {remote_log_dir} --region {aws_region}
sleep {periodic_sync_interval}
done & echo sync initiated""".format(log_dir=log_dir, remote_log_dir=remote_log_dir,
aws_region=config.AWS_REGION_NAME,
periodic_sync_interval=periodic_sync_interval))
if sync_log_on_termination:
sio.write("""
while /bin/true; do
if [ -z $(curl -Is http://169.254.169.254/latest/meta-data/spot/termination-time | head -1 | grep 404 | cut -d \ -f 2) ]
then
logger "Running shutdown hook."
aws s3 cp /home/ubuntu/user_data.log {remote_log_dir}/stdout.log --region {aws_region}
aws s3 cp --recursive {log_dir} {remote_log_dir} --region {aws_region}
break
else
# Spot instance not yet marked for termination.
sleep 5
fi
done & echo log sync initiated
""".format(log_dir=log_dir, remote_log_dir=remote_log_dir, aws_region=config.AWS_REGION_NAME))
sio.write("""
{command}
""".format(command=to_docker_command(params, docker_image, python_command=python_command, script=script,
use_gpu=use_gpu, env=env,
local_code_dir=config.DOCKER_CODE_DIR)))
sio.write("""
aws s3 cp --recursive {log_dir} {remote_log_dir} --region {aws_region}
""".format(log_dir=log_dir, remote_log_dir=remote_log_dir, aws_region=config.AWS_REGION_NAME))
sio.write("""
aws s3 cp /home/ubuntu/user_data.log {remote_log_dir}/stdout.log --region {aws_region}
""".format(remote_log_dir=remote_log_dir, aws_region=config.AWS_REGION_NAME))
if terminate_machine:
sio.write("""
EC2_INSTANCE_ID="`wget -q -O - http://169.254.169.254/latest/meta-data/instance-id || die \"wget instance-id has failed: $?\"`"
aws ec2 terminate-instances --instance-ids $EC2_INSTANCE_ID --region {aws_region}
""".format(aws_region=config.AWS_REGION_NAME))
sio.write("} >> /home/ubuntu/user_data.log 2>&1\n")
full_script = dedent(sio.getvalue())
import boto3
import botocore
if aws_config["spot"]:
ec2 = boto3.client(
"ec2",
region_name=config.AWS_REGION_NAME,
aws_access_key_id=config.AWS_ACCESS_KEY,
aws_secret_access_key=config.AWS_ACCESS_SECRET,
)
else:
ec2 = boto3.resource(
"ec2",
region_name=config.AWS_REGION_NAME,
aws_access_key_id=config.AWS_ACCESS_KEY,
aws_secret_access_key=config.AWS_ACCESS_SECRET,
)
if len(full_script) > 10000 or len(base64.b64encode(full_script.encode()).decode("utf-8")) > 10000:
# Script too long; need to upload script to s3 first.
# We're being conservative here since the actual limit is 16384 bytes
s3_path = upload_file_to_s3(full_script)
sio = StringIO()
sio.write("#!/bin/bash\n")
sio.write("""
aws s3 cp {s3_path} /home/ubuntu/remote_script.sh --region {aws_region} && \\
chmod +x /home/ubuntu/remote_script.sh && \\
bash /home/ubuntu/remote_script.sh
""".format(s3_path=s3_path, aws_region=config.AWS_REGION_NAME))
user_data = dedent(sio.getvalue())
else:
user_data = full_script
instance_args = dict(
ImageId=aws_config["image_id"],
KeyName=aws_config["key_name"],
UserData=user_data,
InstanceType=aws_config["instance_type"],
EbsOptimized=True,
SecurityGroups=aws_config["security_groups"],
SecurityGroupIds=aws_config["security_group_ids"],
NetworkInterfaces=aws_config["network_interfaces"],
IamInstanceProfile=dict(
Name=aws_config["iam_instance_profile_name"],
),
)
if aws_config.get("placement", None) is not None:
instance_args["Placement"] = aws_config["placement"]
if not aws_config["spot"]:
instance_args["MinCount"] = 1
instance_args["MaxCount"] = 1
print("************************************************************")
print(instance_args["UserData"])
print("************************************************************")
if aws_config["spot"]:
instance_args["UserData"] = base64.b64encode(instance_args["UserData"].encode()).decode("utf-8")
spot_args = dict(
DryRun=dry,
InstanceCount=1,
LaunchSpecification=instance_args,
SpotPrice=aws_config["spot_price"],
# ClientToken=params_list[0]["exp_name"],
)
import pprint
pprint.pprint(spot_args)
if not dry:
response = ec2.request_spot_instances(**spot_args)
print(response)
spot_request_id = response['SpotInstanceRequests'][
0]['SpotInstanceRequestId']
for _ in range(10):
try:
ec2.create_tags(
Resources=[spot_request_id],
Tags=[
{'Key': 'Name', 'Value': params_list[0]["exp_name"]}
],
)
break
except botocore.exceptions.ClientError:
continue
else:
import pprint
pprint.pprint(instance_args)
ec2.create_instances(
DryRun=dry,
**instance_args
)
def verse(n):
s = StringIO()
s.write(a.format(n or 'no more', 's' if n-1 else '').capitalize())
s.write(b.format('one' if n-1 else 'it') if n else c)
s.write(d.format((n-1) or 'no more' if n else 99, 's' if (n-2) else ''))
return s.getvalue()
def get_diagnostics_html(self, base_url, force_update=False):
if not self.diagnostics_html or force_update:
fakefile = StringIO()
page_elements = self.page_elements.get()
for line in page_elements["top"]:
fakefile.write(line + "\n")
# a list of 2 part lists containing button information
button_list = [
["Force Channel Update", "chanscan"],
["debug", "debug.json"],
["device.xml", "device.xml"],
["discover.json", "discover.json"],
["lineup.json", "lineup.json"],
["lineup_status.json", "lineup_status.json"],
["cluster.json", "cluster.json"]
]
for button_item in button_list:
button_label = button_item[0]
button_path = button_item[1]
fakefile.write("<div style=\"text-align: center;\">\n")
fakefile.write(" <p><button onclick=\"OpenLink('%s')\">%s</a></button></p>\n" % (button_path, button_label))
fakefile.write("</div>\n")
fakefile.write("\n")
for line in page_elements["end"]:
fakefile.write(line + "\n")
self.diagnostics_html = fakefile.getvalue()
return self.diagnostics_html
def test_get_lines_2(self):
"""tests system control lines"""
with open('junk.bdf', 'w') as bdf_file:
bdf_file.write('CEND\n')
bdf_file.write('BEGIN BULK\n')
bdf_file.write('GRID,1\n')
bdf_file.write('ENDDATA\n')
bdf_filename = StringIO()
bdf_filename.write(
'ASSIGN FEM="junk.bdf"\n'
'ASSIGN MATRIX=junk.mat\n'
'ASSIGN OUTPUT4=junk.op4\n'
'CEND\n'
'BEGIN BULK\n'
'GRID,2'
)
bdf_filename.seek(0)
read_includes = True
dumplines = True
encoding = None
pybdf = BDFInputPy(read_includes, dumplines, encoding, nastran_format='zona',
consider_superelements=False, log=None, debug=False)
bulk_data_lines = pybdf.get_lines(bdf_filename, punch=False, make_ilines=True)[3]
#(unused_system_lines,
#unused_executive_control_lines,
#unused_case_control_lines,
#bulk_data_lines,
#unused_bulk_data_ilines) = out
#print('bulk_data_linesA =', bulk_data_lines)
assert bulk_data_lines == ['GRID,1', 'GRID,2'], bulk_data_lines
#-----------------------------------------------------------------------
bdf_filename = StringIO()
bdf_filename.write(
'ASSIGN FEM=junk.f06\n'
'CEND\n'
'BEGIN BULK\n'
'GRID,2\n'
#'ENDDATA'
)
bdf_filename.seek(0)
read_includes = True
dumplines = True
encoding = None
pybdf = BDFInputPy(read_includes, dumplines, encoding, nastran_format='zona',
consider_superelements=False, log=None, debug=False)
bulk_data_lines = pybdf.get_lines(bdf_filename, punch=False, make_ilines=True)[3]
#print('bulk_data_linesB =', bulk_data_lines)
#-----------------------------------------------------------------------
bdf_filename = StringIO()
bdf_filename.write(
"ASSIGN FEM='junk.f06'\n"
'CEND\n'
'BEGIN BULK\n'
'GRID,3\n'
#'ENDDATA'
)
bdf_filename.seek(0)
read_includes = True
dumplines = True
encoding = None
pybdf = BDFInputPy(read_includes, dumplines, encoding, nastran_format='zona',
consider_superelements=False, log=None, debug=False)
bulk_data_lines = pybdf.get_lines(bdf_filename, punch=False, make_ilines=True)[3]
#print('bulk_data_linesC =', bulk_data_lines)
os.remove('junk.bdf')
def write(self, data):
if not isinstance(data, unicode):
data = data.decode('utf-8')
StringIO.write(self, data)
