def timemap(request, response_format, url):
url = url_with_qs_and_hash(url, request.META['QUERY_STRING'])
data = memento_data_for_url(request, url)
if data:
if response_format == 'json':
response = JsonResponse(data)
elif response_format == 'html':
response = render(request, 'memento/timemap.html', data)
else:
content_type = 'application/link-format'
file = StringIO()
file.writelines(f"{line},\n" for line in [
Rel(data['original_uri'], rel='original'),
Rel(data['timegate_uri'], rel='timegate'),
Rel(data['self'], rel='self', type='application/link-format'),
Rel(data['timemap_uri']['link_format'], rel='timemap', type='application/link-format'),
Rel(data['timemap_uri']['json_format'], rel='timemap', type='application/json'),
Rel(data['timemap_uri']['html_format'], rel='timemap', type='text/html')
] + [
Rel(memento['uri'], rel='memento', datetime=datetime_to_http_date(memento['datetime'])) for memento in data['mementos']['list']
])
file.seek(0)
response = HttpResponse(file, content_type=f'{content_type}')
else:
if response_format == 'html':
response = render(request, 'memento/timemap.html', {"original_uri": url}, status=404)
else:
response = HttpResponseNotFound('404 page not found\n')
response['X-Memento-Count'] = str(len(data['mementos']['list'])) if data else 0
return response
def write_migration(file_obj, migration_class):
header = StringIO()
body = StringIO()
imports = set()
body.write("class Migration(migrations.Migration):\n\n")
for prop_name in MIGRATION_PROPS:
prop_imports, prop_serialized_value = serialize_assignment(
prop_name,
getattr(migration_class, prop_name),
level=1,
)
imports |= prop_imports
body.write(prop_serialized_value)
header.write("# -*- coding: utf-8 -*-\n")
header.write("from __future__ import unicode_literals\n")
header.write("\n")
header.write("from populus import migrations\n")
header.write("\n")
header.writelines([
"import {import_path}\n".format(import_path=import_path)
for import_path in sorted(imports)
])
file_obj.write(header.getvalue())
file_obj.write("\n")
file_obj.write(body.getvalue())
def correct_INCA_format(fp):
fp_list = list()
fp.seek(0)
if '(' in fp.readline():
for line in fp:
line = line.replace(
"(MLX::",
"").replace(
" : ",
"\t").replace(
" :",
"\t").replace(
" ",
"\t").lower().strip().replace(
")",
"\n")
if "record-by" in line:
if "image" in line:
line = "record-by\timage"
if "vector" in line:
line = "record-by\tvector"
if "dont-care" in line:
line = "record-by\tdont-care"
fp_list.append(line)
fp = StringIO()
fp.writelines(fp_list)
fp.seek(0)
return fp
def export_set(dataset): """HTML representation of a Dataset.""" stream = StringIO() page = markup.page() page.table.open() if dataset.headers is not None: new_header = [item if item is not None else '' for item in dataset.headers] page.thead.open() headers = markup.oneliner.th(new_header) page.tr(headers) page.thead.close() for row in dataset: new_row = [item if item is not None else '' for item in row] html_row = markup.oneliner.td(new_row) page.tr(html_row) page.table.close() stream.writelines(str(page)) return stream.getvalue()
def plot_single_area_all_sources(area, sw, cap_limit=2.):
print('-' * 58)
print('-' * 20 + ' ' + area + ' ' + '-' * 20)
print('= Getting data...')
pu = plotter.PlotUtils(sw.png_dir_path, cap_limit=cap_limit)
output = StringIO()
header_set = False
for name, source in sw.sources:
print('= Gathering for:', name)
states, daily_cases_path = sw.get_daily_cases(source=source)
if states is None:
with open(daily_cases_path, 'r') as f:
contents = f.readlines()
for index, line in enumerate(contents):
contents[index] = line.replace(area, area + '_' + name)
if header_set:
contents = contents[1:]
output.writelines(contents)
header_set = True
output.seek(0)
states = pd.read_csv(output,
usecols=[0, 1, 2],
index_col=['state', 'date'],
parse_dates=['date'],
squeeze=True).sort_index()
pu.plot_all_states(states,
dump_file_name='all_sources_realtime_rt',
ncols=3)
print('-' * 26 + ' DONE ' + '-' * 26)
def sort_changelog(stream):
entries = _split_changelog(stream)
log = StringIO()
for time, count, elines in sorted(entries, reverse=True):
log.writelines(elines)
log.write("\n")
return log
def export_set(dataset): """HTML representation of a Dataset.""" stream = StringIO() page = markup.page() page.table.open() if dataset.headers is not None: new_header = [item if item is not None else '' for item in dataset.headers] page.thead.open() headers = markup.oneliner.th(new_header) page.tr(headers) page.thead.close() for row in dataset: new_row = [item if item is not None else '' for item in row] html_row = markup.oneliner.td(new_row) page.tr(html_row) page.table.close() stream.writelines(str(page)) return stream.getvalue()
def save_db(data_dump):
temp_file = StringIO()
temp_file.writelines(data_dump)
temp_file.seek(0)
files = {'scans': temp_file}
values = {'DB': 'scans', 'OUT': 'db', 'SHORT': 'short'}
requests.post('http://127.0.0.1:5002/save_db', files=files, data=values)
def write_migration(file_obj, migration_class):
header = StringIO()
body = StringIO()
imports = set()
body.write("class Migration(migrations.Migration):\n\n")
for prop_name in MIGRATION_PROPS:
prop_imports, prop_serialized_value = serialize_assignment(
prop_name,
getattr(migration_class, prop_name),
level=1,
)
imports |= prop_imports
body.write(prop_serialized_value)
header.write("# -*- coding: utf-8 -*-\n")
header.write("from __future__ import unicode_literals\n")
header.write("\n")
header.write("from populus import migrations\n")
header.write("\n")
header.writelines([
"import {import_path}\n".format(import_path=import_path)
for import_path in sorted(imports)
])
file_obj.write(header.getvalue())
file_obj.write("\n")
file_obj.write(body.getvalue())
def ret_a_stream():
"""
Return a stream of text
"""
ret = StringIO()
ret.writelines(['1. hello\n', '2. world\n'])
return ret
class HugoDoc(object):
"""
A utility for creating hugo/markdown docs.
"""
def __init__(self, title='', tags=[]):
self.title = title
self.meta = {}
self.buff = StringIO()
self.meta['tags'] = tags
def header(self):
lines = ['title = "%s"' % self.title]
for key, val in self.meta.items():
if isinstance(val, list):
lines.append('%s = %s' % (key, val))
else:
lines.append('%s = "%s"' % (key, val))
# auto date
self.date = str(datetime.now().isoformat())
lines.append('date = "%s"' % self.date)
lines.insert(0, '+++')
lines.append('+++')
return '\n'.join(lines)
def writeline(self, s=u''):
self.buff.writelines(s)
self.buff.writelines(u'\n')
def getcontents(self):
return self.header() + '\n\n' + self.buff.getvalue()
def quicklook(item_id, regions=[]):
req = '/api/quicklook?typeid={0}'.format(item_id)
for reg in regions:
req += '®ionlimit={0}'.format(reg)
con = client.HTTPConnection('api.eve-central.com')
con.request('GET', req)
buf = StringIO()
buf.writelines([bs.decode() for bs in con.getresponse().readlines()])
buf.seek(0)
et = ElementTree.parse(buf)
rep = {'buy':[], 'sell':[]}
for order in et.getroot().find('quicklook').find('buy_orders'):
rep['buy'].append({
'region': int(order.find('region').text),
'station': int(order.find('station').text),
'security': float(order.find('security').text),
'range': int(order.find('range').text),
'price': float(order.find('price').text),
'volume': int(order.find('vol_remain').text),
'min': int(order.find('min_volume').text)})
for order in et.getroot().find('quicklook').find('sell_orders'):
rep['sell'].append({
'region': int(order.find('region').text),
'station': int(order.find('station').text),
'security': float(order.find('security').text),
'range': int(order.find('range').text),
'price': float(order.find('price').text),
'volume': int(order.find('vol_remain').text),
'min': int(order.find('min_volume').text)})
return rep
def _send_email(self, msg, tag):
if self.mailto is None:
return -1
header = msg.splitlines()
app = header.append
app("Submitted on: %s" % time.ctime(self.start_time))
app("Completed on: %s" % time.asctime())
app("Elapsed time: %s" % str(self.get_delta_etime()))
app("Number of errored tasks: %d" % self.flow.num_errored_tasks)
app("Number of unconverged tasks: %d" %
self.flow.num_unconverged_tasks)
strio = StringIO()
strio.writelines("\n".join(header) + 4 * "\n")
# Add the status of the flow.
self.flow.show_status(stream=strio)
if self.exceptions:
# Report the list of exceptions.
strio.writelines(self.exceptions)
if tag is None:
tag = " [ALL OK]" if self.flow.all_ok else " [WARNING]"
return sendmail(subject=self.flow.name + tag,
text=strio.getvalue(),
mailto=self.mailto)
def correct_INCA_format(fp):
fp_list = list()
fp.seek(0)
if '(' in fp.readline():
for line in fp:
line = line.replace(
"(MLX::",
"").replace(
" : ",
"\t").replace(
" :",
"\t").replace(
" ",
"\t").lower().strip().replace(
")",
"\n")
if "record-by" in line:
if "image" in line:
line = "record-by\timage"
if "vector" in line:
line = "record-by\tvector"
if "dont-care" in line:
line = "record-by\tdont-care"
fp_list.append(line)
fp = StringIO()
fp.writelines(fp_list)
fp.seek(0)
return fp
def create_standardised_interested_exported_country_table(
countries, output_schema, output_table):
stmt = f"""
SELECT distinct name FROM {DITCountryTerritoryRegister.__tablename__}
"""
result = flask_app.dbi.execute_query(stmt, df=False)
choices = [r[0] for r in result] + list(split_mappings.keys())
lower_choices = [choice.lower() for choice in choices]
columns = ['id', 'country', 'standardised_country', 'similarity']
tsv_lines = []
output_row_count = 1
for country in countries:
mappings = _standardise_country(country, choices, lower_choices)
for standardised_country, similarity in mappings:
line = [
str(output_row_count),
country,
standardised_country or '',
str(similarity),
]
tsv_lines.append('\t'.join(line) + '\n')
output_row_count += 1
tsv_data = StringIO()
tsv_data.writelines(tsv_lines)
tsv_data.seek(0)
_create_output_table(output_schema, output_table, drop=True)
flask_app.dbi.dsv_buffer_to_table(tsv_data,
f'{output_schema}.{output_table}',
columns)
def handle(self, headers, body):
"""
@type headers : dict[str, str]
@type body : str
"""
if GithubHook.payloadHeader in headers:
repository_name = None
try:
json_body = json.loads(body)
if 'repository' in json_body and 'html_url' in json_body['repository']:
repository_name = json_body['repository']['html_url']
log.info('Got a payload %s - %s', repository_name, headers[GithubHook.payloadHeader])
except ValueError, e:
log.warn(e.message)
handlers = [handler for handler in self.handlers if handler.can_handle(headers, body)]
if handlers:
response = StringIO()
status = 200
for handler in handlers:
s, r = handler.handle(body)
response.writelines(unicode(r))
status = max(status, s)
return response.getvalue(), status
else:
raise UnhandledGithubEvent(headers, body)
def _datafile_to_l0_temp(self, file_info):
csv_data = StringIO()
xbrl_parser = XBRLParser()
def flush():
nonlocal csv_data
csv_data.seek(0)
try:
self.dbi.dsv_buffer_to_table(
csv_data,
self._l0_temp_table,
null='None',
columns=[c for c, _ in self._l0_data_column_types],
)
except DataError as e:
logging.error(f'failed to upload accounts data: {e}')
csv_data = StringIO()
count = 0
for fi in DatafileProvider.read_files_from_zip(file_info.data):
if count % 1000 == 999:
flush()
try:
tsv_lines = xbrl_parser.xbrl_to_tsv(fi.data)
csv_data.writelines(tsv_lines)
count += 1
except XMLSyntaxError as e:
logging.error(f'failed to parse document: {e}')
flush()
def get_changelog(pkgdirurl,
another=None,
svn=True,
rev=None,
size=None,
submit=False,
sort=False,
template=None,
macros=[],
exported=None,
oldlog=False,
create=False,
fullnames=False):
"""Generates the changelog for a given package URL
@another: a stream with the contents of a changelog to be merged with
the one generated
@svn: enable changelog from svn
@rev: generate the changelog with the changes up to the given
revision
@size: the number of revisions to be used (as in svn log --limit)
@submit: defines whether the latest unreleased log entries should have
the version parsed from the spec file
@sort: should changelog entries be reparsed and sorted after appending
the oldlog?
@template: the path to the cheetah template used to generate the
changelog from svn
@macros: a list of tuples containing macros to be defined when
parsing the version in the changelog
@exported: the path of a directory containing an already existing
checkout of the package, so that the spec file can be
parsed from there
@oldlog: if set it will try to append the old changelog file defined
in oldurl in repsys.conf
@create: if set, will use rpm -qp rpm instead of --specfile to get release number
"""
newlog = StringIO()
if svn:
if fullnames:
if not usermap:
_map_user_names()
rawsvnlog = svn2rpm(pkgdirurl,
rev=rev,
size=size,
submit=submit,
template=template,
macros=macros,
exported=exported,
create=create)
newlog.write(rawsvnlog)
if another:
newlog.writelines(another)
if oldlog:
newlog.writelines(get_old_log(pkgdirurl))
if sort:
newlog.seek(0)
newlog = sort_changelog(newlog)
newlog.seek(0)
return newlog
def __init__(self, iqsharp_errors : List[str]):
self.iqsharp_errors = iqsharp_errors
error_msg = StringIO()
error_msg.write("The Q# kernel raised the following errors:\n")
error_msg.writelines([
" " + msg for msg in iqsharp_errors
])
super().__init__(error_msg.getvalue())
class FakeStream(object):
def __init__(self):
self.stream = StringIO()
def writeln(self, data):
self.stream.writelines(data)
def read(self):
return self.stream.getvalue()
def __getstate__(self):
"""Dump the database as SQL commands so that it can be pickled."""
db_dump = StringIO()
# It should be safe to assume that the DB is connected (because it is
# done in __init__
db_dump.writelines(self._db_conn.iterdump())
return {'_xml_dir': self._xml_dir, '_db_dump': db_dump.getvalue()}
def __str__(self) -> str:
string = StringIO()
string.writelines("============={classname}=============".format(classname=self.__class__.__name__) + linesep )
for x in self.__dict__.keys():
if x[0:1] == '_': continue
string.writelines("{key} 的值为 {value}".format(key=x, value=self.__dict__[x]) + linesep )
string.seek(0)
desc = string.getvalue()
string.close()
return desc
def copyToDB(cur, dbName, items, colNames):
try:
f = StringIO()
f.writelines(items)
f.seek(0)
cur.copy_from(f,dbName,columns=colNames)
return True
except:
print('could not write to: {}'.format(dbName))
return False
def openfile(self, filename):
output = StringIO()
wb = load_workbook(filename=filename, read_only=True)
ws = wb.worksheets[0]
for row in ws.rows:
if row[0].value:
output.writelines(str(row[0].value) + '\n')
self.output = output
def blocking_generate_html_archive(channel: 'discord.TextChannel',
messages: 'MessageGroups',
msg_count: int) -> StringIO:
archive = StringIO()
ctx = {'guild': channel.guild, 'channel': channel}
output = template.render(ctx=ctx, msg_groups=messages, msg_count=msg_count)
archive.writelines(output)
archive.seek(0)
return archive
def test_write_read_last_message_incomplete(pipe_handler: PipeHandler,
pipe_handler_stream: StringIO):
json_data = json.dumps({
"foo": "bar",
"baz": 2
}, indent=2).splitlines(keepends=True)
pipe_handler_stream.writelines(json_data[:-2])
pipe_handler_stream.seek(0)
with pytest.raises(EsqueIOHandlerReadException):
_ = pipe_handler.read_message()
def reverse_words(words: str):
cs = []
ss = StringIO()
for c in words:
if c.isspace():
ss.writelines(cs[::-1])
ss.write(c)
cs.clear()
else:
cs.append(c)
ss.writelines(cs[::-1])
return ss.getvalue()
def close(self):
temp_csv = StringIO()
temp_csv.writelines(self.index_list)
self.zip_obj.writestr('index.csv', temp_csv.getvalue())
self.index_list = []
self.zip_obj.close()
os.rename('%s.tmp' % self.current_filename, self.current_filename)
def instance_ssh_keys(request, application_id, cookie):
# serves the sshkey of an applicant
# in order to pass it to ganeti while creating the instance
app = get_object_or_404(InstanceApplication, pk=application_id)
if cookie != app.cookie:
t = get_template("403.html")
return HttpResponseForbidden(content=t.render(request=request))
output = StringIO()
output.writelines(
[k.key_line() for k in app.applicant.sshpublickey_set.all()])
return HttpResponse(output.getvalue(), content_type="text/plain")
def pin(length):
"""
Makes a secure PIN.
Usage:
passwordtools.pin(length)
"""
out = StringIO()
out.writelines(sample(DIGITS, length))
password = out.getvalue()
if len(password) < length:
password += sample(DIGITS, 1)[0]
return password
def assemble(cls, file: str, parser_: Parser,
is_pipelined: bool) -> StringIO:
with open(file, 'r') as f:
parser_.clear()
text = parser_.preprocess_assembly(f.readlines())
ret = StringIO()
ret.write(Assembler._HEADER)
ret.writelines([
Assembler._parse_and_format_line(parser_, line, i)
for i, line in enumerate(text)
])
ret.write(Assembler._FOOTER.format(len(text), str(Assembler._NOP)))
return ret
def __init__(self):
sio = StringIO("==='StringIO' demo===\n")
# 不seek到末尾, 初始化内容将被覆盖
sio.seek(len(sio.getvalue()))
sio.write('line\n')
print("round 1:", sio.getvalue())
lines = [f"line{i}\n" for i in range(10)]
sio.writelines(lines)
print("round 2:", sio.getvalue())
# 跳转到开头
sio.seek(0)
print("round 3:", sio.readlines())
sio.close()
def transfer(filename):
skip_rows = 5
head = [
"PRES", "HGHT", "TEMP", "DWPT", "FRPT", "RELH", "RELI", "MIXR", "DRCT",
"SKNT", "THTA", "THTE", "THTV"
]
f = open(filename, "r")
all_text = f.readlines()
s = StringIO()
for index, text in enumerate(all_text):
if text.strip() == "Station information and sounding indices":
end_index = index
break
print(end_index)
for irows in range(skip_rows, end_index):
for icols in range(0, 91, 7):
if (all_text[irows][icols:icols + 7]).isspace() == True:
all_text[irows] = all_text[
irows][:icols] + " nan" + all_text[irows][icols + 7:]
s.writelines(all_text)
s.seek(0, 0)
dat = pd.read_table(s,
delim_whitespace=True,
skiprows=skip_rows,
nrows=end_index - skip_rows,
names=head)
new_dat = pd.DataFrame(columns=[
"StationId", "Lat", "Lon", "SoundingDate", "PresHPa", "HeightM",
"TempC", "DwptC", "RH", "Wind_D", "Wind_S"
])
new_dat["PresHPa"] = dat["PRES"]
new_dat["StationId"] = 72317
new_dat["Lat"] = 32
new_dat["Lon"] = 20
new_dat["SoundingDate"] = 2018060600
new_dat["HeightM"] = dat["HGHT"]
new_dat["TempC"] = dat["TEMP"]
new_dat["DwptC"] = dat["DWPT"]
new_dat["RH"] = dat["RELH"]
new_dat["Wind_S"] = dat["SKNT"] * 0.5144
new_dat["Wind_D"] = dat["DRCT"]
return new_dat
def _gen_header(self, makefile: StringIO) -> None:
port_makefile = self.portdir / "Makefile"
metadata: List[str] = []
if port_makefile.exists():
with port_makefile.open("rU") as makefile_file:
for line in iter(makefile_file.readline, ""):
if line.startswith("# Created by") or line.startswith(
"# $FreeBSD"):
metadata.append(line)
if peek(makefile_file, 1) != "#":
break
else:
metadata.append("# $FreeBSD$\n")
makefile.writelines(metadata)
def generate_status_calendar(status_log: list[LogEntry]) -> StringIO:
calendar = Calendar()
for status, start, duration in status_log:
event = Event()
event.name = f"User was {status}"
event.begin = start.strftime("%Y-%m-%d %H:%M:%S")
event.end = (start + duration).strftime("%Y-%m-%d %H:%M:%S")
calendar.events.add(event)
out = StringIO()
out.writelines(calendar)
out.seek(0)
return out
def stringio(str):
f = StringIO()
f.writelines([
str,
])
f.write(str)
print(f.getvalue())
f.seek(0)
while True:
s = f.read()
if s == '':
break
print(s.strip())
def extract_from_pdf(file, password):
#decrypting the encrypted pdf file
content = pikepdf.open(file, password=password)
inmemory_file = BytesIO()
content.save(inmemory_file)
#reading and extracting data from the decrypted pdf file
pdf_reader = PyPDF2.PdfFileReader(inmemory_file)
num_pages = pdf_reader.getNumPages()
extracted_data = StringIO()
for page in range(num_pages):
extracted_data.writelines(pdf_reader.getPage(page).extractText())
return num_pages, extracted_data
def patch_dockerfile(cls, copr_names: Iterable[str], dockerfile: str) -> str:
out = StringIO()
before_from = True
def write_copr_enabler():
out.write(BEGIN_CDIC_SECTION)
out.write("RUN yum install -y dnf dnf-plugins-core \\\n"
" && mkdir -p /etc/yum.repos.d/\n")
if copr_names:
out.write("RUN ")
out.write(" && \\\n ".join([
"dnf copr enable -y {}".format(copr)
for copr in copr_names
]))
# out.write("\n")
# out.write("RUN dnf clean all\n")
out.write("\n")
out.write(END_CDIC_SECTION)
for raw_line in dockerfile.split(os.linesep):
line = raw_line.strip()
if not line:
continue
if line.startswith("FROM"):
if before_from:
before_from = False
out.write("{}\n".format(line))
write_copr_enabler()
else:
raise PatchDockerfileException("Unexpected command, "
"encountered FROM command twice")
else:
if before_from:
raise PatchDockerfileException("Unexpected command, Dockerfile "
"should start with FROM command")
else:
out.writelines("{}\n".format(line))
return out.getvalue()
def save_by_copy(clusters, quotes):
"""Import a list of clusters and a list of quotes into the database.
This function uses PostgreSQL's COPY command to bulk import clusters and
quotes, and prints its progress to stdout.
Parameters
----------
clusters : list of :class:`Cluster`\ s
List of clusters to import in the database.
quotes : list of :class:`Quote`\ s
List of quotes to import in the database. Any clusters they reference
should be in the `clusters` parameter.
See Also
--------
.load.MemeTrackerParser.parse
"""
# Order the objects inserted so the engine bulks them together.
logger.debug("Saving %s clusters with 'copy_from'", len(clusters))
click.echo('Saving clusters... ', nl=False)
objects = StringIO()
objects.writelines([cluster.format_copy() + '\n' for cluster in clusters])
_copy(objects, Cluster.__tablename__, Cluster.format_copy_columns)
objects.close()
click.secho('OK', fg='green', bold=True)
logger.debug("Saving %s quotes with 'copy_from'", len(quotes))
click.echo('Saving quotes... ', nl=False)
objects = StringIO()
objects.writelines([quote.format_copy() + '\n' for quote in quotes])
_copy(objects, Quote.__tablename__, Quote.format_copy_columns)
objects.close()
click.secho('OK', fg='green', bold=True)
class _FlakyPlugin(object):
_retry_failure_message = ' failed ({0} runs remaining out of {1}).'
_failure_message = ' failed; it passed {0} out of the required {1} times.'
_not_rerun_message = ' failed and was not selected for rerun.'
def __init__(self):
super(_FlakyPlugin, self).__init__()
self._stream = StringIO()
self._flaky_success_report = True
@property
def stream(self):
"""
Returns the stream used for building the flaky report.
Anything written to this stream before the end of the test run
will be written to the flaky report.
:return:
The stream used for building the flaky report.
:rtype:
:class:`StringIO`
"""
return self._stream
def _log_test_failure(self, test_callable_name, err, message):
"""
Add messaging about a test failure to the stream, which will be
printed by the plugin's report method.
"""
self._stream.writelines([
ensure_unicode_string(test_callable_name),
message,
'\n\t',
ensure_unicode_string(err[0]),
'\n\t',
ensure_unicode_string(err[1]),
'\n\t',
ensure_unicode_string(err[2]),
'\n',
])
def _report_final_failure(self, err, flaky, name):
"""
Report that the test has failed too many times to pass at
least min_passes times.
By default, this means that the test has failed twice.
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `class`, :class:`Exception`, `traceback`
:param flaky:
Dictionary of flaky attributes
:type flaky:
`dict` of `unicode` to varies
:param name:
The test name
:type name:
`unicode`
"""
min_passes = flaky[FlakyNames.MIN_PASSES]
current_passes = flaky[FlakyNames.CURRENT_PASSES]
message = self._failure_message.format(
current_passes,
min_passes,
)
self._log_test_failure(name, err, message)
def _log_intermediate_failure(self, err, flaky, name):
"""
Report that the test has failed, but still has reruns left.
Then rerun the test.
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `class`, :class:`Exception`, `traceback`
:param flaky:
Dictionary of flaky attributes
:type flaky:
`dict` of `unicode` to varies
:param name:
The test name
:type name:
`unicode`
"""
max_runs = flaky[FlakyNames.MAX_RUNS]
runs_left = max_runs - flaky[FlakyNames.CURRENT_RUNS]
message = self._retry_failure_message.format(
runs_left,
max_runs,
)
self._log_test_failure(name, err, message)
def _should_handle_test_error_or_failure(self, test):
"""
Whether or not flaky should handle a test error or failure.
Only handle tests marked @flaky.
Count remaining retries and compare with number of required successes that have not yet been achieved.
This method may be called multiple times for the same test run, so it has no side effects.
:param test:
The test that has raised an error
:type test:
:class:`nose.case.Test` or :class:`Function`
:return:
True, if the test needs to be rerun; False, otherwise.
:rtype:
`bool`
"""
if not self._has_flaky_attributes(test):
return False
flaky_attributes = self._get_flaky_attributes(test)
flaky_attributes[FlakyNames.CURRENT_RUNS] += 1
has_failed = self._has_flaky_test_failed(flaky_attributes)
return not has_failed
def _will_handle_test_error_or_failure(self, test, name, err):
"""
Whether or not flaky will handle a test error or failure.
Returns True if the plugin should handle the test result, and
the `rerun_filter` returns True.
:param test:
The test that has raised an error
:type test:
:class:`nose.case.Test` or :class:`Function`
:param name:
The name of the test that has raised an error
:type name:
`unicode`
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `type`, :class:`Exception`, `traceback`
:return:
True, if the test will be rerun by flaky; False, otherwise.
:rtype:
`bool`
"""
return self._should_handle_test_error_or_failure(test) and self._should_rerun_test(test, name, err)
def _handle_test_error_or_failure(self, test, err):
"""
Handle a flaky test error or failure.
Returning True from this method keeps the test runner from reporting
the test as a failure; this way we can retry and only report as a
failure if we are out of retries.
This method may only be called once per test run; it changes persisted flaky attributes.
:param test:
The test that has raised an error
:type test:
:class:`nose.case.Test` or :class:`Function`
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `type`, :class:`Exception`, `traceback`
:return:
True, if the test will be rerun;
False, if the test runner should handle it.
:rtype:
`bool`
"""
try:
name = self._get_test_callable_name(test)
except AttributeError:
return False
if self._has_flaky_attributes(test):
self._add_flaky_test_failure(test, err)
should_handle = self._should_handle_test_error_or_failure(test)
self._increment_flaky_attribute(test, FlakyNames.CURRENT_RUNS)
if should_handle:
flaky_attributes = self._get_flaky_attributes(test)
if self._should_rerun_test(test, name, err):
self._log_intermediate_failure(err, flaky_attributes, name)
self._mark_test_for_rerun(test)
return True
self._log_test_failure(name, err, self._not_rerun_message)
return False
else:
flaky_attributes = self._get_flaky_attributes(test)
self._report_final_failure(err, flaky_attributes, name)
return False
def _should_rerun_test(self, test, name, err):
"""
Whether or not a test should be rerun.
This is a pass-through to the test's rerun filter.
A flaky test will only be rerun if it hasn't failed too many
times to succeed at least min_passes times, and if
this method returns True.
:param test:
The test that has raised an error
:type test:
:class:`nose.case.Test` or :class:`Function`
:param name:
The test name
:type name:
`unicode`
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `class`, :class:`Exception`, `traceback`
:return:
Whether flaky should rerun this test.
:rtype:
`bool`
"""
rerun_filter = self._get_flaky_attribute(test, FlakyNames.RERUN_FILTER)
return rerun_filter(err, name, test, self)
def _mark_test_for_rerun(self, test):
"""
Mark a flaky test for rerun.
:param test:
The test that has raised an error or succeeded
:type test:
:class:`nose.case.Test` or :class:`Function`
"""
raise NotImplementedError # pragma: no cover
def _should_handle_test_success(self, test):
if not self._has_flaky_attributes(test):
return False
flaky = self._get_flaky_attributes(test)
flaky[FlakyNames.CURRENT_PASSES] += 1
flaky[FlakyNames.CURRENT_RUNS] += 1
return not self._has_flaky_test_succeeded(flaky)
def _handle_test_success(self, test):
"""
Handle a flaky test success.
Count remaining retries and compare with number of required successes
that have not yet been achieved; retry if necessary.
Returning True from this method keeps the test runner from reporting
the test as a success; this way we can retry and only report as a
success if the test has passed the required number of times.
:param test:
The test that has raised an error
:type test:
:class:`nose.case.Test` or :class:`Function`
:return:
True, if the test will be rerun; False, if the test runner should handle it.
:rtype:
`bool`
"""
try:
name = self._get_test_callable_name(test)
except AttributeError:
return False
need_reruns = self._should_handle_test_success(test)
if self._has_flaky_attributes(test):
flaky = self._get_flaky_attributes(test)
min_passes = flaky[FlakyNames.MIN_PASSES]
passes = flaky[FlakyNames.CURRENT_PASSES] + 1
self._set_flaky_attribute(test, FlakyNames.CURRENT_PASSES, passes)
self._increment_flaky_attribute(test, FlakyNames.CURRENT_RUNS)
if self._flaky_success_report:
self._stream.writelines([
ensure_unicode_string(name),
' passed {0} out of the required {1} times. '.format(
passes,
min_passes,
),
])
if need_reruns:
self._stream.write(
'Running test again until it passes {0} times.\n'.format(
min_passes,
)
)
else:
self._stream.write('Success!\n')
if need_reruns:
self._mark_test_for_rerun(test)
return need_reruns
@staticmethod
def add_report_option(add_option):
"""
Add an option to the test runner to suppress the flaky report.
:param add_option:
A function that can add an option to the test runner.
Its argspec should equal that of argparse.add_option.
:type add_option:
`callable`
"""
add_option(
'--no-flaky-report',
action='store_false',
dest='flaky_report',
default=True,
help="Suppress the report at the end of the "
"run detailing flaky test results.",
)
add_option(
'--no-success-flaky-report',
action='store_false',
dest='flaky_success_report',
default=True,
help="Suppress reporting flaky test successes"
"in the report at the end of the "
"run detailing flaky test results.",
)
@staticmethod
def add_force_flaky_options(add_option):
"""
Add options to the test runner that force all tests to be flaky.
:param add_option:
A function that can add an option to the test runner.
Its argspec should equal that of argparse.add_option.
:type add_option:
`callable`
"""
add_option(
'--force-flaky',
action="store_true",
dest="force_flaky",
default=False,
help="If this option is specified, we will treat all tests as "
"flaky."
)
add_option(
'--max-runs',
action="store",
dest="max_runs",
type=int,
default=2,
help="If --force-flaky is specified, we will run each test at "
"most this many times (unless the test has its own flaky "
"decorator)."
)
add_option(
'--min-passes',
action="store",
dest="min_passes",
type=int,
default=1,
help="If --force-flaky is specified, we will run each test at "
"least this many times (unless the test has its own flaky "
"decorator)."
)
def _add_flaky_report(self, stream):
"""
Baseclass override. Write details about flaky tests to the test report.
:param stream:
The test stream to which the report can be written.
:type stream:
`file`
"""
value = self._stream.getvalue()
# If everything succeeded and --no-success-flaky-report is specified
# don't print anything.
if not self._flaky_success_report and not value:
return
stream.write('===Flaky Test Report===\n\n')
# Python 2 will write to the stderr stream as a byte string, whereas
# Python 3 will write to the stream as text. Only encode into a byte
# string if the write tries to encode it first and raises a
# UnicodeEncodeError.
try:
stream.write(value)
except UnicodeEncodeError:
stream.write(value.encode('utf-8', 'replace'))
stream.write('\n===End Flaky Test Report===\n')
@classmethod
def _copy_flaky_attributes(cls, test, test_class):
"""
Copy flaky attributes from the test callable or class to the test.
:param test:
The test that is being prepared to run
:type test:
:class:`nose.case.Test`
"""
test_callable = cls._get_test_callable(test)
if test_callable is None:
return
for attr, value in cls._get_flaky_attributes(test_class).items():
already_set = hasattr(test, attr)
if already_set:
continue
attr_on_callable = getattr(test_callable, attr, None)
if attr_on_callable is not None:
cls._set_flaky_attribute(test, attr, attr_on_callable)
elif value is not None:
cls._set_flaky_attribute(test, attr, value)
@staticmethod
def _get_flaky_attribute(test_item, flaky_attribute):
"""
Gets an attribute describing the flaky test.
:param test_item:
The test method from which to get the attribute
:type test_item:
`callable` or :class:`nose.case.Test` or :class:`Function`
:param flaky_attribute:
The name of the attribute to get
:type flaky_attribute:
`unicode`
:return:
The test callable's attribute, or None if the test
callable doesn't have that attribute.
:rtype:
varies
"""
return getattr(test_item, flaky_attribute, None)
@staticmethod
def _set_flaky_attribute(test_item, flaky_attribute, value):
"""
Sets an attribute on a flaky test. Uses magic __dict__ since setattr
doesn't work for bound methods.
:param test_item:
The test callable on which to set the attribute
:type test_item:
`callable` or :class:`nose.case.Test` or :class:`Function`
:param flaky_attribute:
The name of the attribute to set
:type flaky_attribute:
`unicode`
:param value:
The value to set the test callable's attribute to.
:type value:
varies
"""
test_item.__dict__[flaky_attribute] = value
@classmethod
def _increment_flaky_attribute(cls, test_item, flaky_attribute):
"""
Increments the value of an attribute on a flaky test.
:param test_item:
The test callable on which to set the attribute
:type test_item:
`callable` or :class:`nose.case.Test` or :class:`Function`
:param flaky_attribute:
The name of the attribute to set
:type flaky_attribute:
`unicode`
"""
cls._set_flaky_attribute(test_item, flaky_attribute, cls._get_flaky_attribute(test_item, flaky_attribute) + 1)
@classmethod
def _has_flaky_attributes(cls, test):
"""
Returns True if the test callable in question is marked as flaky.
:param test:
The test that is being prepared to run
:type test:
:class:`nose.case.Test` or :class:`Function`
:return:
:rtype:
`bool`
"""
current_runs = cls._get_flaky_attribute(test, FlakyNames.CURRENT_RUNS)
return current_runs is not None
@classmethod
def _get_flaky_attributes(cls, test_item):
"""
Get all the flaky related attributes from the test.
:param test_item:
The test callable from which to get the flaky related attributes.
:type test_item:
`callable` or :class:`nose.case.Test` or :class:`Function`
:return:
:rtype:
`dict` of `unicode` to varies
"""
return dict((
(attr, cls._get_flaky_attribute(
test_item,
attr,
)) for attr in FlakyNames()
))
@classmethod
def _add_flaky_test_failure(cls, test, err):
"""
Store test error information on the test callable.
:param test:
The flaky test on which to update the flaky attributes.
:type test:
:class:`nose.case.Test` or :class:`Function`
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `class`, :class:`Exception`, `traceback`
"""
errs = getattr(test, FlakyNames.CURRENT_ERRORS, None) or []
cls._set_flaky_attribute(test, FlakyNames.CURRENT_ERRORS, errs)
errs.append(err)
@classmethod
def _has_flaky_test_failed(cls, flaky):
"""
Whether or not the flaky test has failed
:param flaky:
Dictionary of flaky attributes
:type flaky:
`dict` of `unicode` to varies
:return:
True if the flaky test should be marked as failure; False if
it should be rerun.
:rtype:
`bool`
"""
max_runs, current_runs, min_passes, current_passes = (
flaky[FlakyNames.MAX_RUNS],
flaky[FlakyNames.CURRENT_RUNS],
flaky[FlakyNames.MIN_PASSES],
flaky[FlakyNames.CURRENT_PASSES],
)
runs_left = max_runs - current_runs
passes_needed = min_passes - current_passes
no_retry = passes_needed > runs_left
return no_retry and not cls._has_flaky_test_succeeded(flaky)
@staticmethod
def _has_flaky_test_succeeded(flaky):
"""
Whether or not the flaky test has succeeded
:param flaky:
Dictionary of flaky attributes
:type flaky:
`dict` of `unicode` to varies
:return:
True if the flaky test should be marked as success; False if
it should be rerun.
:rtype:
`bool`
"""
return flaky[FlakyNames.CURRENT_PASSES] >= flaky[FlakyNames.MIN_PASSES]
@classmethod
def _get_test_callable(cls, test):
"""
Get the test callable, from the test.
:param test:
The test that has raised an error or succeeded
:type test:
:class:`nose.case.Test` or :class:`pytest.Item`
:return:
The test declaration, callable and name that is being run
:rtype:
`callable`
"""
raise NotImplementedError # pragma: no cover
@staticmethod
def _get_test_callable_name(test):
"""
Get the name of the test callable from the test.
:param test:
The test that has raised an error or succeeded
:type test:
:class:`nose.case.Test` or :class:`pytest.Item`
:return:
The name of the test callable that is being run by the test
:rtype:
`unicode`
"""
raise NotImplementedError # pragma: no cover
@classmethod
def _make_test_flaky(cls, test, max_runs, min_passes):
"""
Make a given test flaky.
:param test:
The test in question.
:type test:
:class:`nose.case.Test` or :class:`Function`
:param max_runs:
The value of the FlakyNames.MAX_RUNS attribute to use.
:type max_runs:
`int`
:param min_passes:
The value of the FlakyNames.MIN_PASSES attribute to use.
:type min_passes:
`int`
"""
attrib_dict = defaults.default_flaky_attributes(max_runs, min_passes)
for attr, value in attrib_dict.items():
cls._set_flaky_attribute(test, attr, value)
def dict_to_file(data):
"""
creates file format from transaction records
:param data: list of dicts
:return: file stream
"""
def get_trailer(trailer_totals):
trailer_format = (
u'7' +
u'999-999' +
u' ' * 12 +
u'{net_total:010}' +
u'{credit_total:010}' +
u'{debit_total:010}' +
u' ' * 24 +
u'{count_trans:06}' +
u' ' * 40
)
return trailer_format.format(
net_total=abs(trailer_totals[TOTAL_CREDITS]-trailer_totals[TOTAL_DEBITS]),
credit_total=trailer_totals[TOTAL_CREDITS],
debit_total=trailer_totals[TOTAL_DEBITS],
count_trans=trailer_totals[TOTAL_ITEMS]
)
record_format = (
u'0' +
u' ' * 17 +
u'{reel_seq_num:2.2}' +
u'{name_fin_inst:3}' +
u' ' * 7 +
u'{user_name:26.26}' +
u'{user_num:6.6}' +
u'{file_desc:12.12}' +
u'{date_for_process:6.6}' +
u' ' * 40 +
u'{record_type:1.1}' +
u'{bsb_number:7.7}' +
u'{account_number:9.9}' +
u'{indicator:1.1}' +
u'{tran_code:2.2}' +
u'{amount:10.10}' +
u'{account_title:32.32}' +
u'{lodgement_ref:18.18}' +
u'{trace_bsb_number:7.7}' +
u'{trace_account_number:9.9}' +
u'{name_of_remitter:16.16}' +
u'{withholding_tax_amount:8.8}'
)
LOGGER.debug('record_format={}'.format(record_format))
flat_trans = sorted([(record_format.format(**tran), tran) for tran in data])
# remove duplicate headers and accumulate for trailer
last_header = ''
output_list = []
totals = [0, 0, 0]
for tran, data in flat_trans:
if last_header != tran[:120]:
if len(output_list) != 0:
output_list.append(get_trailer(totals))
totals = [0, 0, 0]
output_list.append(tran[:120])
last_header = tran[:120]
if data['tran_code'] == u'13':
totals[TOTAL_CREDITS] += int(data['amount'])
else:
totals[TOTAL_DEBITS] += int(data['amount'])
totals[TOTAL_ITEMS] += 1
output_list.append(tran[120:])
output_list.append(get_trailer(totals))
# add to stream
output_stream = StringIO()
# add line endings
output_stream.writelines('\n'.join(output_list))
output_stream.seek(0)
return output_stream
def to_full_sbtab(self):
"""Returns a full SBtab description of the model.
Description includes reaction fluxes and per-compound bounds.
"""
generic_header_fmt = "!!SBtab TableName='%s' TableType='%s' Document='%s' SBtabVersion='1.0'"
reaction_header = generic_header_fmt % ('Reaction', 'Reaction', 'Pathway Model')
reaction_cols = ['!ID', '!ReactionFormula', '!Identifiers:kegg.reaction']
sio = StringIO()
sio.writelines([reaction_header + '\n'])
writer = csv.DictWriter(sio, reaction_cols, dialect='excel-tab')
writer.writeheader()
rxn_ids = []
for i, rxn in enumerate(self.reactions):
kegg_id = rxn.stored_reaction_id
rxn_id = kegg_id
if rxn.catalyzing_enzymes:
enz = str(rxn.catalyzing_enzymes[0].FirstName().name)
enz_slug = slugify(enz)[:10]
enz_slug = enz_slug.replace('-', '_')
rxn_id = '%s_%s' % (enz_slug, kegg_id)
elif not kegg_id:
rxn_id = 'RXN%03d' % i
rxn_ids.append(rxn_id)
d = {'!ID': rxn_id,
'!ReactionFormula': rxn.GetSlugQueryString(),
'!Identifiers:kegg.reaction': kegg_id}
writer.writerow(d)
# Relative fluxes
flux_header = generic_header_fmt % ('RelativeFlux', 'Quantity', 'Pathway Model')
flux_cols = ['!QuantityType', '!Reaction', '!Reaction:Identifiers:kegg.reaction', '!Value']
sio.writelines(['%\n', flux_header + '\n'])
writer = csv.DictWriter(sio, flux_cols, dialect='excel-tab')
writer.writeheader()
for i, rxn_id in enumerate(rxn_ids):
d = {'!QuantityType': 'flux',
'!Reaction': rxn_id,
'!Reaction:Identifiers:kegg.reaction': self.reactions[i].stored_reaction_id,
'!Value': self.fluxes[i]}
writer.writerow(d)
# Write KEQs.
keq_header = generic_header_fmt % (
'ReactionConstant', 'Quantity', 'Pathway Model')
keq_cols = ['!QuantityType', '!Reaction', '!Value',
'!Unit', '!Reaction:Identifiers:kegg.reaction', '!ID']
if self.aq_params:
# Write pH and ionic strength in header
aq_params_header = (
"pH='%.2f' 'IonicStrength='%.2f' IonicStrengthUnit='M'")
aq_params_header = aq_params_header % (
self.aq_params.pH, self.aq_params.ionic_strength)
keq_header = '%s %s' % (keq_header, aq_params_header)
sio.writelines(['%\n', keq_header + '\n'])
writer = csv.DictWriter(sio, keq_cols, dialect='excel-tab')
writer.writeheader()
for i, (rxn_id, rxn, dg) in enumerate(zip(rxn_ids, self.reactions, self.dG0_r_prime)):
keq_id = 'kEQ_R%d' % i
keq = numpy.exp(-dg / constants.RT)
d = {'!QuantityType': 'equilibrium constant',
'!Reaction': rxn_id,
'!Value': keq,
'!Unit': 'dimensionless',
'!Reaction:Identifiers:kegg.reaction': rxn.stored_reaction_id,
'!ID': keq_id}
writer.writerow(d)
conc_header = generic_header_fmt % ('ConcentrationConstraint', 'Quantity', 'Pathway Model')
conc_header += " Unit='M'"
conc_cols = ['!QuantityType', '!Compound',
'!Compound:Identifiers:kegg.compound',
'!Concentration:Min', '!Concentration:Max']
sio.writelines(['%\n', conc_header + '\n'])
writer = csv.DictWriter(sio, conc_cols, dialect='excel-tab')
writer.writeheader()
for cid, compound in self.compounds_by_kegg_id.items():
d = {'!QuantityType': 'concentration',
'!Compound': str(compound.name_slug),
'!Compound:Identifiers:kegg.compound': cid,
'!Concentration:Min': self.bounds.GetLowerBound(cid),
'!Concentration:Max': self.bounds.GetUpperBound(cid)}
writer.writerow(d)
return sio.getvalue()
class _FlakyPlugin(object):
_retry_failure_message = ' failed ({0} runs remaining out of {1}).'
_failure_message = ' failed; it passed {0} out of the required {1} times.'
def __init__(self):
super(_FlakyPlugin, self).__init__()
self._stream = StringIO()
def _log_test_failure(self, test_callable_name, err, message):
"""
Add messaging about a test failure to the stream, which will be
printed by the plugin's report method.
"""
self._stream.writelines([
ensure_unicode_string(test_callable_name),
message,
'\n\t',
ensure_unicode_string(err[0]),
'\n\t',
ensure_unicode_string(err[1]),
'\n\t',
ensure_unicode_string(err[2]),
'\n',
])
def _handle_test_error_or_failure(self, test, err):
"""
Handle a flaky test error or failure.
Count remaining retries and compare with number of required successes
that have not yet been achieved; retry if necessary.
Returning True from this method keeps the test runner from reporting
the test as a failure; this way we can retry and only report as a
failure if we are out of retries.
:param test:
The test that has raised an error
:type test:
:class:`nose.case.Test`
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `class`, :class:`Exception`, `traceback`
:return:
True, if the test will be rerun;
False, if the test runner should handle it.
:rtype:
`bool`
"""
try:
_, _, name = self._get_test_declaration_callable_and_name(test)
except AttributeError:
return False
current_runs = self._get_flaky_attribute(
test,
FlakyNames.CURRENT_RUNS,
)
if current_runs is None:
return False
current_runs += 1
self._set_flaky_attribute(
test,
FlakyNames.CURRENT_RUNS,
current_runs,
)
self._add_flaky_test_failure(test, err)
flaky = self._get_flaky_attributes(test)
if not self._has_flaky_test_failed(flaky):
max_runs = flaky[FlakyNames.MAX_RUNS]
runs_left = max_runs - flaky[FlakyNames.CURRENT_RUNS]
message = self._retry_failure_message.format(
runs_left,
max_runs,
)
self._log_test_failure(name, err, message)
self._rerun_test(test)
return True
else:
min_passes = flaky[FlakyNames.MIN_PASSES]
current_passes = flaky[FlakyNames.CURRENT_PASSES]
message = self._failure_message.format(
current_passes,
min_passes,
)
self._log_test_failure(name, err, message)
return False
def _rerun_test(self, test):
"""
Rerun a flaky test.
:param test:
The test that has raised an error or succeeded
:type test:
:class:`Function`
"""
raise NotImplementedError
def _handle_test_success(self, test):
"""
Handle a flaky test success.
Count remaining retries and compare with number of required successes
that have not yet been achieved; retry if necessary.
Returning True from this method keeps the test runner from reporting
the test as a success; this way we can retry and only report as a
success if the test has passed the required number of times.
:param test:
The test that has raised an error
:type test:
:class:`nose.case.Test`
:return:
True, if the test will be rerun; False, if nose should handle it.
:rtype:
`bool`
"""
_, _, name = self._get_test_declaration_callable_and_name(test)
current_runs = self._get_flaky_attribute(
test,
FlakyNames.CURRENT_RUNS
)
if current_runs is None:
return False
current_runs += 1
current_passes = self._get_flaky_attribute(
test,
FlakyNames.CURRENT_PASSES
)
current_passes += 1
self._set_flaky_attribute(
test,
FlakyNames.CURRENT_RUNS,
current_runs
)
self._set_flaky_attribute(
test,
FlakyNames.CURRENT_PASSES,
current_passes
)
flaky = self._get_flaky_attributes(test)
min_passes = flaky[FlakyNames.MIN_PASSES]
self._stream.writelines([
ensure_unicode_string(name),
' passed {0} out of the required {1} times. '.format(
current_passes,
min_passes,
),
])
if not self._has_flaky_test_succeeded(flaky):
self._stream.write(
'Running test again until it passes {0} times.\n'.format(
min_passes,
)
)
self._rerun_test(test)
return True
else:
self._stream.write('Success!\n')
return False
@staticmethod
def add_report_option(add_option):
"""
Add an option to the test runner to suppress the flaky report.
:param add_option:
A function that can add an option to the test runner.
Its argspec should equal that of argparse.add_option.
:type add_option:
`callable`
"""
add_option(
'--no-flaky-report',
action='store_false',
dest='flaky_report',
default=True,
help="Suppress the report at the end of the "
"run detailing flaky test results.",
)
@staticmethod
def add_force_flaky_options(add_option):
"""
Add options to the test runner that force all tests to be flaky.
:param add_option:
A function that can add an option to the test runner.
Its argspec should equal that of argparse.add_option.
:type add_option:
`callable`
"""
add_option(
'--force-flaky',
action="store_true",
dest="force_flaky",
default=False,
help="If this option is specified, we will treat all tests as "
"flaky."
)
add_option(
'--max-runs',
action="store",
dest="max_runs",
type="int",
default=2,
help="If --force-flaky is specified, we will run each test at "
"most this many times (unless the test has its own flaky "
"decorator)."
)
add_option(
'--min-passes',
action="store",
dest="min_passes",
type="int",
default=1,
help="If --force-flaky is specified, we will run each test at "
"least this many times (unless the test has its own flaky "
"decorator)."
)
def _add_flaky_report(self, stream):
"""
Baseclass override. Write details about flaky tests to the test report.
:param stream:
The test stream to which the report can be written.
:type stream:
`file`
"""
stream.write('===Flaky Test Report===\n\n')
stream.write(self._stream.getvalue())
stream.write('\n===End Flaky Test Report===\n')
@classmethod
def _copy_flaky_attributes(cls, test, test_class):
"""
Copy flaky attributes from the test callable or class to the test.
:param test:
The test that is being prepared to run
:type test:
:class:`nose.case.Test`
"""
_, test_callable, _ = cls._get_test_declaration_callable_and_name(test)
for attr, value in cls._get_flaky_attributes(test_class).items():
already_set = hasattr(test, attr)
if already_set:
continue
attr_on_callable = getattr(test_callable, attr, None)
if attr_on_callable is not None:
cls._set_flaky_attribute(test, attr, attr_on_callable)
elif value is not None:
cls._set_flaky_attribute(test, attr, value)
@staticmethod
def _get_flaky_attribute(test_item, flaky_attribute):
"""
Gets an attribute describing the flaky test.
:param test_item:
The test method from which to get the attribute
:type test_item:
`callable` or :class:`nose.case.Test` or :class:`Function`
:param flaky_attribute:
The name of the attribute to get
:type flaky_attribute:
`unicode`
:return:
The test callable's attribute, or None if the test
callable doesn't have that attribute.
:rtype:
varies
"""
return getattr(
test_item,
flaky_attribute,
None,
)
@staticmethod
def _set_flaky_attribute(test_item, flaky_attribute, value):
"""
Sets an attribute on a flaky test. Uses magic __dict__ since setattr
doesn't work for bound methods.
:param test_item:
The test callable on which to set the attribute
:type test_item:
`callable` or :class:`nose.case.Test` or :class:`Function`
:param flaky_attribute:
The name of the attribute to set
:type flaky_attribute:
`unicode`
:param value:
The value to set the test callable's attribute to.
:type value:
varies
"""
test_item.__dict__[flaky_attribute] = value
@classmethod
def _has_flaky_attributes(cls, test):
"""
Returns true if the test callable in question is marked as flaky.
:param test:
The test that is being prepared to run
:type test:
:class:`nose.case.Test` or :class:`Function`
:return:
:rtype:
`bool`
"""
current_runs = cls._get_flaky_attribute(
test,
FlakyNames.CURRENT_RUNS,
)
return current_runs is not None
@classmethod
def _get_flaky_attributes(cls, test_item):
"""
Get all the flaky related attributes from the test.
:param test_item:
The test callable from which to get the flaky related attributes.
:type test_item:
`callable` or :class:`nose.case.Test` or :class:`Function`
:return:
:rtype:
`dict` of `unicode` to varies
"""
return dict((
(attr, cls._get_flaky_attribute(
test_item,
attr,
)) for attr in FlakyNames()
))
@classmethod
def _add_flaky_test_failure(cls, test, err):
"""
Store test error information on the test callable.
:param test:
The flaky test on which to update the flaky attributes.
:type test:
:class:`nose.case.Test` or :class:`Function`
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `class`, :class:`Exception`, `traceback`
"""
errs = getattr(test, FlakyNames.CURRENT_ERRORS, None) or []
cls._set_flaky_attribute(
test,
FlakyNames.CURRENT_ERRORS,
errs,
)
errs.append(err)
@classmethod
def _has_flaky_test_failed(cls, flaky):
"""
Whether or not the flaky test has failed
:param flaky:
Dictionary of flaky attributes
:type flaky:
`dict` of `unicode` to varies
:return:
True if the flaky test should be marked as failure; False if
it should be rerun.
:rtype:
`bool`
"""
max_runs, current_runs, min_passes, current_passes = (
flaky[FlakyNames.MAX_RUNS],
flaky[FlakyNames.CURRENT_RUNS],
flaky[FlakyNames.MIN_PASSES],
flaky[FlakyNames.CURRENT_PASSES],
)
runs_left = max_runs - current_runs
passes_needed = min_passes - current_passes
no_retry = passes_needed > runs_left
return no_retry and not cls._has_flaky_test_succeeded(flaky)
@staticmethod
def _has_flaky_test_succeeded(flaky):
"""
Whether or not the flaky test has succeeded
:param flaky:
Dictionary of flaky attributes
:type flaky:
`dict` of `unicode` to varies
:return:
True if the flaky test should be marked as success; False if
it should be rerun.
:rtype:
`bool`
"""
return flaky[FlakyNames.CURRENT_PASSES] >= flaky[FlakyNames.MIN_PASSES]
@classmethod
def _get_test_declaration_callable_and_name(cls, test):
"""
Get the test declaration, the test callable,
and test callable name from the test.
:param test:
The test that has raised an error or succeeded
:type test:
:class:`nose.case.Test` or :class:`Function`
:return:
The test declaration, callable and name that is being run
:rtype:
`tuple` of `object`, `callable`, `unicode`
"""
raise NotImplementedError
@classmethod
def _make_test_flaky(cls, test, max_runs, min_passes):
"""
Make a given test flaky.
:param test:
The test in question.
:type test:
:class:`nose.case.Test` or :class:`Function`
:param max_runs:
The value of the FlakyNames.MAX_RUNS attribute to use.
:type max_runs:
`int`
:param min_passes:
The value of the FlakyNames.MIN_PASSES attribute to use.
:type min_passes:
`int`
"""
attrib_dict = defaults.default_flaky_attributes(max_runs, min_passes)
for attr, value in attrib_dict.items():
cls._set_flaky_attribute(test, attr, value)
class FlakyPlugin(Plugin):
"""
Plugin for nosetests that allows retrying flaky tests.
"""
name = 'flaky'
_retry_failure_message = ' failed ({} runs remaining out of {}).'
_failure_message = ' failed; it passed {} out of the required {} times.'
def __init__(self):
super(FlakyPlugin, self).__init__()
self._logger = logging.getLogger('nose.plugins.flaky')
self._flaky_tests = []
self._stream = StringIO()
self._flaky_result = TextTestResult(self._stream, [], 0)
def handleError(self, test, err):
"""
Baseclass override. Called when a test raises an exception.
:param test:
The test that has raised an error
:type test:
:class:`nose.case.Test`
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `class`, :class:`Exception`, `traceback`
:return:
True, if the test will be rerun; False, if nose should handle it.
:rtype:
`bool`
"""
# pylint:disable=invalid-name
return self._handle_test_error_or_failure(test, err)
def handleFailure(self, test, err):
"""
Baseclass override. Called when a test fails.
:param test:
The test that has raised an error
:type test:
:class:`nose.case.Test`
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `class`, :class:`Exception`, `traceback`
:return:
True, if the test will be rerun; False, if nose should handle it.
:rtype:
`bool`
"""
# pylint:disable=invalid-name
return self._handle_test_error_or_failure(test, err)
def addSuccess(self, test):
"""
Baseclass override. Called when a test succeeds.
Count remaining retries and compare with number of required successes
that have not yet been achieved; retry if necessary.
Returning True from this method keeps the test runner from reporting
the test as a success; this way we can retry and only report as a
success if we have achieved the required number of successes.
:param test:
The test that has succeeded
:type test:
:class:`nose.case.Test`
:return:
True, if the test will be rerun; False, if nose should handle it.
:rtype:
`bool`
"""
# pylint:disable=invalid-name
test_method, test_method_name = self._get_test_method_and_name(test)
current_runs = self._get_flaky_attribute(
test_method,
FlakyNames.CURRENT_RUNS
)
if current_runs is None:
return False
current_runs += 1
current_passes = self._get_flaky_attribute(
test_method,
FlakyNames.CURRENT_PASSES
)
current_passes += 1
self._set_flaky_attribute(
test_method,
FlakyNames.CURRENT_RUNS,
current_runs
)
self._set_flaky_attribute(
test_method,
FlakyNames.CURRENT_PASSES,
current_passes
)
flaky = self._get_flaky_attributes(test_method)
min_passes = flaky[FlakyNames.MIN_PASSES]
self._stream.writelines([
unicode(test_method_name),
' passed {} out of the required {} times. '.format(
current_passes,
min_passes,
),
])
if not self._has_flaky_test_succeeded(flaky):
self._stream.write(
'Running test again until it passes {} times.\n'.format(
min_passes,
)
)
test.run(self._flaky_result)
return True
else:
self._stream.write('Success!\n')
return False
def report(self, stream):
"""
Baseclass override. Write details about flaky tests to the test report.
:param stream:
The test stream to which the report can be written.
:type stream:
`file`
"""
stream.write('===Flaky Test Report===\n\n')
stream.write(self._stream.getvalue())
stream.write('\n===End Flaky Test Report===\n')
def prepareTestCase(self, test):
"""
Baseclass override. Called right before a test case is run.
If the test is marked flaky and the test method is not, copy the
flaky attributes from the test to the test method.
:param test:
The test that has succeeded
:type test:
:class:`nose.case.Test`
"""
# pylint:disable=invalid-name
test_method, _ = self._get_test_method_and_name(test)
for attr, value in self._get_flaky_attributes(test.test).iteritems():
if value is not None:
if not hasattr(
test_method,
attr,
) or getattr(
test_method,
attr,
) is None:
self._set_flaky_attribute(test_method, attr, value)
def _log_test_failure(self, test_method_name, err, message):
"""
Add messaging about a test failure to the stream, which will be
printed by the plugin's report method.
"""
self._stream.writelines([
unicode(test_method_name),
message,
'\n\t',
unicode(err[0]),
'\n\t',
unicode(err[1].message),
'\n\t',
unicode(err[2]),
'\n',
])
def _handle_test_error_or_failure(self, test, err):
"""
Handle a flaky test error or failure.
Count remaining retries and compare with number of required successes
that have not yet been achieved; retry if necessary.
Returning True from this method keeps the test runner from reporting
the test as a failure; this way we can retry and only report as a
failure if we are out of retries.
:param test:
The test that has raised an error
:type test:
:class:`nose.case.Test`
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `class`, :class:`Exception`, `traceback`
:return:
True, if the test will be rerun; False, if nose should handle it.
:rtype:
`bool`
"""
test_method, test_method_name = self._get_test_method_and_name(test)
current_runs = self._get_flaky_attribute(
test_method,
FlakyNames.CURRENT_RUNS,
)
if current_runs is None:
return False
current_runs += 1
self._set_flaky_attribute(
test_method,
FlakyNames.CURRENT_RUNS,
current_runs,
)
self._add_flaky_test_failure(test_method, err)
flaky = self._get_flaky_attributes(test_method)
if not self._has_flaky_test_failed(flaky):
max_runs = flaky[FlakyNames.MAX_RUNS]
runs_left = max_runs - flaky[FlakyNames.CURRENT_RUNS]
message = self._retry_failure_message.format(
runs_left,
max_runs,
)
self._log_test_failure(test_method_name, err, message)
test.run(self._flaky_result)
return True
else:
min_passes = flaky[FlakyNames.MIN_PASSES]
current_passes = flaky[FlakyNames.CURRENT_PASSES]
message = self._failure_message.format(
current_passes,
min_passes,
)
self._log_test_failure(test_method_name, err, message)
return False
@staticmethod
def _get_flaky_attribute(test_method, flaky_attribute):
"""
Gets an attribute describing the flaky test.
:param test_method:
The test method from which to get the attribute
:type test_method:
`callable`
:param flaky_attribute:
The name of the attribute to get
:type flaky_attribute:
`unicode`
:return:
The test method's attribute, or None if the test method doesn't
have that attribute.
:rtype:
varies
"""
return getattr(
test_method,
flaky_attribute,
None
)
@staticmethod
def _set_flaky_attribute(test_method, flaky_attribute, value):
"""
Sets an attribute on a flaky test. Uses magic __dict__ since setattr
doesn't work for bound methods.
:param test_method:
The test method on which to set the attribute
:type test_method:
`callable`
:param flaky_attribute:
The name of the attribute to set
:type flaky_attribute:
`unicode`
:param value:
The value to set the test method's attribute to.
:type value:
varies
"""
test_method.__dict__[flaky_attribute] = value
@classmethod
def _get_flaky_attributes(cls, test_method):
"""
Get all the flaky related attributes from the test method.
:param test_method:
The test method from which to get the flaky related attributes.
:type test_method:
`callable`
:return:
:rtype:
`dict` of `unicode` to varies
"""
return {
attr: cls._get_flaky_attribute(
test_method,
attr,
) for attr in FlakyNames()
}
@classmethod
def _add_flaky_test_failure(cls, test_method, err):
"""
Store test error information on the test method.
:param test_method:
The test method from which to get the flaky related attributes.
:type test_method:
`callable`
:param err:
Information about the test failure (from sys.exc_info())
:type err:
`tuple` of `class`, :class:`Exception`, `traceback`
"""
if not hasattr(test_method, FlakyNames.CURRENT_ERRORS):
errs = []
cls._set_flaky_attribute(
test_method,
FlakyNames.CURRENT_ERRORS,
errs,
)
else:
errs = getattr(test_method, FlakyNames.CURRENT_ERRORS)
errs.append(err)
@staticmethod
def _get_test_method_name(test):
"""
Get the name of the test method from the test.
:param test:
The test that has raised an error or succeeded
:type test:
:class:`nose.case.Test`
:return:
The name of the test method that is being run by the test
:rtype:
`unicode`
"""
_, _, class_and_method_name = test.address()
first_dot_index = class_and_method_name.index('.')
test_method_name = class_and_method_name[first_dot_index + 1:]
return test_method_name
@classmethod
def _get_test_method_and_name(cls, test):
"""
Get the test method and test method name from the test.
:param test:
The test that has raised an error or succeeded
:type test:
:class:`nose.case.Test`
:return:
The test method (and its name) that is being run by the test
:rtype:
`tuple` of `callable`, `unicode`
"""
method_name = cls._get_test_method_name(test)
return getattr(test.test, method_name), method_name
@classmethod
def _has_flaky_test_failed(cls, flaky):
"""
Whether or not the flaky test has failed
:param flaky:
Dictionary of flaky attributes
:type flaky:
`dict` of `unicode` to varies
:return:
True if the flaky test should be marked as failure; False if
it should be rerun.
:rtype:
`bool`
"""
no_retry = flaky[FlakyNames.CURRENT_RUNS] >= flaky[FlakyNames.MAX_RUNS]
return no_retry and not cls._has_flaky_test_succeeded(flaky)
@staticmethod
def _has_flaky_test_succeeded(flaky):
"""
Whether or not the flaky test has succeeded
:param flaky:
Dictionary of flaky attributes
:type flaky:
`dict` of `unicode` to varies
:return:
True if the flaky test should be marked as success; False if
it should be rerun.
:rtype:
`bool`
"""
return flaky[FlakyNames.CURRENT_PASSES] >= flaky[FlakyNames.MIN_PASSES]
class _TemplateRemover(object):
"""Helper class for the clean() method.
This class exists mainly to factor out some methods.
"""
def __init__(self, html_content, pattern=ALL_PATTERN):
self.html_content = html_content
self.pattern = pattern
self._output = StringIO()
self._index = 0
self._state = 'HTML'
self._pending = []
self._pending_has_blank = False
def _reset_pending(self):
self._pending = []
self._pending_has_blank = False
def _write_content(self, end=None):
self._output.writelines(self._pending)
self._reset_pending()
self._output.write(self.html_content[self._index:end])
def get_clean_content(self):
"""Implementation of the clean() method."""
fill_chars = {'BLANK_TEMPLATE': ' ', 'ECHO_TEMPLATE': '0'}
for match in self.pattern.finditer(self.html_content):
start, end = match.start(), match.end()
tag = _get_tag(match)
if tag == 'ECHO':
self._write_content(start)
self._index = start
self._state = 'ECHO_TEMPLATE'
elif tag == 'START':
if self._index != start:
self._write_content(start)
self._index = start
self._state = 'BLANK_TEMPLATE'
elif tag == 'END':
if self._state not in ('BLANK_TEMPLATE', 'ECHO_TEMPLATE'):
# We got a closing tag but none was open. We decide to carry
# on as it may be the case that it was because of a closing
# dictionary in javascript like: var dict = {foo:{}}.
# See the note on the clean() function for more details.
continue
fill_char = fill_chars[self._state]
fill = fill_char * (end - self._index)
if self._state == 'BLANK_TEMPLATE':
self._pending.append(fill)
self._pending_has_blank = True
else:
assert not self._pending
self._output.write(fill)
self._index = end
self._state = 'HTML'
elif tag == 'SPACES':
self._pending.append(match.group('spaces'))
self._index = end
elif tag == 'NEWLINE':
if self._state == 'HTML':
if self._index != start or not self._pending_has_blank:
self._write_content(start)
self._output.write(match.group('newline'))
elif self._state == 'BLANK_TEMPLATE':
# We discard the content of this template and whatever is in
# self._pending.
self._output.write(match.group('newline'))
elif self._state == 'ECHO_TEMPLATE':
assert False, 'Echo tags should be in just one line.'
self._index = end
self._reset_pending()
assert self._state == 'HTML', 'Tag was not closed'
if self._index != len(self.html_content) or not self._pending_has_blank:
self._write_content()
return self._output.getvalue()
class S3File(object):
def __init__(self, url, key=None, secret=None, expiration_days=0, private=False, content_type=None, create=True):
from boto.s3.connection import S3Connection
from boto.s3.key import Key
if sys.version_info < (3, 0):
self.url = urlparse(url)
self.buffer = cStringIO.StringIO()
else:
self.url = urlparse3(url)
self.buffer = StringIO()
self.expiration_days = expiration_days
self.private = private
self.closed = False
self._readreq = True
self._writereq = False
self.content_type = content_type or mimetypes.guess_type(self.url.path)[0]
bucket = self.url.netloc
if bucket.endswith('.s3.amazonaws.com'):
bucket = bucket[:-17]
self.client = S3Connection(key, secret)
self.name = "s3://" + bucket + self.url.path
if create:
self.bucket = self.client.create_bucket(bucket)
else:
self.bucket = self.client.get_bucket(bucket, validate=False)
self.key = Key(self.bucket)
self.key.key = self.url.path.lstrip("/")
self.buffer.truncate(0)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def _remote_read(self):
""" Read S3 contents into internal file buffer.
Once only
"""
if self._readreq:
self.buffer.truncate(0)
if self.key.exists():
self.key.get_contents_to_file(self.buffer)
self.buffer.seek(0)
self._readreq = False
def _remote_write(self):
""" Write file contents to S3 from internal buffer.
"""
if self._writereq:
self.truncate(self.tell())
headers = {
"x-amz-acl": "private" if self.private else "public-read"
}
if self.content_type:
headers["Content-Type"] = self.content_type
if self.expiration_days:
now = datetime.datetime.utcnow()
then = now + datetime.timedelta(self.expiration_days)
headers["Expires"] = then.strftime("%a, %d %b %Y %H:%M:%S GMT")
headers["Cache-Control"] = 'max-age=%d' % (self.expiration_days * 24 * 3600,)
self.key.set_contents_from_file(self.buffer, headers=headers, rewind=True)
def close(self):
""" Close the file and write contents to S3.
"""
self._remote_write()
self.buffer.close()
self.closed = True
# pass-through methods
def flush(self):
self._remote_write()
def fileno(self):
return 3
def next(self):
self._remote_read()
return self.buffer.next()
def read(self, size=-1):
self._remote_read()
return self.buffer.read(size)
def readline(self, size=-1):
self._remote_read()
return self.buffer.readline(size)
def readlines(self, sizehint=-1):
self._remote_read()
return self.buffer.readlines(sizehint)
def xreadlines(self):
self._remote_read()
return self.buffer
def seek(self, offset, whence=os.SEEK_SET):
self.buffer.seek(offset, whence)
# if it looks like we are moving in the file and we have not written
# anything then we probably should read the contents
if self.tell() != 0 and self._readreq and not self._writereq:
self._remote_read()
self.buffer.seek(offset, whence)
def tell(self):
return self.buffer.tell()
def truncate(self, size=None):
self._writereq = True
self.buffer.truncate(size or self.tell())
def write(self, s):
self._writereq = True
self.buffer.write(s)
def writelines(self, sequence):
self._writereq = True
self.buffer.writelines(sequence)
