def to_string(self):
buf = StringIO()
buf.write('<table border="0" cellpadding="0" cellspacing="0">')
buf.write('<thead><tr>')
for col in self.cols:
if self.col_header_style[col] is not None:
buf.write('<th class="%s">' % e(self.col_header_style[col]))
else:
buf.write('<th>')
buf.write(e(col))
buf.write('</th>')
buf.write('</tr></thead><tbody>')
counter = 1
for row in self.data:
style = ' class="even"' if counter % 2 == 0 else ''
counter += 1
buf.write('<tr%s>' % style)
for i in xrange(self.num_cols):
if i < len(row):
col = self.cols[i]
if self.col_style[col] is not None:
buf.write('<td class="%s">' % e(self.col_style[col]))
else:
buf.write('<td>')
buf.write(row[i])
buf.write('</td>')
buf.write('</tr>')
buf.write('</tbody></table>')
return buf.getvalue()
def done(self):
if self.res_status == '206':
# TODO: check entity headers
# TODO: check content-range
if ('gzip' in self.red.parsed_hdrs.get('content-encoding', [])) == \
('gzip' not in self.parsed_hdrs.get('content-encoding', [])):
self.red.setMessage('header-accept-ranges header-content-encoding',
rs.RANGE_NEG_MISMATCH, self)
return
if self.res_body == self.range_target:
self.red.partial_support = True
self.red.setMessage('header-accept-ranges', rs.RANGE_CORRECT, self)
else:
# the body samples are just bags of bits
self.red.partial_support = False
self.red.setMessage('header-accept-ranges', rs.RANGE_INCORRECT, self,
range="bytes=%s-%s" % (self.range_start, self.range_end),
range_expected=e(self.range_target.encode('string_escape')),
range_expected_bytes = len(self.range_target),
range_received=e(self.res_body.encode('string_escape')),
range_received_bytes = self.res_body_len
)
# TODO: address 416 directly
elif self.res_status == self.red.res_status:
self.red.partial_support = False
self.red.setMessage('header-accept-ranges', rs.RANGE_FULL)
else:
self.red.setMessage('header-accept-ranges', rs.RANGE_STATUS,
range_status=self.res_status,
enc_range_status=e(self.res_status))
def format_body_sample(self, red):
"""show the stored body sample"""
if not hasattr(red, "body_sample"):
return ""
try:
uni_sample = unicode(red.body_sample,
red.link_parser.doc_enc or red.link_parser.http_enc, 'ignore')
except LookupError:
uni_sample = unicode(red.body_sample, 'utf-8', 'ignore')
safe_sample = e(uni_sample)
message = ""
for tag, link_set in red.links.items():
for link in link_set:
def link_to(matchobj):
try:
qlink = urljoin(red.link_parser.base, link)
except ValueError, why:
pass # TODO: pass link problem upstream?
# e.g., ValueError("Invalid IPv6 URL")
return r"%s<a href='%s' class='nocode'>%s</a>%s" % (
matchobj.group(1),
u"?uri=%s" % e_query_arg(qlink),
e(link),
matchobj.group(1)
)
safe_sample = re.sub(r"(['\"])%s\1" % \
re.escape(link), link_to, safe_sample)
def format_category(self, category, red):
"""
For a given category, return all of the non-detail
messages in it as an HTML list.
"""
messages = [msg for msg in red.messages if msg.category == category]
if not messages:
return nl
out = []
if [msg for msg in messages]:
out.append(u"<h3>%s</h3>\n<ul>\n" % category)
for m in messages:
out.append(
u"<li class='%s msg' data-subject='%s' data-name='msgid-%s'><span>%s</span></li>"
% (m.level, e(
m.subject), id(m), e(m.summary[self.lang] % m.vars)))
self.hidden_text.append(
("msgid-%s" % id(m), m.text[self.lang] % m.vars))
subreq = red.subreqs.get(m.subrequest, None)
smsgs = [msg for msg in getattr(subreq, "messages", []) if \
msg.level in [rs.l.BAD]]
if smsgs:
out.append(u"<ul>")
for sm in smsgs:
out.append(
u"<li class='%s msg' data-subject='%s' name='msgid-%s'><span>%s</span></li>"
% (sm.level, e(sm.subject), id(sm),
e(sm.summary[self.lang] % sm.vars)))
self.hidden_text.append(
("msgid-%s" % id(sm), sm.text[self.lang] % sm.vars))
out.append(u"</ul>")
out.append(u"</ul>\n")
return nl.join(out)
def done(self):
if self.res_body_len > 0:
savings = int(100 * ((float(self.res_body_len) - \
self.red.res_body_len) / self.res_body_len))
else:
savings = 0
self.red.gzip_support = True
self.red.gzip_savings = savings
self.red.setMessage('header-content-encoding', rs.CONNEG_GZIP, self,
savings=savings,
orig_size=self.res_body_len
)
vary_headers = self.red.parsed_hdrs.get('vary', [])
if (not "accept-encoding" in vary_headers) and (not "*" in vary_headers):
self.red.setMessage('header-vary header-%s', rs.CONNEG_NO_VARY)
# FIXME: verify that the status/body/hdrs are the same; if it's different, alert
no_conneg_vary_headers = self.parsed_hdrs.get('vary', [])
if 'gzip' in self.parsed_hdrs.get('content-encoding', []):
self.red.setMessage('header-vary header-content-encoding',
rs.CONNEG_GZIP_WITHOUT_ASKING)
if no_conneg_vary_headers != vary_headers:
self.red.setMessage('header-vary', rs.VARY_INCONSISTENT,
conneg_vary=e(", ".join(vary_headers)),
no_conneg_vary=e(", ".join(no_conneg_vary_headers))
)
if self.parsed_hdrs.get('etag', 1) == self.red.parsed_hdrs.get('etag', 2):
self.red.setMessage('header-etag', rs.ETAG_DOESNT_CHANGE) # TODO: weakness?
def format_response(self, red):
"Return the HTTP response line and headers as HTML"
return \
u" <span class='status'>HTTP/%s %s %s</span>\n" % (
e(str(red.res_version)),
e(str(red.res_status)),
e(red.res_phrase)
) + \
nl.join([self.format_header(f,v) for (f,v) in red.res_hdrs])
def format_problems(self):
out = ['<br /><h2>Problems</h2><ol>']
for m in self.problems:
out.append(
u"<li class='%s %s msg' name='msgid-%s'><span>%s</span></li>" %
(m.level, e(m.subject), id(m), e(
m.summary[self.lang] % m.vars)))
self.hidden_text.append(
("msgid-%s" % id(m), m.text[self.lang] % m.vars))
out.append(u"</ol>\n")
return nl.join(out)
def format_header(self, name, value):
"Return an individual HTML header as HTML"
token_name = "header-%s" % name.lower()
py_name = "HDR_" + name.upper().replace("-", "_")
if hasattr(defns, py_name) and token_name not in \
[i[0] for i in self.hidden_text]:
defn = getattr(defns, py_name)[self.lang] % {
'field_name': name,
}
self.hidden_text.append((token_name, defn))
return u" <span name='%s' class='hdr'>%s:%s</span>" % (
e(token_name), e(name), self.header_presenter.Show(name, value))
def format_header(self, name, value, offset):
"Return an individual HTML header as HTML"
token_name = "header-%s" % name.lower()
py_name = "HDR_" + name.upper().replace("-", "_")
if hasattr(defns, py_name) and token_name not in \
[i[0] for i in self.hidden_text]:
defn = getattr(defns, py_name)[self.lang] % {
'field_name': name,
}
self.hidden_text.append((token_name, defn))
return u" <span data-offset='%s' data-name='%s' class='hdr'>%s:%s</span>" % (
offset, e(name.lower()), e(name),
self.header_presenter.Show(name, value))
def join(subject, values, red):
directives = {}
warned = False
for (attr, attr_value) in values:
if directives.has_key(attr) and not warned:
red.set_message(subject, rs.UA_COMPATIBLE_REPEAT)
warned = True
directives[attr] = attr_value
uac_list = u"\n".join([u"<li>%s - %s</li>" % (e(k), e(v)) for
k, v in values])
red.set_message(subject, rs.UA_COMPATIBLE, uac_list=uac_list)
return directives
def join(subject, values, red):
directives = {}
warned = False
for (attr, attr_value) in values:
if directives.has_key(attr) and not warned:
red.set_message(subject, rs.UA_COMPATIBLE_REPEAT)
warned = True
directives[attr] = attr_value
uac_list = u"\n".join(
[u"<li>%s - %s</li>" % (e(k), e(v)) for k, v in values])
red.set_message(subject, rs.UA_COMPATIBLE, uac_list=uac_list)
return directives
def header(self, title):
return """
<html>
<head>
<title>%s</title>
<style type="text/css">
%s
%s
</style>
</head>
<body>
<h1>%s</h1>
""" % (e(title), self.css(), self.colorize_css(), e(title))
def format_problems(self):
out = ['<br /><h2>Problems</h2><ol>']
for m in self.problems:
out.append(u"<li class='%s %s msg' name='msgid-%s'><span>%s</span></li>" % (
m.level,
e(m.subject),
id(m),
e(m.summary[self.lang] % m.vars)
)
)
self.hidden_text.append(
("msgid-%s" % id(m), m.text[self.lang] % m.vars)
)
out.append(u"</ol>\n")
return nl.join(out)
def x_ua_compatible(self, name, values):
directives = {}
for directive in values:
try:
attr, value = directive.split("=", 1)
except ValueError:
attr = directive
value = None
if directives.has_key(attr):
self.setMessage(name, rs.UA_COMPATIBLE_REPEAT)
directives[attr] = value
uac_list = u"\n".join([u"<li>%s - %s</li>" % (e(k), e(v)) for
k, v in directives.items()])
self.setMessage(name, rs.UA_COMPATIBLE, uac_list=uac_list)
return directives
def format_response(self, red):
"Return the HTTP response line and headers as HTML"
offset = 0
headers = []
for (name, value) in red.res_hdrs:
offset += 1
headers.append(self.format_header(name, value, offset))
return \
u" <span class='status'>HTTP/%s %s %s</span>\n" % (
e(str(red.res_version)),
e(str(red.res_status)),
e(red.res_phrase)
) + \
nl.join(headers)
def location(self, name, values):
if self.red.res_status not in ["201", "300", "301", "302", "303", "305", "307"]:
self.setMessage(name, rs.LOCATION_UNDEFINED)
if not re.match(r"^\s*%s\s*$" % URI, values[-1], re.VERBOSE):
self.setMessage(name, rs.LOCATION_NOT_ABSOLUTE,
full_uri=e(urljoin(self.red.uri, values[-1])))
return values[-1]
def finish_output(self, red):
self.final_status()
if red.res_complete:
self.header_presenter = HeaderPresenter(red.uri)
self.output(self.template % {
'response': self.format_response(red),
'options': self.format_options(red),
'messages': nl.join([self.format_category(cat, red) \
for cat in self.msg_categories]),
'body': self.format_body_sample(red),
'footer': self.format_footer(),
'hidden_list': self.format_hidden_list(),
})
else:
if red.res_error == None:
pass # usually a global timeout...
elif red.res_error['desc'] == nberror.ERR_CONNECT['desc']:
self.output(self.error_template % \
"Could not connect to the server (%s)" % \
red.res_error.get('detail', "unknown"))
elif red.res_error['desc'] == nberror.ERR_URL['desc']:
self.output(self.error_template % red.res_error.get(
'detail', "RED can't fetch that URL."))
elif red.res_error['desc'] == nberror.ERR_READ_TIMEOUT['desc']:
self.output(self.error_template % red.res_error['desc'])
elif red.res_error['desc'] == nberror.ERR_HTTP_VERSION['desc']:
self.output(self.error_template % \
"<code>%s</code> isn't HTTP." % \
e(red.res_error.get('detail', '')[:20]))
else:
raise AssertionError, "Unknown incomplete response error."
def done(self):
if self.state.res_status == '304':
self.base.inm_support = True
self.setMessage('header-etag', rs.INM_304, self.state)
# TODO : check Content- headers, esp. length.
elif self.state.res_status == self.base.res_status:
if self.state.res_body_md5 == self.base.res_body_md5:
self.base.inm_support = False
self.setMessage('header-etag', rs.INM_FULL)
else:
if self.base.parsed_hdrs['etag'] == \
self.state.parsed_hdrs['etag']:
if self.base.parsed_hdrs['etag'][0]: # weak
self.setMessage('header-etag', rs.INM_DUP_ETAG_WEAK)
else: # strong
self.setMessage('header-etag', rs.INM_DUP_ETAG_STRONG,
etag=self.base.parsed_hdrs['etag'])
else:
self.setMessage('header-etag', rs.INM_UNKNOWN)
else:
self.setMessage('header-etag',
rs.INM_STATUS,
inm_status = self.state.res_status,
enc_inm_status = e(self.state.res_status)
)
# TODO: check entity headers
def finish_output(self):
self.final_status()
if self.red.res_complete:
self.header_presenter = HeaderPresenter(self.red.uri)
self.output(self.template % {
'response': self.format_response(self.red),
'options': self.format_options(self.red),
'messages': nl.join([self.format_category(cat, self.red) \
for cat in self.msg_categories]),
'body': self.format_body_sample(self.red),
'footer': self.format_footer(),
'hidden_list': self.format_hidden_list(),
})
else:
if self.red.res_error == None:
pass # usually a global timeout...
elif isinstance(self.red.res_error, httperr.ConnectError):
self.output(self.error_template % \
"Could not connect to the server (%s)" % \
self.red.res_error.detail or "unknown")
elif isinstance(self.red.res_error, httperr.UrlError):
self.output(self.error_template % \
self.red.res_error.detail or "RED can't fetch that URL.")
elif isinstance(self.red.res_error, httperr.ReadTimeoutError):
self.output(self.error_template % self.red.res_error.desc)
elif isinstance(self.red.res_error, httperr.HttpVersionError):
self.output(self.error_template % \
"<code>%s</code> isn't HTTP." % \
e(self.red.res_error.detail or '')[:20])
else:
raise AssertionError, "Unknown incomplete response error."
self.done()
def format_body_sample(self, red):
"""show the stored body sample"""
if not hasattr(red, "body_sample"):
return ""
try:
uni_sample = unicode(red.body_sample,
red.link_parser.doc_enc or red.link_parser.http_enc, 'ignore')
except LookupError:
uni_sample = unicode(red.body_sample, 'utf-8', 'ignore')
safe_sample = e(uni_sample)
message = ""
for tag, link_set in red.links.items():
for link in link_set:
def link_to(matchobj):
return r"%s<a href='%s' class='nocode'>%s</a>%s" % (
matchobj.group(1),
u"?uri=%s" % e_query_arg(
urljoin(red.link_parser.base, link)),
e(link),
matchobj.group(1)
)
safe_sample = re.sub(r"(['\"])%s\1" % \
re.escape(link), link_to, safe_sample)
if not self.sample_complete:
message = "<p class='note'>RED isn't showing the whole body, because it's so big!</p>"
return """<pre class="prettyprint">%s</pre>\n%s""" % (
safe_sample, message)
def done(self):
if self.state.res_status == '304':
self.base.inm_support = True
self.set_message('header-etag', rs.INM_304, self.state)
# TODO : check Content- headers, esp. length.
elif self.state.res_status == self.base.res_status:
if self.state.res_body_md5 == self.base.res_body_md5:
self.base.inm_support = False
self.set_message('header-etag', rs.INM_FULL)
else: # bodies are different
if self.base.parsed_hdrs['etag'] == \
self.state.parsed_hdrs.get('etag', 1):
if self.base.parsed_hdrs['etag'][0]: # weak
self.set_message('header-etag', rs.INM_DUP_ETAG_WEAK)
else: # strong
self.set_message('header-etag',
rs.INM_DUP_ETAG_STRONG,
etag=self.base.parsed_hdrs['etag'])
else:
self.set_message('header-etag', rs.INM_UNKNOWN)
else:
self.set_message('header-etag',
rs.INM_STATUS,
inm_status=self.state.res_status,
enc_inm_status=e(self.state.res_status
or '(unknown)'))
def __init__(self, test_uri):
"""
Constractor
@param test_uri: Test Uri
"""
self.test_uri = test_uri
try:
self.red = InspectingResourceExpertDroid(self.test_uri)
self.result = ""
self.done = False
self.groups = []
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
if self.red.res_complete:
self.result = self._generate_output_xml(test_uri).toprettyxml()
else:
error_string = ""
if self.red.res_error['desc'] == nbhttp.error.ERR_CONNECT['desc']:
error_string = "Could not connect to the server (%s)" % self.red.res_error.get('detail', "unknown")
elif self.red.res_error['desc'] == nbhttp.error.ERR_URL['desc']:
error_string = self.red.res_error.get('detail', "RED can't fetch that URL.")
elif self.red.res_error['desc'] == nbhttp.error.ERR_READ_TIMEOUT['desc']:
error_string = self.red.res_error['desc']
elif self.red.res_error['desc'] == nbhttp.error.ERR_HTTP_VERSION['desc']:
error_string = "<code>%s</code> isn't HTTP." % e(self.red.res_error.get('detail', '')[:20])
else:
raise AssertionError, "Unidentified incomplete response error."
self.result = self._generate_error_xml(error_string).toprettyxml()
except:
import traceback
logging.error(traceback.format_exc())
self.result = """<?xml version="1.0" ?>
def start_output(self):
if self.kw.get('is_saved', None):
extra_title = " <span class='save'>saved results for...</span>"
else:
extra_title = ""
if self.kw.get('is_blank', None):
extra_body_class = "blank"
else:
extra_body_class = ""
if self.kw.get('descend', False):
descend = "&descend=True"
else:
descend = ''
self.output(html_header.__doc__ % {
'static': static_root,
'version': droid.__version__,
'html_uri': e(self.uri),
'js_uri': e_js(self.uri),
'config': urllib.quote(json.dumps({
'redbot_uri': self.uri,
'redbot_req_hdrs': self.req_hdrs,
'redbot_version': droid.__version__
})),
'js_req_hdrs': ", ".join(['["%s", "%s"]' % (
e_js(n), e_js(v)) for n,v in self.req_hdrs]),
'extra_js': self.format_extra('.js'),
'test_id': self.kw.get('test_id', ""),
'extra_title': extra_title,
'extra_body_class': extra_body_class,
'descend': descend
})
def done(self):
if self.state.res_body_len > 0:
savings = int(100 *
(
(float(self.state.res_body_len) - \
self.base.res_body_len
) / self.state.res_body_len
)
)
else:
savings = 0
self.base.gzip_support = True
self.base.gzip_savings = savings
if savings >= 0:
self.setMessage('header-content-encoding',
rs.CONNEG_GZIP_GOOD,
savings=savings,
orig_size=f_num(self.state.res_body_len),
gzip_size=f_num(self.base.res_body_len)
)
else:
self.setMessage('header-content-encoding',
rs.CONNEG_GZIP_BAD,
savings=abs(savings),
orig_size=f_num(self.state.res_body_len),
gzip_size=f_num(self.base.res_body_len)
)
vary_headers = self.base.parsed_hdrs.get('vary', [])
if (not "accept-encoding" in vary_headers) \
and (not "*" in vary_headers):
self.setMessage('header-vary header-%s', rs.CONNEG_NO_VARY)
# TODO: verify that the status/body/hdrs are the same;
# if it's different, alert
no_conneg_vary_headers = self.state.parsed_hdrs.get('vary', [])
if 'gzip' in self.state.parsed_hdrs.get('content-encoding', []) or \
'x-gzip' in self.state.parsed_hdrs.get('content-encoding', []):
self.setMessage('header-vary header-content-encoding',
rs.CONNEG_GZIP_WITHOUT_ASKING)
if no_conneg_vary_headers != vary_headers:
self.setMessage('header-vary',
rs.VARY_INCONSISTENT,
conneg_vary=e(", ".join(vary_headers)),
no_conneg_vary=e(", ".join(no_conneg_vary_headers))
)
if self.state.parsed_hdrs.get('etag', 1) \
== self.base.parsed_hdrs.get('etag', 2):
self.setMessage('header-etag', rs.ETAG_DOESNT_CHANGE)
def done(self):
if self.state.res_status == '206':
# TODO: check entity headers
# TODO: check content-range
ce = 'content-encoding'
if ('gzip' in self.base.parsed_hdrs.get(ce, [])) == \
('gzip' not in self.state.parsed_hdrs.get(ce, [])):
self.set_message(
'header-accept-ranges header-content-encoding',
rs.RANGE_NEG_MISMATCH
)
return
if self.state.parsed_hdrs.get('etag', 1) == \
self.base.parsed_hdrs.get('etag', 2):
if self.state.res_body == self.range_target:
self.base.partial_support = True
self.set_message('header-accept-ranges', rs.RANGE_CORRECT)
else:
# the body samples are just bags of bits
self.base.partial_support = False
self.set_message('header-accept-ranges',
rs.RANGE_INCORRECT,
range="bytes=%s-%s" % (
self.range_start, self.range_end
),
range_expected=e(
self.range_target.encode('string_escape')
),
range_expected_bytes = f_num(len(self.range_target)),
range_received = e(
self.state.res_body.encode('string_escape')
),
range_received_bytes = f_num(self.state.res_body_len)
)
else:
self.set_message('header-accept-ranges', rs.RANGE_CHANGED)
# TODO: address 416 directly
elif self.state.res_status == self.base.res_status:
self.base.partial_support = False
self.set_message('header-accept-ranges', rs.RANGE_FULL)
else:
self.set_message('header-accept-ranges',
rs.RANGE_STATUS,
range_status=self.state.res_status,
enc_range_status=e(self.state.res_status or '(unknown)')
)
def _response_body(self, chunk):
"Process a chunk of the response body."
state = self.state
state.res_body_sample.append((state.res_body_len, chunk))
if len(state.res_body_sample) > 4:
state.res_body_sample.pop(0)
self._md5_processor.update(chunk)
state.res_body_len += len(chunk)
if state.res_status == "206":
# Store only partial responses completely, for error reporting
state.res_body += chunk
state.res_body_decode_len += len(chunk)
# Don't actually try to make sense of a partial body...
return
content_codings = state.parsed_hdrs.get('content-encoding', [])
content_codings.reverse()
for coding in content_codings:
# TODO: deflate support
if coding == 'gzip' and self._gzip_ok:
if not self._in_gzip_body:
self._gzip_header_buffer += chunk
try:
chunk = self._read_gzip_header(
self._gzip_header_buffer
)
self._in_gzip_body = True
except IndexError:
return # not a full header yet
except IOError, gzip_error:
state.setMessage('header-content-encoding',
rs.BAD_GZIP,
gzip_error=e(str(gzip_error))
)
self._gzip_ok = False
return
try:
chunk = self._gzip_processor.decompress(chunk)
except zlib.error, zlib_error:
state.setMessage(
'header-content-encoding',
rs.BAD_ZLIB,
zlib_error=e(str(zlib_error)),
ok_zlib_len=f_num(state.res_body_sample[-1][0]),
chunk_sample=e(chunk[:20].encode('string_escape'))
)
self._gzip_ok = False
return
def BARE_URI(self, name, value):
"Present a bare URI header value"
value = value.rstrip()
svalue = value.lstrip()
space = len(value) - len(svalue)
return u"%s<a href='?uri=%s'>%s</a>" % (
" " * space, e_query_arg(urljoin(
self.URI, svalue)), self.I(e(svalue), len(name)))
def parse(subject, value, red):
# check to see if there are any non-gzip encodings, because
# that's the only one we ask for.
if value.lower() != 'gzip':
red.set_message(subject,
rs.ENCODING_UNWANTED,
unwanted_codings=e(value))
return value.lower()
def link_to(matchobj):
return r"%s<a href='%s' class='nocode'>%s</a>%s" % (
matchobj.group(1),
u"?uri=%s" % e_query_arg(
urljoin(red.link_parser.base, link)),
e(link),
matchobj.group(1)
)
def status(self, message):
"Update the status bar of the browser"
self.output(u"""
<script>
<!-- %3.3f
window.status="%s";
-->
</script>
""" % (nbhttp.now() - self.start, e(message)))
def content_encoding(self, name, values):
values = [v.lower() for v in values]
for value in values:
# check to see if there are any non-gzip encodings, because
# that's the only one we ask for.
if value != 'gzip':
self.setMessage(name, rs.ENCODING_UNWANTED, encoding=e(value))
break
return values
def status(self, message):
"Update the status bar of the browser"
self.output(u"""
<script>
<!-- %3.3f
window.status="%s";
-->
</script>
""" % (thor.time() - self.start, e(message)))
def parse(subject, value, red):
if red.res_status not in [
"201", "300", "301", "302", "303", "305", "307"
]:
red.set_message(subject, rs.LOCATION_UNDEFINED)
if not re.match(r"^\s*%s\s*$" % syntax.URI, value, re.VERBOSE):
red.set_message(subject, rs.LOCATION_NOT_ABSOLUTE,
full_uri=e(urljoin(red.uri, value)))
return value
def parse(subject, value, red):
# check to see if there are any non-gzip encodings, because
# that's the only one we ask for.
if value.lower() != 'gzip':
red.set_message(subject,
rs.ENCODING_UNWANTED,
unwanted_codings=e(value)
)
return value.lower()
def parse_params(red, subject, instr, nostar=None, delim=";"):
"""
Parse parameters into a dictionary.
@param red: the red instance to use
@param subject: the subject identifier
@param instr: string to be parsed
@param nostar: list of parameters that definitely don't get a star. If
True, no parameter can be starred.
@param delim: delimter between params, default ";"
@return: dictionary of {name: value}
"""
param_dict = {}
instr = instr.encode("ascii")
for param in split_string(instr, syntax.PARAMETER, r"\s*%s\s*" % delim):
try:
k, v = param.split("=", 1)
except ValueError:
param_dict[param.lower()] = None
continue
k_norm = k.lower() # TODO: warn on upper-case in param?
if param_dict.has_key(k_norm):
red.set_message(subject, rs.PARAM_REPEATS, param=e(k_norm))
if v[0] == v[-1] == "'":
red.set_message(
subject, rs.PARAM_SINGLE_QUOTED, param=e(k_norm), param_val=e(v), param_val_unquoted=e(v[1:-1])
)
if k[-1] == "*":
if nostar is True or (nostar and k_norm[:-1] in nostar):
red.set_message(subject, rs.PARAM_STAR_BAD, param=e(k_norm[:-1]))
else:
if v[0] == '"' and v[-1] == '"':
red.set_message(subject, rs.PARAM_STAR_QUOTED, param=e(k_norm))
v = unquote_string(v)
try:
enc, lang, esc_v = v.split("'", 3)
except ValueError:
red.set_message(subject, rs.PARAM_STAR_ERROR, param=e(k_norm))
continue
enc = enc.lower()
lang = lang.lower()
if enc == "":
red.set_message(subject, rs.PARAM_STAR_NOCHARSET, param=e(k_norm))
continue
elif enc not in ["utf-8"]:
red.set_message(subject, rs.PARAM_STAR_CHARSET, param=e(k_norm), enc=e(enc))
continue
# TODO: catch unquoting errors, range of chars, charset
unq_v = urllib.unquote(esc_v)
dec_v = unq_v.decode(enc) # ok, because we limit enc above
param_dict[k_norm] = dec_v
else:
param_dict[k_norm] = unquote_string(v)
return param_dict
def _response_body(self, chunk):
"Process a chunk of the response body."
state = self.state
state.res_body_sample.append((state.res_body_len, chunk))
if len(state.res_body_sample) > 4:
state.res_body_sample.pop(0)
self._md5_processor.update(chunk)
state.res_body_len += len(chunk)
if state.res_status == "206":
# Store only partial responses completely, for error reporting
state.res_body += chunk
state.res_body_decode_len += len(chunk)
# Don't actually try to make sense of a partial body...
return
content_codings = state.parsed_hdrs.get('content-encoding', [])
content_codings.reverse()
for coding in content_codings:
# TODO: deflate support
if coding == 'gzip' and self._gzip_ok:
if not self._in_gzip_body:
self._gzip_header_buffer += chunk
try:
chunk = self._read_gzip_header(
self._gzip_header_buffer)
self._in_gzip_body = True
except IndexError:
return # not a full header yet
except IOError, gzip_error:
state.set_message('header-content-encoding',
rs.BAD_GZIP,
gzip_error=e(str(gzip_error)))
self._gzip_ok = False
return
try:
chunk = self._gzip_processor.decompress(chunk)
except zlib.error, zlib_error:
state.set_message(
'header-content-encoding',
rs.BAD_ZLIB,
zlib_error=e(str(zlib_error)),
ok_zlib_len=f_num(state.res_body_sample[-1][0]),
chunk_sample=e(chunk[:20].encode('string_escape')))
self._gzip_ok = False
return
def done(self):
if self.state.res_status == '206':
# TODO: check entity headers
# TODO: check content-range
ce = 'content-encoding'
if ('gzip' in self.base.parsed_hdrs.get(ce, [])) == \
('gzip' not in self.state.parsed_hdrs.get(ce, [])):
self.set_message(
'header-accept-ranges header-content-encoding',
rs.RANGE_NEG_MISMATCH)
return
if self.state.parsed_hdrs.get('etag', 1) == \
self.base.parsed_hdrs.get('etag', 2):
if self.state.res_body == self.range_target:
self.base.partial_support = True
self.set_message('header-accept-ranges', rs.RANGE_CORRECT)
else:
# the body samples are just bags of bits
self.base.partial_support = False
self.set_message(
'header-accept-ranges',
rs.RANGE_INCORRECT,
range="bytes=%s-%s" %
(self.range_start, self.range_end),
range_expected=e(
self.range_target.encode('string_escape')),
range_expected_bytes=f_num(len(self.range_target)),
range_received=e(
self.state.res_body.encode('string_escape')),
range_received_bytes=f_num(self.state.res_body_len))
else:
self.set_message('header-accept-ranges', rs.RANGE_CHANGED)
# TODO: address 416 directly
elif self.state.res_status == self.base.res_status:
self.base.partial_support = False
self.set_message('header-accept-ranges', rs.RANGE_FULL)
else:
self.set_message('header-accept-ranges',
rs.RANGE_STATUS,
range_status=self.state.res_status,
enc_range_status=e(self.state.res_status
or '(unknown)'))
def setMessage(self, name, msg, **vars):
if name:
ident = 'status %s' % name
else:
ident = 'status'
self.red.setMessage(ident, msg,
status=self.red.res_status,
enc_status=e(self.red.res_status),
**vars
)
def join(subject, values, red):
unwanted = set([c for c in values if c not in
['chunked', 'identity']]
) or False
if unwanted:
red.set_message(subject, rs.TRANSFER_CODING_UNWANTED,
unwanted_codings=e(", ".join(unwanted)))
if 'identity' in values:
red.set_message(subject, rs.TRANSFER_CODING_IDENTITY)
return values
def BARE_URI(self, name, value):
"Present a bare URI header value"
value = value.rstrip()
svalue = value.lstrip()
space = len(value) - len(svalue)
return u"%s<a href='?uri=%s'>%s</a>" % (
" " * space,
e_query_arg(urljoin(self.URI, svalue)),
self.I(e(svalue), len(name))
)
def format_category(self, category, red):
"""
For a given category, return all of the non-detail
messages in it as an HTML list.
"""
messages = [msg for msg in red.messages if msg.category == category]
if not messages:
return nl
out = []
if [msg for msg in messages]:
out.append(u"<h3>%s</h3>\n<ul>\n" % category)
for m in messages:
out.append(
u"<li class='%s %s msg' name='msgid-%s'><span>%s</span></li>" % (
m.level,
e(m.subject),
id(m),
e(m.summary[self.lang] % m.vars)
)
)
self.hidden_text.append(
("msgid-%s" % id(m), m.text[self.lang] % m.vars)
)
subreq = red.subreqs.get(m.subrequest, None)
smsgs = [msg for msg in getattr(subreq, "messages", []) if \
msg.level in [rs.l.BAD]]
if smsgs:
out.append(u"<ul>")
for sm in smsgs:
out.append(u"<li class='%s %s msg' name='msgid-%s'><span>%s</span></li>" % (
sm.level,
e(sm.subject),
id(sm),
e(sm.summary[self.lang] % sm.vars)
)
)
self.hidden_text.append(
("msgid-%s" % id(sm), sm.text[self.lang] % sm.vars)
)
out.append(u"</ul>")
out.append(u"</ul>\n")
return nl.join(out)
def Show(self, name, value):
"""
Return the given header name/value pair after
presentation processing.
"""
name = name.lower()
name_token = name.replace('-', '_')
if name_token[0] != "_" and hasattr(self, name_token):
return getattr(self, name_token)(name, value)
else:
return self.I(e(value), len(name))
def Show(self, name, value):
"""
Return the given header name/value pair after
presentation processing.
"""
name = name.lower()
name_token = name.replace('-', '_')
if name_token[0] != "_" and hasattr(self, name_token):
return getattr(self, name_token)(name, value)
else:
return self.I(e(value), len(name))
def finish_output(self):
self.final_status()
if self.red.res_complete:
self.header_presenter = HeaderPresenter(self.red.uri)
self.output(self.template % {
'response': self.format_response(self.red),
'options': self.format_options(self.red),
'messages': nl.join([self.format_category(cat, self.red) \
for cat in self.msg_categories]),
'body': self.format_body_sample(self.red),
'footer': self.format_footer(),
'hidden_list': self.format_hidden_list(),
})
else:
if self.red.res_error == None:
pass # usually a global timeout...
elif isinstance(self.red.res_error, httperr.ConnectError):
self.output(self.error_template % \
"Could not connect to the server (%s)" % \
self.red.res_error.detail or "unknown")
elif isinstance(self.red.res_error, httperr.UrlError):
self.output(self.error_template % \
"URL error: %s" % self.red.res_error.detail \
or "RED can't fetch that URL.")
elif isinstance(self.red.res_error, httperr.DuplicateCLError):
self.output(self.error_template % self.red.res_error.desc)
elif isinstance(self.red.res_error, httperr.MalformedCLError):
self.output(self.error_template % \
"%s (<code>%s</code>)" % (
self.red.res_error.desc, e(self.red.res_error.detail)
))
elif isinstance(self.red.res_error, httperr.ReadTimeoutError):
self.output(self.error_template % self.red.res_error.desc)
elif isinstance(self.red.res_error, httperr.HttpVersionError):
self.output(self.error_template % \
"<code>%s</code> isn't HTTP." % \
e(self.red.res_error.detail or '')[:20])
else:
raise AssertionError, \
"Unknown incomplete response error %s" % self.red.res_error
self.done()
def parse(subject, value, red):
try:
link, params = value.split(";", 1)
except ValueError:
link, params = value, ''
link = link[1:-1] # trim the angle brackets
param_dict = rh.parse_params(red, subject, params,
['rel', 'rev', 'anchor', 'hreflang', 'type', 'media'])
if param_dict.has_key('rel'): # relation_types
pass # TODO: check relation type
if param_dict.has_key('rev'):
red.set_message(subject, rs.LINK_REV,
link=e(link), rev=e(param_dict['rev']))
if param_dict.has_key('anchor'): # URI-Reference
if not re.match(r"^\s*%s\s*$" % syntax.URI_reference,
param_dict['anchor'], re.VERBOSE):
red.set_message(subject, rs.LINK_BAD_ANCHOR,
link=e(link),
anchor=e(param_dict['anchor']))
# TODO: check media-type in 'type'
# TODO: check language tag in 'hreflang'
return link, param_dict
def format_body_sample(self, red):
"""show the stored body sample"""
if not hasattr(red, "body_sample"):
return ""
try:
uni_sample = unicode(red.body_sample, red.res_body_enc, 'ignore')
except LookupError:
uni_sample = unicode(red.body_sample, 'utf-8', 'ignore')
safe_sample = e(uni_sample)
message = ""
for tag, link_set in red.links.items():
for link in link_set:
def link_to(matchobj):
try:
qlink = urljoin(red.base_uri, link)
except ValueError, why:
pass # TODO: pass link problem upstream?
# e.g., ValueError("Invalid IPv6 URL")
return r"%s<a href='%s' class='nocode'>%s</a>%s" % (
matchobj.group(1), u"?uri=%s" % e_query_arg(qlink),
e(link), matchobj.group(1))
safe_sample = re.sub(r"(['\"])%s\1" % \
re.escape(link), link_to, safe_sample)
def _response_error(self, error):
state = self.state
state.res_done_ts = thor.time()
state.res_error = error
if isinstance(error, httperr.BodyForbiddenError):
state.set_message('header-none', rs.BODY_NOT_ALLOWED)
# elif isinstance(error, httperr.ExtraDataErr):
# state.res_body_len += len(err.get('detail', ''))
elif isinstance(error, httperr.ChunkError):
err_msg = error.detail[:20] or ""
state.set_message('header-transfer-encoding',
rs.BAD_CHUNK,
chunk_sample=e(err_msg.encode('string_escape')))
self.done()
self.finish_task()
def done(self):
if self.state.res_status == '304':
self.base.ims_support = True
self.set_message('header-last-modified', rs.IMS_304)
# TODO : check Content- headers, esp. length.
elif self.state.res_status == self.base.res_status:
if self.state.res_body_md5 == self.base.res_body_md5:
self.base.ims_support = False
self.set_message('header-last-modified', rs.IMS_FULL)
else:
self.set_message('header-last-modified', rs.IMS_UNKNOWN)
else:
self.set_message('header-last-modified',
rs.IMS_STATUS,
ims_status=self.state.res_status,
enc_ims_status=e(self.state.res_status
or '(unknown)'))
def start_output(self):
if self.kw.get('is_saved', None):
extra_title = " <span class='save'>saved results for...</span>"
else:
extra_title = ""
if self.kw.get('is_blank', None):
extra_body_class = "blank"
else:
extra_body_class = ""
if self.kw.get('descend', False):
descend = "&descend=True"
else:
descend = ''
self.output(
html_header.__doc__ % {
'static':
static_root,
'version':
droid.__version__,
'html_uri':
e(self.uri),
'js_uri':
e_js(self.uri),
'config':
urllib.quote(
json.dumps({
'redbot_uri': self.uri,
'redbot_req_hdrs': self.req_hdrs,
'redbot_version': droid.__version__
})),
'js_req_hdrs':
", ".join([
'["%s", "%s"]' % (e_js(n), e_js(v))
for n, v in self.req_hdrs
]),
'extra_js':
self.format_extra('.js'),
'test_id':
self.kw.get('test_id', ""),
'extra_title':
extra_title,
'extra_body_class':
extra_body_class,
'descend':
descend
})
def parse(subject, value, red):
try:
disposition, params = value.split(";", 1)
except ValueError:
disposition, params = value, ''
disposition = disposition.lower()
param_dict = rh.parse_params(red, subject, params)
if disposition not in ['inline', 'attachment']:
red.set_message(subject,
rs.DISPOSITION_UNKNOWN,
disposition=e(disposition))
if not param_dict.has_key('filename'):
red.set_message(subject, rs.DISPOSITION_OMITS_FILENAME)
if "%" in param_dict.get('filename', ''):
red.set_message(subject, rs.DISPOSITION_FILENAME_PERCENT)
if "/" in param_dict.get('filename', '') or \
r"\\" in param_dict.get('filename*', ''):
red.set_message(subject, rs.DISPOSITION_FILENAME_PATH_CHAR)
return disposition, param_dict
def __init__(self, test_uri):
"""
Constractor
@param test_uri: Test Uri
"""
self.test_uri = test_uri
try:
self.red = InspectingResourceExpertDroid(self.test_uri)
self.result = ""
self.done = False
self.groups = []
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
if self.red.res_complete:
self.result = self._generate_output_xml(test_uri).toprettyxml()
else:
error_string = ""
if self.red.res_error['desc'] == nbhttp.error.ERR_CONNECT[
'desc']:
error_string = "Could not connect to the server (%s)" % self.red.res_error.get(
'detail', "unknown")
elif self.red.res_error['desc'] == nbhttp.error.ERR_URL[
'desc']:
error_string = self.red.res_error.get(
'detail', "RED can't fetch that URL.")
elif self.red.res_error[
'desc'] == nbhttp.error.ERR_READ_TIMEOUT['desc']:
error_string = self.red.res_error['desc']
elif self.red.res_error[
'desc'] == nbhttp.error.ERR_HTTP_VERSION['desc']:
error_string = "<code>%s</code> isn't HTTP." % e(
self.red.res_error.get('detail', '')[:20])
else:
raise AssertionError, "Unidentified incomplete response error."
self.result = self._generate_error_xml(
error_string).toprettyxml()
except:
import traceback
logging.error(traceback.format_exc())
self.result = """<?xml version="1.0" ?>
def done(self):
# see if it was compressed when not negotiated
no_conneg_vary_headers = self.state.parsed_hdrs.get('vary', [])
if 'gzip' in self.state.parsed_hdrs.get('content-encoding', []) or \
'x-gzip' in self.state.parsed_hdrs.get('content-encoding', []):
self.set_message('header-vary header-content-encoding',
rs.CONNEG_GZIP_WITHOUT_ASKING)
else: # Apparently, content negotiation is happening.
# check status
if self.base.res_status != self.state.res_status:
self.set_message('status',
rs.VARY_STATUS_MISMATCH,
neg_status=self.base.res_status,
noneg_status=self.state.res_status)
return # Can't be sure what's going on...
# check headers that should be invariant
for hdr in ['content-type']:
if self.base.parsed_hdrs.get(hdr) != \
self.state.parsed_hdrs.get(hdr, None):
self.set_message('header-%s' % hdr,
rs.VARY_HEADER_MISMATCH,
header=hdr)
# TODO: expose on-the-wire values.
# check Vary headers
vary_headers = self.base.parsed_hdrs.get('vary', [])
if (not "accept-encoding" in vary_headers) and \
(not "*" in vary_headers):
self.set_message('header-vary', rs.CONNEG_NO_VARY)
if no_conneg_vary_headers != vary_headers:
self.set_message('header-vary',
rs.VARY_INCONSISTENT,
conneg_vary=e(", ".join(vary_headers)),
no_conneg_vary=e(
", ".join(no_conneg_vary_headers)))
# check body
if self.base.res_body_post_md5 != self.state.res_body_md5:
self.set_message('body', rs.VARY_BODY_MISMATCH)
return # Can't be sure what's going on...
# check ETag
if self.state.parsed_hdrs.get('etag', 1) \
== self.base.parsed_hdrs.get('etag', 2):
self.set_message('header-etag', rs.VARY_ETAG_DOESNT_CHANGE)
# TODO: weakness?
# check compression efficiency
if self.state.res_body_len > 0:
savings = int(100 *
(
(float(self.state.res_body_len) - \
self.base.res_body_len
) / self.state.res_body_len
)
)
else:
savings = 0
self.base.gzip_support = True
self.base.gzip_savings = savings
if savings >= 0:
self.set_message('header-content-encoding',
rs.CONNEG_GZIP_GOOD,
savings=savings,
orig_size=f_num(self.state.res_body_len),
gzip_size=f_num(self.base.res_body_len))
else:
self.set_message('header-content-encoding',
rs.CONNEG_GZIP_BAD,
savings=abs(savings),
orig_size=f_num(self.state.res_body_len),
gzip_size=f_num(self.base.res_body_len))
def format_droid(self, red):
out = [u'<tr class="droid %s">']
m = 50
if red.parsed_hdrs.get('content-type', [""])[0][:6] == 'image/':
cl = " class='preview'"
else:
cl = ""
if len(red.uri) > m:
out.append(
u"""<td class="uri"><a href="%s" title="%s"%s>%s<span class="fade1">%s</span><span class="fade2">%s</span><span class="fade3">%s</span></a></td>"""
% (
u"?uri=%s" % e_query_arg(red.uri),
e(red.uri),
cl,
e(red.uri[:m - 2]),
e(red.uri[m - 2]),
e(red.uri[m - 1]),
e(red.uri[m]),
))
else:
out.append(
u'<td class="uri"><a href="%s" title="%s"%s>%s</a></td>' %
(u"?uri=%s" % e(red.uri), e(red.uri), cl, e(red.uri)))
if red.res_complete:
if red.res_status in ['301', '302', '303', '307'] and \
red.parsed_hdrs.has_key('location'):
out.append(u'<td><a href="?descend=True&uri=%s">%s</a></td>' %
(urljoin(red.uri, red.parsed_hdrs['location']),
red.res_status))
elif red.res_status in ['400', '404', '410']:
out.append(u'<td class="bad">%s</td>' % red.res_status)
else:
out.append(u'<td>%s</td>' % red.res_status)
# pconn
out.append(self.format_yes_no(red.store_shared))
out.append(self.format_yes_no(red.store_private))
out.append(self.format_time(red.age))
out.append(self.format_time(red.freshness_lifetime))
out.append(self.format_yes_no(red.stale_serveable))
out.append(self.format_yes_no(red.ims_support))
out.append(self.format_yes_no(red.inm_support))
if red.gzip_support:
out.append(u"<td>%s%%</td>" % red.gzip_savings)
else:
out.append(self.format_yes_no(red.gzip_support))
out.append(self.format_yes_no(red.partial_support))
problems = [m for m in red.messages if \
m.level in [rs.l.WARN, rs.l.BAD]]
# TODO: problems += sum([m[2].messages for m in red.messages if m[2] != None], [])
out.append(u"<td>")
pr_enum = []
for problem in problems:
if problem not in self.problems:
self.problems.append(problem)
pr_enum.append(self.problems.index(problem))
# add the problem number to the <tr> so we can highlight
out[0] = out[0] % u" ".join(["%d" % p for p in pr_enum])
# append the actual problem numbers to the final <td>
for p in pr_enum:
m = self.problems[p]
out.append(
"<span class='prob_num'> %s <span class='hidden'>%s</span></span>"
% (p + 1, e(m.summary[self.lang] % m.vars)))
else:
if red.res_error == None:
err = "response incomplete"
else:
err = red.res_error.desc or 'unknown problem'
out.append('<td colspan="11">%s' % err)
out.append(u"</td>")
out.append(u'</tr>')
return nl.join(out)
def parse(subject, value, red):
value = value.lower()
if value not in ['bytes', 'none']:
red.set_message(subject, rs.UNKNOWN_RANGE, range=e(value))
return value
def parse_params(red, subject, instr, nostar=None, delim=";"):
"""
Parse parameters into a dictionary.
@param red: the red instance to use
@param subject: the subject identifier
@param instr: string to be parsed
@param nostar: list of parameters that definitely don't get a star. If
True, no parameter can be starred.
@param delim: delimter between params, default ";"
@return: dictionary of {name: value}
"""
param_dict = {}
instr = instr.encode('ascii')
for param in split_string(instr, syntax.PARAMETER, r"\s*%s\s*" % delim):
try:
k, v = param.split("=", 1)
except ValueError:
param_dict[param.lower()] = None
continue
k_norm = k.lower() # TODO: warn on upper-case in param?
if param_dict.has_key(k_norm):
red.set_message(subject, rs.PARAM_REPEATS, param=e(k_norm))
if v[0] == v[-1] == "'":
red.set_message(subject,
rs.PARAM_SINGLE_QUOTED,
param=e(k_norm),
param_val=e(v),
param_val_unquoted=e(v[1:-1])
)
if k[-1] == '*':
if nostar is True or (nostar and k_norm[:-1] in nostar):
red.set_message(subject, rs.PARAM_STAR_BAD,
param=e(k_norm[:-1]))
else:
if v[0] == '"' and v[-1] == '"':
red.set_message(subject, rs.PARAM_STAR_QUOTED,
param=e(k_norm))
v = unquote_string(v)
try:
enc, lang, esc_v = v.split("'", 3)
except ValueError:
red.set_message(subject, rs.PARAM_STAR_ERROR,
param=e(k_norm))
continue
enc = enc.lower()
lang = lang.lower()
if enc == '':
red.set_message(subject,
rs.PARAM_STAR_NOCHARSET, param=e(k_norm))
continue
elif enc not in ['utf-8']:
red.set_message(subject,
rs.PARAM_STAR_CHARSET,
param=e(k_norm),
enc=e(enc)
)
continue
# TODO: catch unquoting errors, range of chars, charset
unq_v = urllib.unquote(esc_v)
dec_v = unq_v.decode(enc) # ok, because we limit enc above
param_dict[k_norm] = dec_v
else:
param_dict[k_norm] = unquote_string(v)
return param_dict
def join(subject, values, red):
via_list = u"<ul>" + u"\n".join(
[u"<li><code>%s</code></li>" % e(v) for v in values]) + u"</ul>"
red.set_message(subject, rs.VIA_PRESENT, via_list=via_list)
if uri_path.count("/") < 2:
cookie_path = "/"
else:
cookie_path = uri_path[:uri_path.rindex("/")]
else:
cookie_path = attribute_value
cookie_attribute_list.append(("Path", cookie_path))
elif case_norm_attribute_name == "secure":
cookie_attribute_list.append(("Secure", ""))
elif case_norm_attribute_name == "httponly":
cookie_attribute_list.append(("HttpOnly", ""))
else:
red.set_message(subject,
rs.SET_COOKIE_UNKNOWN_ATTRIBUTE,
cookie_name=cookie_name,
attribute=e(attribute_name))
return (cookie_name, cookie_value, cookie_attribute_list)
DELIMITER = r'(?:[\x09\x20-\x2F\x3B-\x40\x5B-\x60\x7B-\x7E])'
NON_DELIMTER = r'(?:[\x00-\x08\x0A-\x1F0-0\:a-zA-Z\x7F-\xFF])'
MONTHS = {
'jan': 1,
'feb': 2,
'mar': 3,
'apr': 4,
'may': 5,
'jun': 6,
'jul': 7,
'aug': 8,
'sep': 9,