def Parse(self, data, isfinal=False):
# The 'data' argument should be an encoded text: a str instance that
# represents an array of bytes. If instead it is a unicode string,
# only the us-ascii range is considered safe enough to be silently
# converted.
if isinstance(data, unicode):
data = data.encode(sys.getdefaultencoding())
self._data.append(data)
if isfinal:
bytes = StringUtil.toBytes(self._data.toString())
byte_stream = ByteArrayInputStream(bytes)
source = InputSource(byte_stream)
if self.encoding is not None:
source.setEncoding(self.encoding)
try:
self._reader.parse(source)
except SAXParseException, sax_error:
# Experiments tend to show that the '_Next*' parser locations
# match more closely expat behavior than the 'Current*' or sax
# error locations.
self.ErrorLineNumber = self._NextLineNumber
self.ErrorColumnNumber = self._NextColumnNumber
self.ErrorCode = None
raise self._expat_error(sax_error)
return 1
def doActiveScan(self, ihrr, isip):
withPayload = isip.buildRequest(StringUtil.toBytes('${1336+1}'))
newReqRes = self.callbacks.makeHttpRequest(ihrr.getHttpService(),
withPayload)
variation = self.helpers.analyzeResponseVariations(
ihrr.getResponse(), newReqRes.getResponse())
pageChanges = variation.getVariantAttributes()
length = False
bodyContent = False
match = False
for change in pageChanges:
if change == 'content_length':
length = True
if change == 'whole_body_content':
bodyContent = True
match = '1337' in self.helpers.bytesToString(newReqRes.getResponse())
if length and bodyContent and match:
issues = ArrayList()
issues.add(ELJ(newReqRes, self.callbacks, self.helpers))
return issues
else:
return None
def Parse(self, data, isfinal=False):
# The 'data' argument should be an encoded text: a str instance that
# represents an array of bytes. If instead it is a unicode string,
# only the us-ascii range is considered safe enough to be silently
# converted.
if isinstance(data, unicode):
data = data.encode(sys.getdefaultencoding())
self._data.append(data)
if isfinal:
bytes = StringUtil.toBytes(self._data.toString())
byte_stream = ByteArrayInputStream(bytes)
source = InputSource(byte_stream)
if self.encoding is not None:
source.setEncoding(self.encoding)
try:
self._reader.parse(source)
except SAXParseException, sax_error:
# Experiments tend to show that the '_Next*' parser locations
# match more closely expat behavior than the 'Current*' or sax
# error locations.
self.ErrorLineNumber = self._NextLineNumber
self.ErrorColumnNumber = self._NextColumnNumber
self.ErrorCode = None
raise self._expat_error(sax_error)
return 1
def send_to_repeater(self, host, payload):
req = self._requests[host]['POST'] or self._requests[host][
'PUT'] or self._requests[host]['GET']
if req and self._callbacks and self._helpers:
info = req[0]
body = req[1]
nobody = body[:info.getBodyOffset()].tostring()
rstripoffset = info.getBodyOffset() - len(nobody.rstrip())
headers = body[:info.getBodyOffset() - rstripoffset].tostring()
try:
self._overrideheaders[host]
except KeyError:
self._overrideheaders[host] = []
headers = override_headers(headers, self._overrideheaders[host])
repeater_body = StringUtil.toBytes(
string_join(
headers,
body[info.getBodyOffset() -
rstripoffset:info.getBodyOffset()].tostring(),
payload))
self._callbacks.sendToRepeater(
info.getUrl().getHost(),
info.getUrl().getPort(),
info.getUrl().getProtocol() == 'https', repeater_body,
'GraphQL #%s' % self._index)
self._index += 1
def doActiveScan(self, ihrr, isip):
# 1 - Create a new request with our custom payload (tip: buildRequest)
withPayload = isip.buildRequest(StringUtil.toBytes('${1336+1}'))
# 2 - Send the HTTP request
newReqRes = self.callbacks.makeHttpRequest(ihrr.getHttpService(), withPayload)
# 3 - Diff original and new responses (tip: analyzeResponseVariations and getVariantAttributes)
variation = self.helpers.analyzeResponseVariations(ihrr.getResponse(), newReqRes.getResponse())
pageChanges = variation.getVariantAttributes()
# 4 - Based on page changes, determine whether the page is vulnerable or not
length = False
bodyContent = False
match = False
for change in pageChanges:
if change == 'content_length':
length = True
if change == 'whole_body_content':
bodyContent = True
match = '1337' in self.helpers.bytesToString(newReqRes.getResponse())
# 5 - If vulnerable, create a new IScanIssue and return the List<IScanIssue>
if length and bodyContent and match:
pass
# TODO
else:
return None
def actionPerformed(self, e):
"""
Overrides ActionListener behaviour. Send current query to repeater.
:param e: unused
:return: None
"""
req = self.requests[self._host]['POST'] or self.requests[
self._host]['PUT'] or self.requests[self._host]['GET']
if req:
info = req[0]
body = req[1]
nobody = body[:info.getBodyOffset()].tostring()
rstripoffset = info.getBodyOffset() - len(nobody.rstrip())
headers = body[:info.getBodyOffset() - rstripoffset].tostring()
try:
self._overrideheaders[self._host]
except KeyError:
self._overrideheaders[self._host] = []
repeater_body = StringUtil.toBytes(
string_join(
override_headers(headers,
self._overrideheaders[self._host]),
body[info.getBodyOffset() -
rstripoffset:info.getBodyOffset()].tostring(),
self._payload))
self._callbacks.sendToRepeater(
info.getUrl().getHost(),
info.getUrl().getPort(),
info.getUrl().getProtocol() == 'https', repeater_body,
'GraphQL #%s' % self._index)
self._index += 1
def load_image(name, data):
var_name = "_{}_img".format(name)
if _USE_EMBEDDED_IMAGES:
globals()[var_name] = ImageIcon(
StringUtil.toBytes(base64.b64decode(data)))
else:
globals()[var_name] = ImageIcon(
file=os.path.join(os.path.dirname(__file__), name + ".gif"))
def write(fd, string):
"""write(fd, string) -> byteswritten
Write a string to a file descriptor.
"""
from java.nio import ByteBuffer
from org.python.core.util import StringUtil
rawio = FileDescriptors.get(fd)
return _handle_oserror(rawio.write,
ByteBuffer.wrap(StringUtil.toBytes(string)))
def test_bmp_utf8stream(self):
# Program written in Unicode with codes above 255
a = u"畫蛇添足 Λόγος"
prog = u'a = u"{:s}"\nprint repr(a)'.format(a)
# Program as bytes with declared encoding for execfile(InputStream)
progbytes = '# coding: utf-8\n' + prog.encode('utf-8')
stream = java.io.ByteArrayInputStream(StringUtil.toBytes(progbytes))
self.do_test(stream,
u'{}\n'.format(repr(a)),
{'a': a})
def write(fd, string):
"""write(fd, string) -> byteswritten
Write a string to a file descriptor.
"""
from java.nio import ByteBuffer
from org.python.core.util import StringUtil
rawio = FileDescriptors.get(fd)
return _handle_oserror(rawio.write,
ByteBuffer.wrap(StringUtil.toBytes(string)))
def process(self, inputStream, outputStream):
try:
# Read input FlowFile content
input_text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
input_obj = json.loads(input_text)
# Transform content
# for i in range(0, len(input_obj)):
output_arr = []
count = 0
while (count <= 59):
detester = input_obj['create_time']
date = (datetime.datetime.strptime(detester, '%Y-%m-%d %H:%M:%S.%f') + datetime.timedelta(
minutes=count))
output_obj = {
"id": input_obj['id'],
"meas_type": input_obj['meas_type'],
"create_time": date.strftime("%Y-%m-%d %H:%M:%S.%f"),
"create_day": datetime.datetime.strptime(date.strftime("%Y-%m-%d"), '%Y-%m-%d').strftime(
"%Y-%m-%d %H:%M:%S.%f"),
"create_minute": date.strftime("%H:%M:%S.%f")
}
if count == 0:
output_obj['vtype'] = '时间'
elif count == 1:
output_obj['vtype'] = '最大值'
elif count == 2:
output_obj['vtype'] = '最小值'
else:
output_obj['vtype'] = '平均值'
if count < 10:
objkey = 'v0' + str(count)
if input_obj[objkey] is None:
output_obj['val'] = 0
else:
output_obj['val'] = input_obj[objkey]
else:
objkey = 'v' + str(count)
if input_obj[objkey] is None:
output_obj['val'] = 0
else:
output_obj['val'] = input_obj[objkey]
# Write output content
output_arr.append(output_obj)
count = count + 1
output_text = json.dumps(output_arr)
outputStream.write(StringUtil.toBytes(output_text))
except:
traceback.print_exc(file=sys.stdout)
raise
def do_test(self, source, ref_out=u'', ref_var={}, inp=None):
out = java.io.StringWriter()
err = java.io.StringWriter()
var = {}
if inp is not None:
if isinstance(inp, bytes):
inp = java.io.ByteArrayInputStream(StringUtil.toBytes(inp))
elif isinstance(inp, unicode):
inp = java.io.StringReader(inp)
exec_code_in_pi(source, inp, out, err, var)
self.assertEquals(ref_var, var)
self.assertEquals(ref_out, out.toString())
def __post(self):
if (self.ready):
while self.qbuff.qsize():
msg = self.qbuff.get()
#print("[->] " + msg)
try:
self.postpv.write(StringUtil.toBytes(msg))
except:
pass
sleep(0.250)
else:
while self.qbuff.qsize():
discard = self.qbuff.get()
def buildRequest(self, payload):
# Gross workaround via Dafydd - https://support.portswigger.net/customer/portal/questions/12431820-design-of-active-scanner-plugin-vs-insertionpoints
if payload.tostring() not in methods:
raise Exception(
'Just stopping Burp from using our custom insertion point')
else:
requestStr = self._baseRequest.tostring()
newRequest = requestStr.replace(self._baseValue, payload)
newRequestB = StringUtil.toBytes(newRequest)
# update the request with the new parameter value
return newRequestB
def runMessage(self, messageIndex):
messageEntry = self._db.arrayOfMessages[messageIndex]
# Clear Previous Results:
messageEntry._roleResults = {}
messageEntry._userRuns = {}
messageInfo = messageEntry._requestResponse
requestInfo = self._helpers.analyzeRequest(messageInfo)
reqBody = messageInfo.getRequest()[requestInfo.getBodyOffset():]
for userIndex in self._db.getActiveUserIndexes():
userEntry = self._db.arrayOfUsers[userIndex]
headers = self.getNewHeader(requestInfo, userEntry)
# Add static CSRF token if available
# TODO: Kinda hacky, but for now it will add the token as long as there is some content in the post body
# Even if its a GET request. This screws up when original requests have no body though... oh well...
newBody = reqBody
if userEntry._staticcsrf and len(reqBody):
delimeter = userEntry._staticcsrf.find("=")
if delimeter >= 0:
csrfname = userEntry._staticcsrf[0:delimeter]
csrfvalue = userEntry._staticcsrf[delimeter+1:]
params = requestInfo.getParameters()
for param in params:
if str(param.getName())==csrfname:
# Handle CSRF Tokens in Body
if param.getType() == 1:
newBody = reqBody[0:param.getValueStart()-requestInfo.getBodyOffset()] + StringUtil.toBytes(csrfvalue) + reqBody[param.getValueEnd()-requestInfo.getBodyOffset():]
# Handle CSRF Tokens in Cookies (for Cookie==Body mitigation technique):
if param.getType() == 2:
# TODO: required moving above portion to a function
# TODO: also need to think about when cookie name != postarg name
print "Cookie CSRF Tokens are not currently supported"
if newBody == reqBody:
newBody = reqBody+StringUtil.toBytes("&"+userEntry._staticcsrf)
# Construct and send a message with the new headers
message = self._helpers.buildHttpMessage(headers, newBody)
requestResponse = self._callbacks.makeHttpRequest(messageInfo.getHttpService(),message)
messageEntry.addRunByUserIndex(userIndex, self._callbacks.saveBuffersToTempFiles(requestResponse))
# Grab all active roleIndexes that should succeed
activeSuccessRoles = [index for index in messageEntry._roles.keys() if messageEntry._roles[index] and not self._db.arrayOfRoles[index].isDeleted()]
# Check Role Results of message
for roleIndex in self._db.getActiveRoleIndexes():
success = self.checkResult(messageEntry, roleIndex, activeSuccessRoles)
messageEntry.setRoleResultByRoleIndex(roleIndex, success)
def process(self, inputStream, outputStream):
try:
# Read input FlowFile content
input_text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
input_obj = json.loads(input_text)
# Transform content
output_obj = input_obj
output_obj['value'] = output_obj['value'] * output_obj['value']
output_obj['message'] = 'Hello World'
# Write output content
output_text = json.dumps(output_obj)
outputStream.write(StringUtil.toBytes(output_text))
except:
traceback.print_exc(file=sys.stdout)
raise
def process(self, inputStream, outputStream):
try:
# Read input FlowFile content
input_text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
input_obj = json.loads(input_text)
# Transform content
output_obj = input_obj
output_obj['value'] = output_obj['value'] * output_obj['value']
output_obj['message'] = 'Hello World'
# Write output content
output_text = json.dumps(output_obj)
outputStream.write(StringUtil.toBytes(output_text))
except:
traceback.print_exc(file=sys.stdout)
raise
def process(self, inputStream, outputStream):
try:
# Read input FlowFile content
input_text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
input_obj = json.loads(input_text)
# Transform content
output_arr = []
start_time = input_obj['start_time']
end_time = input_obj['end_time']
id = input_obj['id']
meas_type = input_obj['meas_type']
date = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
endtime = datetime.datetime.strptime(end_time, '%Y-%m-%d %H:%M:%S')
while date < endtime:
output_obj = {
"id": id,
"meas_type": meas_type,
"val": 0,
"create_time": date.strftime("%Y-%m-%d %H:%M:%S"),
"create_day": datetime.datetime.strptime(date.strftime("%Y-%m-%d"), '%Y-%m-%d').strftime(
"%Y-%m-%d %H:%M:%S"),
"create_minute": date.strftime("%H:%M:%S")
}
if date.strftime("%M") == "00":
output_obj['vtype'] = '时间'
elif date.strftime("%M") == "01":
output_obj['vtype'] = '最大值'
elif date.strftime("%M") == "02":
output_obj['vtype'] = '最小值'
else:
output_obj['vtype'] = '平均值'
date = date + datetime.timedelta(minutes=1)
print(output_obj)
# Write output content
output_arr.append(output_obj)
output_text = json.dumps(output_arr)
outputStream.write(StringUtil.toBytes(output_text))
except:
traceback.print_exc(file=sys.stdout)
raise
def process(self, inputStream, outputStream):
try:
# Read input FlowFile content
input_text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
input_obj = json.loads(input_text)
# Transform content
output_obj = input_obj
#output_obj['value'] = output_obj['value'] * output_obj['value']
#output_obj['message'] = 'Hello'
precision_level = 8
output_obj['geohash'] = output_obj.apply(
lambda x: pgh.encode(x.lat, x.lon, precision=precision_level),
axis=1) # >>> ‘ezs42’
# Write output content
output_text = json.dumps(output_obj)
outputStream.write(StringUtil.toBytes(output_text))
except:
traceback.print_exc(file=sys.stdout)
raise
def send_to_repeater_get_query(self, host, payload):
req = self._requests[host]['POST'] or self._requests[host][
'PUT'] or self._requests[host]['GET']
if req and self._callbacks and self._helpers:
info = req[0]
body = req[1]
nobody = body[:info.getBodyOffset()].tostring()
rstripoffset = info.getBodyOffset() - len(nobody.rstrip())
metadata = body[:info.getBodyOffset() - rstripoffset].tostring()
try:
self._overrideheaders[host]
except KeyError:
self._overrideheaders[host] = []
metadata = override_headers(metadata, self._overrideheaders[host])
# remove Content-Type on GET requests
metadata = re.sub(r'(?m)^Content-Type:.*\n?', '', metadata)
content = json.loads(payload)
if isinstance(content, list):
content = content[0]
metadata = override_uri(metadata,
method="GET",
query=urlencode(
querify(clean_dict(content))))
repeater_body = StringUtil.toBytes(
string_join(
metadata,
body[info.getBodyOffset() -
rstripoffset:info.getBodyOffset()].tostring()))
self._callbacks.sendToRepeater(
info.getUrl().getHost(),
info.getUrl().getPort(),
info.getUrl().getProtocol() == 'https', repeater_body,
'GraphQL - GET query #%s' % self._index)
self._index += 1
def send_to_repeater_post_form_data_body(self, host, payload):
req = self._requests[host]['POST'] or self._requests[host][
'PUT'] or self._requests[host]['GET']
if req and self._callbacks and self._helpers:
info = req[0]
body = req[1]
nobody = body[:info.getBodyOffset()].tostring()
rstripoffset = info.getBodyOffset() - len(nobody.rstrip())
headers = body[:info.getBodyOffset() - rstripoffset].tostring()
try:
self._overrideheaders[host]
except KeyError:
self._overrideheaders[host] = []
headers = override_headers(headers, self._overrideheaders[host])
boundary = "---------------------------%s" % random_string()
headers = override_headers(headers, [
("Content-Type", "multipart/form-data, boundary=%s" % boundary)
])
headers = override_uri(headers, method="POST")
content = json.loads(payload)
if isinstance(content, list):
content = content[0]
repeater_body = StringUtil.toBytes(
string_join(
headers,
body[info.getBodyOffset() -
rstripoffset:info.getBodyOffset()].tostring(),
multipart(data=querify(clean_dict(content)),
boundary=boundary)))
self._callbacks.sendToRepeater(
info.getUrl().getHost(),
info.getUrl().getPort(),
info.getUrl().getProtocol() == 'https', repeater_body,
'GraphQL - POST form-data #%s' % self._index)
self._index += 1
def send_to_repeater_post_urlencoded_body(self, host, payload):
req = self._requests[host]['POST'] or self._requests[host][
'PUT'] or self._requests[host]['GET']
if req and self._callbacks and self._helpers:
info = req[0]
body = req[1]
nobody = body[:info.getBodyOffset()].tostring()
rstripoffset = info.getBodyOffset() - len(nobody.rstrip())
headers = body[:info.getBodyOffset() - rstripoffset].tostring()
try:
self._overrideheaders[host]
except KeyError:
self._overrideheaders[host] = []
headers = override_headers(headers, self._overrideheaders[host])
headers = override_headers(
headers,
[("Content-Type", "application/x-www-form-urlencoded")])
headers = override_uri(headers, method="POST")
content = json.loads(payload)
if isinstance(content, list):
content = content[0]
repeater_body = StringUtil.toBytes(
string_join(
headers,
body[info.getBodyOffset() -
rstripoffset:info.getBodyOffset()].tostring(),
urlencode(querify(clean_dict(content)))))
self._callbacks.sendToRepeater(
info.getUrl().getHost(),
info.getUrl().getPort(),
info.getUrl().getProtocol() == 'https', repeater_body,
'GraphQL - POST urlencoded #%s' % self._index)
self._index += 1
def dialogaction(action,var):
cutil.printVar("test mail",action,var)
if action == "sendmail" :
res = cmail.sendMail("hello","Sending from Jython","*****@*****.**","test.jython")
print res
assert res == None
var["OK"] = True
if action == "getmailno" :
no = cmail.getMailNo()
print no
var["OK"] = True
if action == "getlist" :
li = cmail.getMailList()
for l in li :
print l.getHeader()
print l.getSentDate()
var["OK"] = True
if action == "sendmailattachment" :
s = "I'm your attachment sent from Jython code.\n What do you think about it ?\n Do you like me ?"
key = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY,"TESTJ",StringUtil.toBytes(s))
print key
res = cmail.sendMailSingleAttach("Attachment from Jython","Sending from Jython","*****@*****.**","test.jython",cutil.PDFTEMPORARY,key,"my.txt")
print res
assert res == None
var["OK"] = True
if action == "sendmailattachment3" :
s = "First attachment"
key1 = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY,"TESTJ",StringUtil.toBytes(s))
aList = cmail.createAttachList(None,cutil.PDFTEMPORARY,key1,"attach1.txt")
s = "Second attachment"
key2 = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY,"TESTJ",StringUtil.toBytes(s))
aList = cmail.createAttachList(aList,cutil.PDFTEMPORARY,key2,"attach2.txt")
s = "Third attachment"
key3 = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY,"TESTJ",StringUtil.toBytes(s))
aList = cmail.createAttachList(aList,cutil.PDFTEMPORARY,key3,"attach3.txt")
res = cmail.sendMail("3 attachments from Jython","Sending from Jython","*****@*****.**","test.jython",aList)
print res
assert res == None
var["OK"] = True
if action == "createbloblist" :
B.removeBlob(var)
for a in range(10) :
s = "Attachment no " + str(a)
key = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY,"TESTCREATE",StringUtil.toBytes(s))
B.addBlob(var,"Hello",key)
var["OK"] = True
if action == "sendbloblist" :
B.readBlobList(var)
li = var["JLIST_MAP"]["blist"]
print li
realM = B.getRealm(var)
aList = None
no = 0
for l in li :
key = l["blob_key"]
aList = cmail.createAttachList(aList,realM,key,"attach" + str(no) +".txt")
no = no + 1
res = cmail.sendMail("A lot of attachments BLOB","Sending from Jython","*****@*****.**","test.jython",aList)
print res
assert res == None
var["OK"] = True
if action == "setxmail" :
M = cmail.CMAIL(var)
res = M.sendMail("hello","Sending from CMAIL class","*****@*****.**","test.jython")
print res
assert res != None
resmail = res.getSendResult()
print resmail
assert resmail == None
var["OK"] = True
if action == "readxmail" :
M = cmail.CMAIL(var)
li = M.getList()
print li
for l in li : print l.getDescription()
assert len(li) == 1
var["OK"] = True
if action == "removexmail" :
M = cmail.CMAIL(var)
li = M.getList()
assert len(li) == 1
li = M.getList()
l = li[0]
M.deleteElem(l)
li = M.getList()
assert len(li) == 0
var["OK"] = True
def b64d(self, input):
if (input != None):
return self.helpers.base64Decode(input)
return StringUtil.toBytes('')
def process(self, inputStream, outputStream):
try:
# Read input FlowFile content
input_text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
input_obj = json.loads(input_text)
# Transform content
# for i in range(0, len(input_obj)):
output_arr = []
# detester = '2018-04-12 00:00:00.0'
detester = input_obj['create_time']
#starttime = detester
#endtime = datetime.datetime.strptime(detester, '%Y-%m-%d %H:%M:%S.%f') + datetime.timedelta(days=1)
# '2018-04-09 00:00:00.0'
# detesterend = '2018-04-12 00:00:00.0'
# ids \
id = input_obj['id']
# ['130199010600000501', '130199010600000502']
# input_obj['id']
# meas_types
meas_type = input_obj['meas_type']
# ['1001', '3101']
# input_obj['meas_type']
# for id in ids:
# for meas_type in meas_types:
# for k in range(0, 38):
for i in range(0, 24):
for j in range(0, 60):
date = datetime.datetime.strptime(
detester, '%Y-%m-%d %H:%M:%S.%f') + datetime.timedelta(
hours=i) + datetime.timedelta(minutes=j)
output_obj = {
"id":
id,
"meas_type":
meas_type,
"val":
0,
"create_time":
date.strftime("%Y-%m-%d %H:%M:%S.%f"),
"create_day":
datetime.datetime.strptime(
date.strftime("%Y-%m-%d"),
'%Y-%m-%d').strftime("%Y-%m-%d %H:%M:%S.%f"),
"create_minute":
date.strftime("%H:%M:%S.%f")
}
if j == 0:
output_obj['vtype'] = '时间'
elif j == 1:
output_obj['vtype'] = '最大值'
elif j == 2:
output_obj['vtype'] = '最小值'
else:
output_obj['vtype'] = '平均值'
# Write output content
output_arr.append(output_obj)
output_text = json.dumps(output_arr)
outputStream.write(StringUtil.toBytes(output_text))
except:
traceback.print_exc(file=sys.stdout)
raise
def process(self, inputStream, outputStream):
text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
listCustNoOlist = extractCustNoList(text, 'customer_unique_id')
result = cleanText(str(listCustNoOlist))
outputStream.write(StringUtil.toBytes(result))
def dialogaction(action, var):
cutil.printVar("test mail", action, var)
if action == "sendmail":
res = cmail.sendMail("hello", "Sending from Jython",
"*****@*****.**", "test.jython")
print res
assert res == None
var["OK"] = True
if action == "getmailno":
no = cmail.getMailNo()
print no
var["OK"] = True
if action == "getlist":
li = cmail.getMailList()
for l in li:
print l.getHeader()
print l.getSentDate()
var["OK"] = True
if action == "sendmailattachment":
s = "I'm your attachment sent from Jython code.\n What do you think about it ?\n Do you like me ?"
key = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY, "TESTJ",
StringUtil.toBytes(s))
print key
res = cmail.sendMailSingleAttach("Attachment from Jython",
"Sending from Jython",
"*****@*****.**",
"test.jython", cutil.PDFTEMPORARY,
key, "my.txt")
print res
assert res == None
var["OK"] = True
if action == "sendmailattachment3":
s = "First attachment"
key1 = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY, "TESTJ",
StringUtil.toBytes(s))
aList = cmail.createAttachList(None, cutil.PDFTEMPORARY, key1,
"attach1.txt")
s = "Second attachment"
key2 = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY, "TESTJ",
StringUtil.toBytes(s))
aList = cmail.createAttachList(aList, cutil.PDFTEMPORARY, key2,
"attach2.txt")
s = "Third attachment"
key3 = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY, "TESTJ",
StringUtil.toBytes(s))
aList = cmail.createAttachList(aList, cutil.PDFTEMPORARY, key3,
"attach3.txt")
res = cmail.sendMail("3 attachments from Jython",
"Sending from Jython",
"*****@*****.**", "test.jython",
aList)
print res
assert res == None
var["OK"] = True
if action == "createbloblist":
B.removeBlob(var)
for a in range(10):
s = "Attachment no " + str(a)
key = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY, "TESTCREATE",
StringUtil.toBytes(s))
B.addBlob(var, "Hello", key)
var["OK"] = True
if action == "sendbloblist":
B.readBlobList(var)
li = var["JLIST_MAP"]["blist"]
print li
realM = B.getRealm(var)
aList = None
no = 0
for l in li:
key = l["blob_key"]
aList = cmail.createAttachList(aList, realM, key,
"attach" + str(no) + ".txt")
no = no + 1
res = cmail.sendMail("A lot of attachments BLOB",
"Sending from Jython",
"*****@*****.**", "test.jython",
aList)
print res
assert res == None
var["OK"] = True
if action == "setxmail":
M = cmail.CMAIL(var)
res = M.sendMail("hello", "Sending from CMAIL class",
"*****@*****.**", "test.jython")
print res
assert res != None
resmail = res.getSendResult()
print resmail
assert resmail == None
var["OK"] = True
if action == "readxmail":
M = cmail.CMAIL(var)
li = M.getList()
print li
for l in li:
print l.getDescription()
assert len(li) == 1
var["OK"] = True
if action == "removexmail":
M = cmail.CMAIL(var)
li = M.getList()
assert len(li) == 1
li = M.getList()
l = li[0]
M.deleteElem(l)
li = M.getList()
assert len(li) == 0
var["OK"] = True
def runMessage(self, messageIndex):
messageEntry = self._db.arrayOfMessages[messageIndex]
# Clear Previous Results:
messageEntry._roleResults = {}
messageEntry._userRuns = {}
messageInfo = messageEntry._requestResponse
requestInfo = self._helpers.analyzeRequest(messageInfo)
reqBody = messageInfo.getRequest()[requestInfo.getBodyOffset():]
for userIndex in self._db.getActiveUserIndexes():
userEntry = self._db.arrayOfUsers[userIndex]
headers = requestInfo.getHeaders()
if userEntry.isCookie():
cookieHeader = "Cookie:"
newheader = cookieHeader
# getHeaders has to be called again here cuz of java references
for header in requestInfo.getHeaders():
if str(header).startswith(cookieHeader):
newheader = str(header).strip()
headers.remove(header)
# If its a valid cookie
equals = userEntry._token.find("=")
if equals >= 0:
cookieIndex = newheader.find(userEntry._token[0:equals +
1])
# Cookie already exists and must be removed
if cookieIndex >= 0:
semicolon = newheader.find(";", cookieIndex)
if semicolon >= 0:
# Remove extra whitespace
if newheader[semicolon + 1:semicolon + 2] == " ":
newheader = newheader.replace(
newheader[cookieIndex:semicolon + 2], "")
else:
newheader = newheader.replace(
newheader[cookieIndex:semicolon + 1], "")
else:
newheader = newheader.replace(
newheader[cookieIndex:], "")
# Removing tailing semicolon and white space
newheader = newheader.rstrip("; ")
# Handle when the only cookie is the target cookie
if not newheader == cookieHeader:
newheader += ";"
newheader += " " + userEntry._token
# Header
else:
newheader = userEntry._token
colon = newheader.find(":")
if colon >= 0:
# getHeaders has to be called again here cuz of java references
for header in requestInfo.getHeaders():
if str(header).startswith(newheader[0:colon + 1]):
headers.remove(header)
headers.add(newheader)
# Add static CSRF token if available
# TODO: Kinda hacky, but for now it will add the token as long as there is some content in the post body
# Even if its a GET request. This screws up when original requests have no body though... oh well...
newBody = reqBody
if userEntry._staticcsrf and len(reqBody):
delimeter = userEntry._staticcsrf.find("=")
if delimeter >= 0:
csrfname = userEntry._staticcsrf[0:delimeter]
csrfvalue = userEntry._staticcsrf[delimeter + 1:]
params = requestInfo.getParameters()
for param in params:
if str(param.getName()) == csrfname:
# Handle CSRF Tokens in Body
if param.getType() == 1:
newBody = reqBody[0:param.getValueStart(
) - requestInfo.getBodyOffset(
)] + StringUtil.toBytes(csrfvalue) + reqBody[
param.getValueEnd() -
requestInfo.getBodyOffset():]
# Handle CSRF Tokens in Cookies (for Cookie==Body mitigation technique):
if param.getType() == 2:
# TODO: required moving above portion to a function
# TODO: also need to think about when cookie name != postarg name
print "Cookie CSRF Tokens are not currently supported"
if newBody == reqBody:
newBody = reqBody + StringUtil.toBytes(
"&" + userEntry._staticcsrf)
# Construct and send a message with the new headers
message = self._helpers.buildHttpMessage(headers, newBody)
requestResponse = self._callbacks.makeHttpRequest(
messageInfo.getHttpService(), message)
messageEntry.addRunByUserIndex(
userIndex,
self._callbacks.saveBuffersToTempFiles(requestResponse))
# Grab all active roleIndexes that should succeed
activeSuccessRoles = [
index for index in messageEntry._roles.keys()
if messageEntry._roles[index]
and not self._db.arrayOfRoles[index].isDeleted()
]
# Check Role Results of message
for roleIndex in self._db.getActiveRoleIndexes():
success = self.checkResult(messageEntry, roleIndex,
activeSuccessRoles)
messageEntry.setRoleResultByRoleIndex(roleIndex, success)
def runMessage(self, messageIndex):
messageEntry = self._db.arrayOfMessages[messageIndex]
# Clear Previous Results:
messageEntry._roleResults = {}
messageEntry._userRuns = {}
messageInfo = messageEntry._requestResponse
requestInfo = self._helpers.analyzeRequest(messageInfo)
reqBody = messageInfo.getRequest()[requestInfo.getBodyOffset():]
for userIndex in self._db.getActiveUserIndexes():
userEntry = self._db.arrayOfUsers[userIndex]
headers = requestInfo.getHeaders()
if userEntry.isCookie():
cookieHeader = "Cookie:"
newheader = cookieHeader
# getHeaders has to be called again here cuz of java references
for header in requestInfo.getHeaders():
if str(header).startswith(cookieHeader):
newheader = str(header).strip()
headers.remove(header)
# If its a valid cookie
equals = userEntry._token.find("=")
if equals >= 0:
cookieIndex = newheader.find(userEntry._token[0:equals+1])
# Cookie already exists and must be removed
if cookieIndex >= 0:
semicolon = newheader.find(";",cookieIndex)
if semicolon >=0:
# Remove extra whitespace
if newheader[semicolon+1:semicolon+2] == " ":
newheader = newheader.replace(newheader[cookieIndex:semicolon+2],"")
else:
newheader = newheader.replace(newheader[cookieIndex:semicolon+1],"")
else:
newheader = newheader.replace(newheader[cookieIndex:],"")
# Removing tailing semicolon and white space
newheader = newheader.rstrip("; ")
# Handle when the only cookie is the target cookie
if not newheader == cookieHeader:
newheader += ";"
newheader += " "+userEntry._token
# Header
else:
newheader = userEntry._token
colon = newheader.find(":")
if colon >= 0:
# getHeaders has to be called again here cuz of java references
for header in requestInfo.getHeaders():
if str(header).startswith(newheader[0:colon+1]):
headers.remove(header)
headers.add(newheader)
# Add static CSRF token if available
# TODO: Kinda hacky, but for now it will add the token as long as there is some content in the post body
# Even if its a GET request. This screws up when original requests have no body though... oh well...
newBody = reqBody
if userEntry._staticcsrf and len(reqBody):
delimeter = userEntry._staticcsrf.find("=")
if delimeter >= 0:
csrfname = userEntry._staticcsrf[0:delimeter]
csrfvalue = userEntry._staticcsrf[delimeter+1:]
params = requestInfo.getParameters()
for param in params:
if str(param.getName())==csrfname:
# Handle CSRF Tokens in Body
if param.getType() == 1:
newBody = reqBody[0:param.getValueStart()-requestInfo.getBodyOffset()] + StringUtil.toBytes(csrfvalue) + reqBody[param.getValueEnd()-requestInfo.getBodyOffset():]
# Handle CSRF Tokens in Cookies (for Cookie==Body mitigation technique):
if param.getType() == 2:
# TODO: required moving above portion to a function
# TODO: also need to think about when cookie name != postarg name
print "Cookie CSRF Tokens are not currently supported"
if newBody == reqBody:
newBody = reqBody+StringUtil.toBytes("&"+userEntry._staticcsrf)
# Construct and send a message with the new headers
message = self._helpers.buildHttpMessage(headers, newBody)
requestResponse = self._callbacks.makeHttpRequest(messageInfo.getHttpService(),message)
messageEntry.addRunByUserIndex(userIndex, self._callbacks.saveBuffersToTempFiles(requestResponse))
# Grab all active roleIndexes that should succeed
activeSuccessRoles = [index for index in messageEntry._roles.keys() if messageEntry._roles[index] and not self._db.arrayOfRoles[index].isDeleted()]
# Check Role Results of message
for roleIndex in self._db.getActiveRoleIndexes():
success = self.checkResult(messageEntry, roleIndex, activeSuccessRoles)
messageEntry.setRoleResultByRoleIndex(roleIndex, success)
# Create connection (doesn't have to be a ssl one, despite the name)
factory = SSLConnection(broker).createConnection()
connection = factory.newConnection()
channel = connection.createChannel()
# Get token from consumer
tokenGetter = CreateToken(broker, channel, queueName)
token = tokenGetter.getToken()
# Attach token to message
message = token + ":"
messages = messages+[message]
for arg in command_args[2:]:
messages = messages + [str(arg)]
parsedMessage = ParseMessage(messages)
sendMessage = parsedMessage.getMessage()
channel.queueDeclare(queueName, False, False, True, None)
channel.basicPublish("", queueName, None, StringUtil.toBytes(str(sendMessage)))
print (" [x] Sent " + sendMessage )
channel.close()
connection.close()
def write(self, val):
# print >> sys.stderr, val
self.editor.text = self.editor.text + StringUtil.toBytes(val)
# Find name of consumer queue
uuidQueue = UUIDManager(QUEUE_FILE)
queueName = uuidQueue.checkUUID()
# Create connection (doesn't have to be a ssl one, despite the name)
factory = SSLConnection(broker).createConnection()
connection = factory.newConnection()
channel = connection.createChannel()
# Get token from consumer
tokenGetter = CreateToken(broker, channel, queueName)
token = tokenGetter.getToken()
# Attach token to message
message = token + ":"
messages = messages + [message]
for arg in command_args[2:]:
messages = messages + [str(arg)]
parsedMessage = ParseMessage(messages)
sendMessage = parsedMessage.getMessage()
channel.queueDeclare(queueName, False, False, True, None)
channel.basicPublish("", queueName, None, StringUtil.toBytes(str(sendMessage)))
print(" [x] Sent " + sendMessage)
channel.close()
connection.close()
def process(self, outputStream):
try:
outputStream.write(StringUtil.toBytes(self.content_text))
except:
traceback.print_exc(file=sys.stdout)
raise
def dialogaction(action, var):
cutil.printVar("testmail", action, var)
if action == "sendmail":
C = cmail.CMAIL(var)
res = C.sendMail("Mail from hotel", "What do you think about us ?",
"*****@*****.**", "Jython")
print res.getSendResult()
assert res.getSendResult() == None
assert res.getName() != None
var["OK"] = True
if action == "checkmail":
C = cmail.CMAIL(var)
li = C.getList()
print li
for l in li:
print l.getDescription(), l.getContent()
assert len(li) == 1
var["OK"] = True
if action == "sendmailattach":
C = cmail.CMAIL(var)
li = C.getList()
for l in li:
C.removeElem(l)
li = C.getList()
assert len(li) == 0
s = "Attachment"
key = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY, "TESTJ",
StringUtil.toBytes(s))
aList = cmail.createAttachList(None, cutil.PDFTEMPORARY, key,
"attach.txt")
res = C.sendMail("Mail from hotel with attachment",
"Nothing interesting ",
"*****@*****.**", "Jython", aList)
print res.getSendResult()
assert res.getSendResult() == None
var["OK"] = True
if action == "checkmailattach":
B = SHolder.getBlobHandler()
C = cmail.CMAIL(var)
li = C.getList()
assert len(li) == 1
l = li[0]
att = l.getaList()
assert len(att) == 1
for a in att:
print a.getFileName(), a.getRealm(), a.getBlobKey()
va = B.findBlob(a.getRealm(), a.getBlobKey())
assert va != None
ss = StringUtil.fromBytes(va)
print ss
assert ss == "Attachment"
var["OK"] = True
if action == "sendmultiattach":
C = cmail.CMAIL(var)
li = C.getList()
for l in li:
C.removeElem(l)
li = C.getList()
assert len(li) == 0
aList = None
for i in range(20):
s = "Attachment content" + str(i)
key = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY, "TESTJ",
StringUtil.toBytes(s))
aList = cmail.createAttachList(aList, cutil.PDFTEMPORARY, key,
"attach" + str(i) + ".txt")
res = C.sendMail("Mail from hotel with 20 attachment",
"Nothing interesting inside ",
"*****@*****.**", "Jython", aList)
print res.getSendResult()
assert res.getSendResult() == None
var["OK"] = True
if action == "checkmultiattach":
B = SHolder.getBlobHandler()
C = cmail.CMAIL(var)
li = C.getList()
assert len(li) == 1
l = li[0]
att = l.getaList()
assert len(att) == 20
for a in att:
print a.getFileName(), a.getRealm(), a.getBlobKey()
va = B.findBlob(a.getRealm(), a.getBlobKey())
assert va != None
ss = StringUtil.fromBytes(va)
print ss
assert not cutil.emptyS(ss)
var["OK"] = True
def replaceDomain(requestData, oldDomain, newDomain):
reqstr = StringUtil.fromBytes(requestData)
reqstr = reqstr.replace(oldDomain, newDomain)
newreq = StringUtil.toBytes(reqstr)
return newreq
def dialogaction(action,var):
cutil.printVar("testmail",action,var)
if action == "sendmail" :
C = cmail.CMAIL(var)
res = C.sendMail("Mail from hotel","What do you think about us ?","*****@*****.**","Jython")
print res.getSendResult()
assert res.getSendResult() == None
assert res.getName() != None
var["OK"] = True
if action == "checkmail":
C = cmail.CMAIL(var)
li = C.getList()
print li
for l in li :
print l.getDescription(),l.getContent()
assert len(li) == 1
var["OK"] = True
if action == "sendmailattach" :
C = cmail.CMAIL(var)
li = C.getList()
for l in li :
C.removeElem(l)
li = C.getList()
assert len(li) == 0
s = "Attachment"
key = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY,"TESTJ",StringUtil.toBytes(s))
aList = cmail.createAttachList(None,cutil.PDFTEMPORARY,key,"attach.txt")
res = C.sendMail("Mail from hotel with attachment","Nothing interesting ","*****@*****.**","Jython",aList)
print res.getSendResult()
assert res.getSendResult() == None
var["OK"] = True
if action == "checkmailattach" :
B = SHolder.getBlobHandler()
C = cmail.CMAIL(var)
li = C.getList()
assert len(li) == 1
l = li[0]
att = l.getaList()
assert len(att) == 1
for a in att :
print a.getFileName(),a.getRealm(),a.getBlobKey()
va = B.findBlob(a.getRealm(),a.getBlobKey())
assert va != None
ss = StringUtil.fromBytes(va)
print ss
assert ss == "Attachment"
var["OK"] = True
if action == "sendmultiattach" :
C = cmail.CMAIL(var)
li = C.getList()
for l in li :
C.removeElem(l)
li = C.getList()
assert len(li) == 0
aList = None
for i in range(20) :
s = "Attachment content" + str(i)
key = ADDBLOB.addNewBlob(cutil.PDFTEMPORARY,"TESTJ",StringUtil.toBytes(s))
aList = cmail.createAttachList(aList,cutil.PDFTEMPORARY,key,"attach" + str(i) + ".txt")
res = C.sendMail("Mail from hotel with 20 attachment","Nothing interesting inside ","*****@*****.**","Jython",aList)
print res.getSendResult()
assert res.getSendResult() == None
var["OK"] = True
if action == "checkmultiattach" :
B = SHolder.getBlobHandler()
C = cmail.CMAIL(var)
li = C.getList()
assert len(li) == 1
l = li[0]
att = l.getaList()
assert len(att) == 20
for a in att :
print a.getFileName(),a.getRealm(),a.getBlobKey()
va = B.findBlob(a.getRealm(),a.getBlobKey())
assert va != None
ss = StringUtil.fromBytes(va)
print ss
assert not cutil.emptyS(ss)
var["OK"] = True
def write(filePath, vars, script):
content = (vars + Strings.FileUtils_separator + script)
Files.write(filePath, StringUtil.toBytes(content))
def run(self):
"""
Where all the work is done.
Three steps
1) Make the request again to make sure it's stable (We use python difflib)
2) Make the request without any cookies, if it's similar the cookies aren't required
3) Loop through removing one cookie at a time, if the diff isn't big we leave it out, if
it is we can figure that it's required and we leave it in
At the end we're left with the required cookies.
"""
selected_message = self.selected_message
original_request_bytes = selected_message.getRequest()
original_request = StringUtil.fromBytes(original_request_bytes)
original_response_bytes = selected_message.getResponse()
original_response = StringUtil.fromBytes(original_response_bytes)
http_service = selected_message.getHttpService()
helpers = self.callbacks.getHelpers()
request_info = helpers.analyzeRequest(http_service, original_request_bytes)
parameters = request_info.getParameters();
cookie_parameters = [parameter for parameter in parameters if parameter.getType() == IParameter.PARAM_COOKIE]
num_requests_needed = len(cookie_parameters) + 2
num_requests_made = 0
self.progressBar.setMaximum(num_requests_needed)
self.progressBar.setValue(num_requests_made)
#===========================================================
# We will resubmit the request to make sure it is somewhat stable
self.statusTextArea.append("Making baseline request which we will compare to original request (1 of 1)")
self.statusTextArea.append("\n")
baseline_reqres_pair = self.callbacks.makeHttpRequest(http_service, original_request)
threshold = httpresponse_bytes_diffratio(helpers, original_response_bytes, baseline_reqres_pair.getResponse())
if (threshold < 0.6):
self.statusTextArea.append("ERROR: Not a stable HTTP request, try another where it's nearly the same every request")
self.statusTextArea.append("\n")
self.progressBar.setValue(num_requests_needed)
return
# Pad it a little by decreasing the threshold...
threshold = threshold * 0.8 #TODO: We should automate discovering the right threshold
self.statusTextArea.append("...Complete, threshold ratio is %s" % (threshold))
self.statusTextArea.append("\n")
#===========================================================
if (cancelThread is True):
print "Canceled"
return
num_requests_made = num_requests_made + 1
self.progressBar.setValue(num_requests_made)
#===========================================================
# Now we'll check if it actually requires authentication by removing all cookies
nocookies_request_bytes = StringUtil.toBytes(original_request)
for parameter in cookie_parameters:
nocookies_request_bytes = helpers.removeParameter(nocookies_request_bytes, parameter)
self.statusTextArea.append("Making no-cookie request to make sure it requires them (1 of 1)")
self.statusTextArea.append("\n")
nocookie_reqres_pair = self.callbacks.makeHttpRequest(http_service, nocookies_request_bytes)
nocookiediff = httpresponse_bytes_diffratio(helpers, original_response_bytes, nocookie_reqres_pair.getResponse())
if (nocookiediff > threshold):
self.statusTextArea.append("ERROR: Cookies don't seem to be required or it's too close to tell")
self.statusTextArea.append("\n")
self.progressBar.setValue(num_requests_needed)
return
self.statusTextArea.append("...Complete, confirmed at least one cookie is required")
self.statusTextArea.append("\n")
#===========================================================
if (cancelThread is True):
print "Canceled"
return
num_requests_made = num_requests_made + 1
self.progressBar.setValue(num_requests_made)
#===========================================================
# Now iterate over all the cookie values, removing one at a time until left with required ones
self.statusTextArea.append("Making requests, removing each cookie one at a time. (%d requests)" % (len(cookie_parameters)))
self.statusTextArea.append("\n")
minimumcookies_request_bytes = StringUtil.toBytes(original_request)
for parameter in cookie_parameters:
if (cancelThread is True):
print "Canceled"
return
missingcookie_request_bytes = helpers.removeParameter(minimumcookies_request_bytes, parameter)
missingcookie_reqres_pair = self.callbacks.makeHttpRequest(http_service, missingcookie_request_bytes)
missingcookiediff = httpresponse_bytes_diffratio(helpers, original_response_bytes, missingcookie_reqres_pair.getResponse())
if (missingcookiediff > threshold):
self.statusTextArea.append(" Cookie '%s' is not required (%s similarity ratio)" % (parameter.getName(), missingcookiediff))
self.statusTextArea.append("\n")
minimumcookies_request_bytes = missingcookie_request_bytes
else:
self.statusTextArea.append("* Cookie '%s' is required (%s similarity ratio)" % (parameter.getName(), missingcookiediff))
self.statusTextArea.append("\n")
num_requests_made = num_requests_made + 1
self.progressBar.setValue(num_requests_made)
#===========================================================
#===========================================================
# minimumcookies_request_bytes now contains a request with the minimum number
# of cookies needed to maintain the session.
#===========================================================
# Display the results
self.statusTextArea.append("== Required Cookies ==")
self.statusTextArea.append("\n")
mininumcookies_request_info = helpers.analyzeRequest(http_service, minimumcookies_request_bytes)
minimum_cookie_parameters = [parameter for parameter in mininumcookies_request_info.getParameters() if parameter.getType() == IParameter.PARAM_COOKIE]
for cookie_parameter in minimum_cookie_parameters:
self.statusTextArea.append(cookie_parameter.getName())
self.statusTextArea.append("\n")
#===========================================================
return
# setup namespaces
ElementTree._namespace_map["http://www.loc.gov/MARC21/slim"] = "marc"
ElementTree._namespace_map["http://www.w3.org/2001/XMLSchema-instance"] = "xsi"
# parse the marc payload and reserialize with proper namespaces
props = object.getMetadata()
try:
#sourcePayload = object.getPayload(object.getSourceId())
sourcePayload = object.getPayload("DS1")
tree = ElementTree.ElementTree()
elem = tree.parse(FileUtil.wrap(sourcePayload.open()))
xml = ElementTree.tostring(elem)
sourcePayload.close()
# save to new payload
xmlBytes = ByteArrayInputStream(StringUtil.toBytes(xml))
try:
newPayloadId = "MARC"
props.setProperty("modified", "true");
#Can be processing multiple payload...
props.setProperty("payloadToDelete.1", "DS1")
props.setProperty("payloadToExport.1", "MARC")
fixedPayload = object.createStoredPayload(newPayloadId, xmlBytes)
except:
print "Fail to create new payload, '%s' is already exist" % newPayloadId
#revert back the properties as this object might be modified before but not exported yet
restoreProperty(props, "indexOnHarvest")
restoreProperty(props, "harvestQueue")
restoreProperty(props, "renderQueue")
restoreProperty(props, "jythonScript")
def run(self):
"""
Where all the work is done.
Three steps
1) Make the request again to make sure it's stable (We use python difflib)
2) Make the request without any cookies, if it's similar the cookies aren't required
3) Loop through removing one cookie at a time, if the diff isn't big we leave it out, if
it is we can figure that it's required and we leave it in
At the end we're left with the required cookies.
"""
selected_message = self.selected_message
original_request_bytes = selected_message.getRequest()
original_request = StringUtil.fromBytes(original_request_bytes)
original_response_bytes = selected_message.getResponse()
original_response = StringUtil.fromBytes(original_response_bytes)
http_service = selected_message.getHttpService()
helpers = self.callbacks.getHelpers()
request_info = helpers.analyzeRequest(http_service,
original_request_bytes)
parameters = request_info.getParameters()
cookie_parameters = [
parameter for parameter in parameters
if parameter.getType() == IParameter.PARAM_COOKIE
]
num_requests_needed = len(cookie_parameters) + 2
num_requests_made = 0
self.progressBar.setMaximum(num_requests_needed)
self.progressBar.setValue(num_requests_made)
#===========================================================
# We will resubmit the request to make sure it is somewhat stable
self.statusTextArea.append(
"Making baseline request which we will compare to original request (1 of 1)"
)
self.statusTextArea.append("\n")
baseline_reqres_pair = self.callbacks.makeHttpRequest(
http_service, original_request)
threshold = httpresponse_bytes_diffratio(
helpers, original_response_bytes,
baseline_reqres_pair.getResponse())
if (threshold < 0.6):
self.statusTextArea.append(
"ERROR: Not a stable HTTP request, try another where it's nearly the same every request"
)
self.statusTextArea.append("\n")
self.progressBar.setValue(num_requests_needed)
return
# Pad it a little by decreasing the threshold...
threshold = threshold * 0.8 #TODO: We should automate discovering the right threshold
self.statusTextArea.append("...Complete, threshold ratio is %s" %
(threshold))
self.statusTextArea.append("\n")
#===========================================================
if (cancelThread is True):
print "Canceled"
return
num_requests_made = num_requests_made + 1
self.progressBar.setValue(num_requests_made)
#===========================================================
# Now we'll check if it actually requires authentication by removing all cookies
nocookies_request_bytes = StringUtil.toBytes(original_request)
for parameter in cookie_parameters:
nocookies_request_bytes = helpers.removeParameter(
nocookies_request_bytes, parameter)
self.statusTextArea.append(
"Making no-cookie request to make sure it requires them (1 of 1)")
self.statusTextArea.append("\n")
nocookie_reqres_pair = self.callbacks.makeHttpRequest(
http_service, nocookies_request_bytes)
nocookiediff = httpresponse_bytes_diffratio(
helpers, original_response_bytes,
nocookie_reqres_pair.getResponse())
if (nocookiediff > threshold):
self.statusTextArea.append(
"ERROR: Cookies don't seem to be required or it's too close to tell"
)
self.statusTextArea.append("\n")
self.progressBar.setValue(num_requests_needed)
return
self.statusTextArea.append(
"...Complete, confirmed at least one cookie is required")
self.statusTextArea.append("\n")
#===========================================================
if (cancelThread is True):
print "Canceled"
return
num_requests_made = num_requests_made + 1
self.progressBar.setValue(num_requests_made)
#===========================================================
# Now iterate over all the cookie values, removing one at a time until left with required ones
self.statusTextArea.append(
"Making requests, removing each cookie one at a time. (%d requests)"
% (len(cookie_parameters)))
self.statusTextArea.append("\n")
minimumcookies_request_bytes = StringUtil.toBytes(original_request)
for parameter in cookie_parameters:
if (cancelThread is True):
print "Canceled"
return
missingcookie_request_bytes = helpers.removeParameter(
minimumcookies_request_bytes, parameter)
missingcookie_reqres_pair = self.callbacks.makeHttpRequest(
http_service, missingcookie_request_bytes)
missingcookiediff = httpresponse_bytes_diffratio(
helpers, original_response_bytes,
missingcookie_reqres_pair.getResponse())
if (missingcookiediff > threshold):
self.statusTextArea.append(
" Cookie '%s' is not required (%s similarity ratio)" %
(parameter.getName(), missingcookiediff))
self.statusTextArea.append("\n")
minimumcookies_request_bytes = missingcookie_request_bytes
else:
self.statusTextArea.append(
"* Cookie '%s' is required (%s similarity ratio)" %
(parameter.getName(), missingcookiediff))
self.statusTextArea.append("\n")
num_requests_made = num_requests_made + 1
self.progressBar.setValue(num_requests_made)
#===========================================================
#===========================================================
# minimumcookies_request_bytes now contains a request with the minimum number
# of cookies needed to maintain the session.
#===========================================================
# Display the results
self.statusTextArea.append("== Required Cookies ==")
self.statusTextArea.append("\n")
mininumcookies_request_info = helpers.analyzeRequest(
http_service, minimumcookies_request_bytes)
minimum_cookie_parameters = [
parameter
for parameter in mininumcookies_request_info.getParameters()
if parameter.getType() == IParameter.PARAM_COOKIE
]
for cookie_parameter in minimum_cookie_parameters:
self.statusTextArea.append(cookie_parameter.getName())
self.statusTextArea.append("\n")
#===========================================================
return
def process(self, inputStream, outputStream):
text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
outputList = convertJsonMapping(text)
outputText = json.dumps(outputList, indent=1)
outputStream.write(StringUtil.toBytes(outputText))
