def aFixityTest(self):
self.log("Create a binary")
headers = {
'Content-type': 'image/jpeg',
}
with open(os.path.join(os.getcwd(), 'resources', 'basic_image.jpg'), 'rb') as fp:
data = fp.read()
r = self.do_post(self.getBaseUri(), headers=headers, body=data)
self.assertEqual(201, r.status_code, 'Did not create binary')
location = self.get_location(r)
fixity_endpoint = location + "/" + TestConstants.FCR_FIXITY
self.log("Get a fixity result")
headers = {
'Accept': TestConstants.JSONLD_MIMETYPE
}
r = self.do_get(fixity_endpoint, headers=headers)
self.assertEqual(200, r.status_code, "Can't get the fixity result")
body = r.content.decode('UTF-8').rstrip('\ny')
json_body = json.loads(body)
fixity_id = pyjq.first('.[0]."http://www.loc.gov/premis/rdf/v1#hasFixity"| .[]?."@id"', json_body)
fixity_result = pyjq.first('.[] | select(."@id" == "{0}") | '
'."http://www.loc.gov/premis/rdf/v1#hasMessageDigest" | .[]?."@id"'.format(fixity_id),
json_body)
self.assertEqual(self.FIXITY_RESULT, fixity_result, "Fixity result was not a match for expected.")
self.log("Passed")
def get(self, irc, msg, args, selector, data):
"""<selector> <json data>
Uses the selector against the json data and returns the selected
value.
See http://stedolan.github.io/jq/tutorial/ to learn about the
selector syntax."""
irc.reply(pyjq.first(selector, json.loads(data)))
def get_transaction_provider(self):
headers = {
'Accept': TestConstants.JSONLD_MIMETYPE
}
r = self.do_get(self.getFedoraBase(), headers=headers)
self.assertEqual(200, r.status_code, "Did not get expected response")
body = r.content.decode('UTF-8')
json_body = json.loads(body)
tx_provider = pyjq.first('.[]."http://fedora.info/definitions/v4/repository#hasTransactionProvider" | .[]."@id"',
json_body)
return tx_provider
def _pyjq(self, w_query, w_json = None):
"""
Runs a jq query against a json structure
"""
if w_json == None:
q_json = self.last_response
else:
q_json = w_json
try:
r = pyjq.first(w_query, q_json)
except:
raise
#
return r
def validate_project_response(content):
return pyjq.first('.data.createProject.project', content)
def validate_user_story_name_select(content):
return pyjq.first('.data.userStoryByName', content)
def validate_threat_model_query(content):
return pyjq.first('.data.createOrUpdateThreatModel.threatModel', content)
def validate_abuser_story(content):
return pyjq.first('.data.createOrUpdateAbuserStory.abuserStory', content)
def get_site_contact_type_id():
response = requests.get(GWS_URL + '/contactTypes?code=SiteContact')
response.raise_for_status()
return pyjq.first('._embedded.contactTypes[0].id', json.loads(response.text))
def step_impl(context):
assert context.ushard_group != None
assert context.mysql_groups != None
assert len(context.mysql_groups) >= 2
configs = api_get(context, context.version_3 + "/ushard_deploy/get_config",
{
"group_id": context.ushard_group["group_id"],
})
id = generate_id()
user1 = {
"name": "user1_" + id,
"password": "******" + id,
"schema": "schema_" + id
}
id = generate_id()
user2 = {
"name": "user2_" + id,
"password": "******" + id,
"schema": "schema_" + id
}
context.ushard_users = {"user1": user1, "user2": user2}
rule_id = "rule_" + generate_id()
mysql_group_1 = get_mysql_group_brief(context,
context.mysql_groups[0]["group_id"])
mysql_group_2 = get_mysql_group_brief(context,
context.mysql_groups[1]["group_id"])
# prepare database in MySQL
api_get(
context, context.version_3 + "/helper/query_mysql", {
"mysql_id": mysql_group_1["master_id"],
"user": "******",
"password": mysql_group_1["root_password"],
"query": "create database {0}_db1".format(user1["schema"])
})
api_get(
context, context.version_3 + "/helper/query_mysql", {
"mysql_id": mysql_group_1["master_id"],
"user": "******",
"password": mysql_group_1["root_password"],
"query": "create database {0}_db2".format(user1["schema"])
})
api_get(
context, context.version_3 + "/helper/query_mysql", {
"mysql_id": mysql_group_2["master_id"],
"user": "******",
"password": mysql_group_2["root_password"],
"query": "create database {0}_db1".format(user2["schema"])
})
api_get(
context, context.version_3 + "/helper/query_mysql", {
"mysql_id": mysql_group_2["master_id"],
"user": "******",
"password": mysql_group_2["root_password"],
"query": "create database {0}_db2".format(user2["schema"])
})
# update dble
## server.xml
serverXmlStr = pyjq.first('.[] | select(.name == "server.xml") | .value',
configs)
assert serverXmlStr is not None
serverXml = read_xml(serverXmlStr)
for user in [user1, user2]:
serverXml.getroot().append(
etree.fromstring("""
<user name="{0}">
<property name="password">{1}</property>
<property name="readOnly">false</property>
<property name="schemas">{2}</property>
</user>""".format(user["name"], user["password"], user["schema"])))
## schema.xml
schemaXmlStr = pyjq.first('.[] | select(.name == "schema.xml") | .value',
configs)
assert schemaXmlStr is not None
schemaXml = read_xml(schemaXmlStr)
newSections = """
<newSections>
<schema name="{1}" sqlMaxLimit="100">
<table name="test" type="default" primaryKey="id" needAddLimit="true" rule="{0}" dataNode="{1}_dn1,{1}_dn2"></table>
</schema>
<schema name="{6}" sqlMaxLimit="100">
<table name="test" type="default" primaryKey="id" needAddLimit="true" rule="{0}" dataNode="{6}_dn3,{6}_dn4"></table>
</schema>
<dataNode name="{1}_dn1" dataHost="{1}_dh1" database="{1}_db1"></dataNode>
<dataNode name="{1}_dn2" dataHost="{1}_dh1" database="{1}_db2"></dataNode>
<dataNode name="{6}_dn3" dataHost="{6}_dh2" database="{6}_db1"></dataNode>
<dataNode name="{6}_dn4" dataHost="{6}_dh2" database="{6}_db2"></dataNode>
<dataHost name="{1}_dh1" maxCon="100" minCon="10" balance="1" switchType="-1" slaveThreshold="-1">
<heartbeat>show slave status</heartbeat>
<writeHost host="{1}_M" url="{2}" user="******" password="******">
<readHost host="{1}_S1" url="{3}" user="******" password="******"></readHost>
</writeHost>
</dataHost>
<dataHost name="{6}_dh2" maxCon="100" minCon="10" balance="1" switchType="-1" slaveThreshold="-1">
<heartbeat>show slave status</heartbeat>
<writeHost host="{6}_M" url="{7}" user="******" password="******">
<readHost host="{6}_S1" url="{8}" user="******" password="******"></readHost>
</writeHost>
</dataHost>
</newSections>""".format(
rule_id, user1["schema"], mysql_group_1["master_addr"],
mysql_group_1["slave_addr"], "root", mysql_group_1["root_password"],
user2["schema"], mysql_group_2["master_addr"],
mysql_group_2["slave_addr"], "root", mysql_group_2["root_password"])
for item in etree.fromstring(newSections):
schemaXml.getroot().append(item)
## rule.xml
ruleXmlStr = pyjq.first('.[] | select(.name == "rule.xml") | .value',
configs)
assert ruleXmlStr is not None
ruleXml = read_xml(ruleXmlStr)
ruleXml.getroot().insert(
0,
etree.fromstring("""
<tableRule name="{0}">
<rule>
<columns>id</columns>
<algorithm>{0}</algorithm>
</rule>
</tableRule>""".format(rule_id)))
ruleXml.getroot().append(
etree.fromstring("""
<function name="{0}" class="Hash">
<property name="partitionCount">2</property>
<property name="partitionLength">1</property>
</function>""".format(rule_id)))
## update config
cacheServiceXmlStr = pyjq.first(
'.[] | select(.name == "cacheservice.properties") | .value', configs)
assert cacheServiceXmlStr is not None
ehcacheXmlStr = pyjq.first('.[] | select(.name == "ehcache.xml") | .value',
configs)
assert ehcacheXmlStr is not None
config = [{
"name": "rule.xml",
"value": xml_to_str(ruleXml),
}, {
"name": "schema.xml",
"value": xml_to_str(schemaXml),
}, {
"name": "server.xml",
"value": xml_to_str(serverXml),
}, {
"name": "cacheservice.properties",
"value": cacheServiceXmlStr,
}, {
"name": "ehcache.xml",
"value": ehcacheXmlStr,
}]
api_request_post(
context, context.version_3 + "/ushard_deploy/save_config", {
"is_sync": True,
"group_id": context.ushard_group["group_id"],
"ushard_config": json.dumps(config)
})
def step_imp(context, should_or_not, type):
alert_info = get_all_alert_info(context)
match = pyjq.first(
'.|.receivers[]|.universe_configs[]|select(."type"=="{0}")'.format(
type), alert_info)
assert (match and should_or_not) or (not match and not should_or_not)
def step_impl(context, should_or_not):
resp = get_all_unresolved_alert_record(context)
alert_record = pyjq.first(
'.[]|select(.fingerprint=="{0}")'.format(context.fingerprint), resp)
assert (alert_record and should_or_not) or (not alert_record
and not should_or_not)
def step_imp(context, code):
resp = get_alert_record_in_order(context, 'timestamp')
alert_info = pyjq.first('.[]|select(."code" == {0})'.format(code), resp)
if alert_info is not None:
return True
def test_first(self):
self.assertEqual(
pyjq.first('.[] | . + $foo', ['val1', 'val2'],
vars=dict(foo='bar')), 'val1bar')
def get_conn(config, prefix=None, tojq=None, **kwargs):
'''
Returns a DB Cursors (with pymysql) that supports all of the
hotness
'''
# Given a Config Dictionary with an optional prefix and tojq
# Pull the Data Out and Connect to the Database
logger = logging.getLogger("db_helper.py")
if isinstance(config, dict) is False:
raise TypeError("config is not a dictionary")
dbconfig = None
if tojq is not None and isinstance(tojq, str):
try:
dbconfig = pyjq.first(tojq, config)
except Exception as jq_error:
logger.error("Unable to find config at jq rule : {}".format(tojq))
logger.info("Error : {}".format(jq_error))
raise jq_error
else:
logger.debug("Successfully tojq'ed this configuration.")
else:
dbconfig = config
if isinstance(dbconfig, dict) is False:
logger.error("DBConfig isn't here!")
raise TypeError("Problem reading Database Information")
pymysql_args = {
"host":
dbconfig["{}dbhostname".format(prefix)],
"port":
int(dbconfig["{}dbport".format(prefix)]),
"user":
dbconfig["{}dbusername".format(prefix)],
"password":
dbconfig["{}dbpassword".format(prefix)],
"database":
dbconfig["{}dbdb".format(prefix)],
"autocommit":
dbconfig.get("{}autocommit".format(prefix), kwargs.get("ac_def",
True)),
"charset":
dbconfig.get("{}charset".format(prefix),
kwargs.get("char_def", "utf8mb4"))
}
if dbconfig.get("{}ssl", False) is True:
pymysql_args["ssl"] = dict()
ca = dbconfig.get("{}dbsslca".format(prefix), None)
capath = dbconfig.get("{}dbsslcapath".format(prefix), None)
cert = dbconfig.get("{}dbsslcert".format(prefix), None)
key = dbconfig.get("{}dbsslkey".format(prefix), None)
cipher = dbconfig.get("{}dbsslcipher".format(prefix), None)
if ca is not None:
pymysql_args["ssl"]["ca"] = ca
if capath is not None:
pymysql_args["ssl"]["capath"] = capath
if cert is not None:
pymysql_args["ssl"]["cert"] = cert
if key is not None:
pymysql_args["ssl"]["key"] = key
if cipher is not None:
pymysql_args["ssl"]["cipher"] = cipher
try:
db_conn = pymysql.connect(**pymysql_args)
logger.debug("Connected to {user}@{host}:{port}/{database}".format(
**pymysql_args))
except Exception as connection_error:
logger.warning(
"Connection Failed to {user}@{host}:{port}/{database}".format(
**pymysql_args))
logger.debug("Error {}".format(connection_error))
raise connection_error
return db_conn
#!/usr/bin/env python3
import pyjq
golden_data = {"param_name": "PKG_TAG_NAME", "param_type": None}
data = dict(
parameters=[
dict(name="PKG_TAG_NAME", value="trunk"),
dict(name="GIT_COMMIT", value="master"),
dict(name="TRIGGERED_JOB", value="trunk-buildall"),
],
id="2013-12-27_00-09-37",
changeSet=dict(items=[], kind="git"),
)
d = pyjq.first('.parameters[] | {"param_name": .name, "param_type":.type}',
data)
assert d == golden_data
def get_silence_list_by_rule_name(context, rule_name):
resp = api_get(context, context.version_3 + "/alert/list_silence",
{"rule_name": rule_name})
rule = pyjq.first('.[]', resp)
return rule
def getObject(key,object):
return pyjq.first(key,object)
def test_first():
assert pyjq.first(".[] | . + $foo", ["val1", "val2"],
vars=dict(foo="bar")) == "val1bar"
def public(accounts, config):
for account in accounts:
# Get the data from the `prepare` command
outputfilter = {
'internal_edges': False,
'read_replicas': False,
'inter_rds_edges': False,
'azs': False,
'collapse_by_tag': None,
'collapse_asgs': True,
'mute': True
}
network = build_data_structure(account, config, outputfilter)
# Look at all the edges for ones connected to the public Internet (0.0.0.0/0)
for edge in pyjq.all(
'.[].data|select(.type=="edge")|select(.source=="0.0.0.0/0")',
network):
# Find the node at the other end of this edge
target = {'arn': edge['target'], 'account': account['name']}
target_node = pyjq.first(
'.[].data|select(.id=="{}")'.format(target['arn']), network,
{})
# Depending on the type of node, identify what the IP or hostname is
if target_node['type'] == 'elb':
target['type'] = 'elb'
target['hostname'] = target_node['node_data']['DNSName']
elif target_node['type'] == 'autoscaling':
target['type'] = 'autoscaling'
target['hostname'] = target_node['node_data'].get(
'PublicIpAddress', '')
if target['hostname'] == '':
target['hostname'] = target_node['node_data'][
'PublicDnsName']
elif target_node['type'] == 'rds':
target['type'] = 'rds'
target['hostname'] = target_node['node_data']['Endpoint'][
'Address']
elif target_node['type'] == 'ec2':
target['type'] = 'ec2'
dns_name = target_node['node_data'].get('PublicDnsName', '')
target['hostname'] = target_node['node_data'].get(
'PublicIpAddress', dns_name)
else:
print(
pyjq.first(
'.[].data|select(.id=="{}")|[.type, (.node_data|keys)]'
.format(target['arn']), network, {}))
# Check if any protocol is allowed (indicated by IpProtocol == -1)
ingress = pyjq.all('.[]', edge.get('node_data', {}))
sg_group_allowing_all_protocols = pyjq.first(
'select(.IpPermissions[]|.IpProtocol=="-1")|.GroupId', ingress,
None)
public_sgs = set()
if sg_group_allowing_all_protocols is not None:
log_warning(
'All protocols allowed access to {} due to {}'.format(
target, sg_group_allowing_all_protocols))
range_string = '0-65535'
public_sgs.add(sg_group_allowing_all_protocols)
else:
# from_port and to_port mean the beginning and end of a port range
# We only care about TCP (6) and UDP (17)
# For more info see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/security-group-rules-reference.html
port_ranges = []
for sg in ingress:
for ip_permission in sg['IpPermissions']:
selection = 'select((.IpProtocol=="tcp") or (.IpProtocol=="udp")) | select(.IpRanges[].CidrIp=="0.0.0.0/0")'
port_ranges.extend(
pyjq.all(
'{}| [.FromPort,.ToPort]'.format(selection),
ip_permission))
public_sgs.add(sg['GroupId'])
range_string = port_ranges_string(regroup_ranges(port_ranges))
target['ports'] = range_string
target['public_sgs'] = list(public_sgs)
if target['ports'] == "":
issue_msg = 'No ports open for tcp or udp (probably can only be pinged). Rules that are not tcp or udp: {} -- {}'
log_warning(
issue_msg.format(
json.dumps(
pyjq.all(
'.[]|select((.IpProtocol!="tcp") and (.IpProtocol!="udp"))'
.format(selection), ingress)), account))
print(json.dumps(target, indent=4, sort_keys=True))
account = Account(None, account)
for region_json in get_regions(account):
region = Region(account, region_json)
# Look for CloudFront
if region.name == 'us-east-1':
json_blob = query_aws(region.account,
'cloudfront-list-distributions', region)
for distribution in json_blob.get('DistributionList',
{}).get('Items', []):
if not distribution['Enabled']:
continue
target = {
'arn': distribution['ARN'],
'account': account.name
}
target['type'] = 'cloudfront'
target['hostname'] = distribution['DomainName']
target['ports'] = '80,443'
print(json.dumps(target, indent=4, sort_keys=True))
# Look for API Gateway
json_blob = query_aws(region.account, 'apigateway-get-rest-apis',
region)
if json_blob is not None:
for api in json_blob.get('items', []):
target = {'arn': api['id'], 'account': account.name}
target['type'] = 'apigateway'
target[
'hostname'] = '{}.execute-api.{}.amazonaws.com'.format(
api['id'], region.name)
target['ports'] = '80,443'
print(json.dumps(target, indent=4, sort_keys=True))
def get_public_nodes(account, config, use_cache=False):
# TODO Look for IPv6 also
# TODO Look at more services from https://github.com/arkadiyt/aws_public_ips
# TODO Integrate into something to more easily port scan and screenshot web services
# Try reading from cache
cache_file_path = 'account-data/{}/public_nodes.json'.format(
account['name'])
if use_cache:
if os.path.isfile(cache_file_path):
with open(cache_file_path) as f:
return json.load(f), []
# Get the data from the `prepare` command
outputfilter = {
'internal_edges': False,
'read_replicas': False,
'inter_rds_edges': False,
'azs': False,
'collapse_by_tag': None,
'collapse_asgs': True,
'mute': True
}
network = build_data_structure(account, config, outputfilter)
public_nodes = []
warnings = []
# Look at all the edges for ones connected to the public Internet (0.0.0.0/0)
for edge in pyjq.all(
'.[].data|select(.type=="edge")|select(.source=="0.0.0.0/0")',
network):
# Find the node at the other end of this edge
target = {'arn': edge['target'], 'account': account['name']}
target_node = pyjq.first(
'.[].data|select(.id=="{}")'.format(target['arn']), network, {})
# Depending on the type of node, identify what the IP or hostname is
if target_node['type'] == 'elb':
target['type'] = 'elb'
target['hostname'] = target_node['node_data']['DNSName']
elif target_node['type'] == 'elbv2':
target['type'] = 'elbv2'
target['hostname'] = target_node['node_data']['DNSName']
elif target_node['type'] == 'autoscaling':
target['type'] = 'autoscaling'
target['hostname'] = target_node['node_data'].get(
'PublicIpAddress', '')
if target['hostname'] == '':
target['hostname'] = target_node['node_data']['PublicDnsName']
elif target_node['type'] == 'rds':
target['type'] = 'rds'
target['hostname'] = target_node['node_data']['Endpoint'][
'Address']
elif target_node['type'] == 'ec2':
target['type'] = 'ec2'
dns_name = target_node['node_data'].get('PublicDnsName', '')
target['hostname'] = target_node['node_data'].get(
'PublicIpAddress', dns_name)
elif target_node['type'] == 'ecs':
target['type'] = 'ecs'
target['hostname'] = ''
for ip in target_node['node_data']['ips']:
if is_public_ip(ip):
target['hostname'] = ip
elif target_node['type'] == 'redshift':
target['type'] = 'redshift'
target['hostname'] = target_node['node_data'].get(
'Endpoint', {}).get('Address', '')
else:
# Unknown node
raise Exception('Unknown type: {}'.format(target_node['type']))
# Check if any protocol is allowed (indicated by IpProtocol == -1)
ingress = pyjq.all('.[]', edge.get('node_data', {}))
sg_group_allowing_all_protocols = pyjq.first(
'select(.IpPermissions[]|.IpProtocol=="-1")|.GroupId', ingress,
None)
public_sgs = {}
if sg_group_allowing_all_protocols is not None:
warnings.append(
'All protocols allowed access to {} due to {}'.format(
target, sg_group_allowing_all_protocols))
range_string = '0-65535'
# I would need to redo this code in order to get the name of the security group
public_sgs[sg_group_allowing_all_protocols] = {
'public_ports': '0-65535'
}
else:
# from_port and to_port mean the beginning and end of a port range
# We only care about TCP (6) and UDP (17)
# For more info see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/security-group-rules-reference.html
port_ranges = []
for sg in ingress:
sg_port_ranges = []
for ip_permission in sg['IpPermissions']:
selection = 'select((.IpProtocol=="tcp") or (.IpProtocol=="udp")) | select(.IpRanges[].CidrIp=="0.0.0.0/0")'
sg_port_ranges.extend(
pyjq.all('{}| [.FromPort,.ToPort]'.format(selection),
ip_permission))
public_sgs[sg['GroupId']] = {
'GroupName':
sg['GroupName'],
'public_ports':
port_ranges_string(regroup_ranges(sg_port_ranges))
}
port_ranges.extend(sg_port_ranges)
range_string = port_ranges_string(regroup_ranges(port_ranges))
target['ports'] = range_string
target['public_sgs'] = public_sgs
if target['ports'] == "":
issue_msg = 'No ports open for tcp or udp (probably can only be pinged). Rules that are not tcp or udp: {} -- {}'
warnings.append(
issue_msg.format(
json.dumps(
pyjq.all(
'.[]|select((.IpProtocol!="tcp") and (.IpProtocol!="udp"))'
.format(selection), ingress)), account))
public_nodes.append(target)
# For the network diagram, if an ELB has availability across 3 subnets, I put one node in each subnet.
# We don't care about that when we want to know what is public and it makes it confusing when you
# see 3 resources with the same hostname, when you view your environment as only having one ELB.
# This same issue exists for RDS.
# Reduce these to single nodes.
reduced_nodes = {}
for node in public_nodes:
reduced_nodes[node['hostname']] = node
public_nodes = []
for _, node in reduced_nodes.items():
public_nodes.append(node)
account = Account(None, account)
for region_json in get_regions(account):
region = Region(account, region_json)
# Look for CloudFront
if region.name == 'us-east-1':
json_blob = query_aws(region.account,
'cloudfront-list-distributions', region)
for distribution in json_blob.get('DistributionList',
{}).get('Items', []):
if not distribution['Enabled']:
continue
target = {'arn': distribution['ARN'], 'account': account.name}
target['type'] = 'cloudfront'
target['hostname'] = distribution['DomainName']
target['ports'] = '80,443'
public_nodes.append(target)
# Look for API Gateway
json_blob = query_aws(region.account, 'apigateway-get-rest-apis',
region)
if json_blob is not None:
for api in json_blob.get('items', []):
target = {'arn': api['id'], 'account': account.name}
target['type'] = 'apigateway'
target['hostname'] = '{}.execute-api.{}.amazonaws.com'.format(
api['id'], region.name)
target['ports'] = '80,443'
public_nodes.append(target)
# Write cache file
with open(cache_file_path, 'w') as f:
f.write(json.dumps(public_nodes, indent=4, sort_keys=True))
return public_nodes, warnings
def validate_vulnerability_response_id(content):
return pyjq.first('.data.createVulnerability.vulnerability.id', content)
def step_impl(context):
resp = get_sippool_list(context)
info = pyjq.first('.[]|select(.sip=="{0}")'.format(context.valid_sip),
resp)
component_group_id = info["used_by"]
assert component_group_id == context.component_group_id
def validate_evidence_response(content):
return pyjq.first('.data.createVulnerabilityEvidence.vulnEvidence.name',
content)
def validate_repo_query(content):
return pyjq.first('.data.repoByName', content)
def validate_scan_response(content):
return pyjq.first('.data.createScan.scan.name', content)
def validate_test_case_query(content):
return pyjq.first('.data.createOrUpdateTestCase.case.name', content)
def validate_target_response(content):
return pyjq.first('.data.createTarget.target.name', content)
def validate_user_stories(content):
return pyjq.first('.data.userStories', content)
def __k8s(self):
kubectl_cmd = DockerCommand(
self._cmd, self._context.opt['--docker-image-kubectl'],
self._context.opt['--volume-from'], True)
helm_cmd = DockerCommand(self._cmd,
self._context.opt['--docker-image-helm'],
self._context.opt['--volume-from'], True)
# Use release name instead of the namespace name for release
release = self.__getRelease().replace('/', '-')
namespace = self.__getNamespace()
host = self.__getHost()
# Need to create default helm charts
if self._context.opt['--create-default-helm']:
# Check that the chart dir no exists
if os.path.isdir('%s/templates' %
self._context.opt['--deploy-spec-dir']):
raise ValueError(
'Directory %s/templates already exists, while --deploy-spec-dir has been selected.'
% self._context.opt['--deploy-spec-dir'])
elif os.path.isfile('%s/values.yaml' %
self._context.opt['--deploy-spec-dir']):
raise ValueError(
'File %s/values.yaml already exists, while --deploy-spec-dir has been selected.'
% self._context.opt['--deploy-spec-dir'])
elif os.path.isfile('%s/Chart.yaml' %
self._context.opt['--deploy-spec-dir']):
raise ValueError(
'File %s/Chart.yaml already exists, while --deploy-spec-dir has been selected.'
% self._context.opt['--deploy-spec-dir'])
else:
os.makedirs('%s/templates' %
self._context.opt['--deploy-spec-dir'])
self._cmd.run_command('cp -R /cdp/k8s/charts/* %s/' %
self._context.opt['--deploy-spec-dir'])
with open(
'%s/Chart.yaml' %
self._context.opt['--deploy-spec-dir'],
'w') as outfile:
data = dict(apiVersion='v1',
description='A Helm chart for Kubernetes',
name=os.environ['CI_PROJECT_NAME'],
version='0.1.0')
yaml.dump(data, outfile)
final_deploy_spec_dir = '%s_final' % self._context.opt[
'--deploy-spec-dir']
final_template_deploy_spec_dir = '%s/templates' % final_deploy_spec_dir
try:
os.makedirs(final_template_deploy_spec_dir)
shutil.copyfile(
'%s/Chart.yaml' % self._context.opt['--deploy-spec-dir'],
'%s/Chart.yaml' % final_deploy_spec_dir)
except OSError as e:
LOG.error(str(e))
command = 'upgrade %s' % release
command = '%s %s' % (command, final_deploy_spec_dir)
command = '%s --timeout %s' % (command, self._context.opt['--timeout'])
set_command = '--set namespace=%s' % namespace
#Deprecated, we will detect if tiller is available in our namespace or in kube-system
if self._context.opt['--tiller-namespace']:
command = '%s --tiller-namespace=%s' % (command, namespace)
tiller_length = 0
tiller_json = ''
try:
if not self._context.opt['--tiller-namespace']:
tiller_json = ''.join(
kubectl_cmd.run(
'get pod --namespace %s -l name="tiller" -o json --ignore-not-found=false'
% (namespace)))
tiller_length = len(
pyjq.first('.items[] | .metadata.labels.name',
json.loads(tiller_json)))
command = '%s --tiller-namespace=%s' % (command, namespace)
except Exception as e:
# Not present
LOG.verbose(str(e))
# Need to create default helm charts
if self._context.opt['--create-default-helm']:
set_command = '%s --set service.internalPort=%s' % (
set_command, self._context.opt['--internal-port'])
if self._context.opt['--image-tag-latest']:
tag = self.__getTagLatest()
pullPolicy = 'Always'
elif self._context.opt['--image-tag-sha1']:
tag = self.__getTagSha1()
pullPolicy = 'IfNotPresent'
else:
tag = self.__getTagBranchName()
pullPolicy = 'Always'
set_command = '%s --set ingress.host=%s' % (set_command, host)
set_command = '%s --set ingress.subdomain=%s' % (
set_command, os.getenv('CDP_DNS_SUBDOMAIN', None))
set_command = '%s --set image.commit.sha=sha-%s' % (
set_command, os.environ['CI_COMMIT_SHA'][:8])
set_command = '%s --set image.registry=%s' % (set_command,
self._context.registry)
set_command = '%s --set image.repository=%s' % (
set_command, self._context.repository)
set_command = '%s --set image.tag=%s' % (set_command, tag)
set_command = '%s --set image.pullPolicy=%s' % (set_command,
pullPolicy)
# Need to add secret file for docker registry
if not self._context.opt['--use-aws-ecr'] and not self._context.opt[
'--use-registry'] == 'aws-ecr':
# Add secret (Only if secret is not exist )
self._cmd.run_command(
'cp /cdp/k8s/secret/cdp-secret.yaml %s/templates/' %
self._context.opt['--deploy-spec-dir'])
set_command = '%s --set image.credentials.username=%s' % (
set_command, self._context.registry_user_ro)
set_command = '%s --set image.credentials.password=%s' % (
set_command, self._context.registry_token_ro)
set_command = '%s --set image.imagePullSecrets=cdp-%s-%s' % (
set_command, self._context.registry.replace(':', '-'), release)
if self._context.opt['--create-gitlab-secret']:
if os.getenv('CI_ENVIRONMENT_NAME', None) is None:
LOG.err(
'Can not use gitlab secret because environment is not defined in gitlab job.'
)
secretEnvPattern = 'CDP_SECRET_%s_' % os.getenv(
'CI_ENVIRONMENT_NAME', None)
secretFileCreated = False
#LOG.info('Looking for environnement variables starting with : %s' % secretEnvPattern)
for envVar, envValue in dict(os.environ).items():
if envVar.startswith(secretEnvPattern.upper(), 0):
if not secretFileCreated:
#LOG.info('Some secrets has been found ! Generating a kubernetes secret file !')
#Get the secret templates if we envVar to transform into secret
self._cmd.run_command(
'cp /cdp/k8s/secret/cdp-gitlab-secret.yaml %s/templates/'
% self._context.opt['--deploy-spec-dir'])
secretFileCreated = True
#For each envVar of the right environnement we had a line in the secret
self._cmd.run_secret_command(
'echo " %s: \'%s\'" >> %s/templates/cdp-gitlab-secret.yaml'
% (envVar[len(secretEnvPattern):], envValue,
self._context.opt['--deploy-spec-dir']))
command = '%s --debug' % command
command = '%s -i' % command
command = '%s --namespace=%s' % (command, namespace)
command = '%s --force' % command
command = '%s --wait' % command
command = '%s --atomic' % command
now = datetime.datetime.utcnow()
date_format = '%Y-%m-%dT%H%M%S'
if self._context.opt['--delete-labels']:
command = '%s --description deletionTimestamp=%sZ' % (
command, (now + datetime.timedelta(
minutes=int(self._context.opt['--delete-labels']))
).strftime(date_format))
elif namespace[:53] == self.__getName(False)[:53]:
command = '%s --description deletionTimestamp=%sZ' % (command, (
now +
datetime.timedelta(minutes=int(240))).strftime(date_format))
# Template charts for secret
tmp_templating_file = '%s/all_resources.tmp' % final_deploy_spec_dir
template_command = 'template %s' % self._context.opt[
'--deploy-spec-dir']
template_command = '%s %s' % (template_command, set_command)
if self._context.opt['--values']:
valuesFiles = self._context.opt['--values'].strip().split(',')
values = '--values %s/' % self._context.opt['--deploy-spec-dir'] + (
' --values %s/' %
self._context.opt['--deploy-spec-dir']).join(valuesFiles)
template_command = '%s %s' % (template_command, values)
template_command = '%s --name=%s' % (template_command, release)
template_command = '%s --namespace=%s' % (template_command, namespace)
template_command = '%s > %s' % (template_command, tmp_templating_file)
helm_cmd.run(template_command)
image_pull_secret_value = 'cdp-%s-%s' % (self._context.registry,
release)
image_pull_secret_value = image_pull_secret_value.replace(':', '-')
with open(tmp_templating_file, 'r') as stream:
docs = list(yaml.load_all(stream))
final_docs = []
for doc in docs:
if doc is not None:
LOG.verbose(doc)
final_docs.append(doc)
#Manage Deployement and StatefullSate
if not self._context.opt['--use-aws-ecr'] and not self._context.opt[
'--use-registry'] == 'aws-ecr' and 'kind' in doc and 'spec' in doc and (
'template' in doc['spec']
or 'jobTemplate' in doc['spec']):
doc = CLIDriver.addImageSecret(
doc, image_pull_secret_value)
with open('%s/all_resources.yaml' % final_template_deploy_spec_dir,
'w') as outfile:
LOG.info(yaml.dump_all(final_docs))
yaml.dump_all(final_docs, outfile)
# Install or Upgrade environnement
helm_cmd.run(command)
if self._context.opt['--delete-labels']:
kubectl_cmd.run(
'label namespace %s deletable=true creationTimestamp=%sZ deletionTimestamp=%sZ --namespace=%s --overwrite'
% (namespace, now.strftime(date_format),
(now + datetime.timedelta(
minutes=int(self._context.opt['--delete-labels']))
).strftime(date_format), namespace))
elif not self._context.opt[
'--delete-labels'] and namespace[:53] == self.__getName(
False)[:53]:
kubectl_cmd.run(
'label namespace %s deletable=true creationTimestamp=%sZ deletionTimestamp=%sZ --namespace=%s --overwrite'
% (namespace, now.strftime(date_format),
(now + datetime.timedelta(minutes=int(240))
).strftime(date_format), namespace))
self.__update_environment()
def validate_user_story(content):
return pyjq.first('.data.createOrUpdateUserStory.userStory.shortName',
content)
def run(self):
consumer = Consumer(self.config)
callbacks = collections.defaultdict(list)
functions = self.functions
def close():
log.info('Closing consumer')
consumer.close()
atexit.register(close)
while True:
add, update, remove = functions.refresh()
if add or update or remove:
existing_topics = set(callbacks.keys())
for f in add:
callbacks[functions.arguments(f).get('topic')].append(f)
for f in update:
pass
for f in remove:
callbacks[functions.arguments(f).get('topic')].remove(f)
interested_topics = set(callbacks.keys())
if existing_topics.symmetric_difference(interested_topics):
log.debug(f'Subscribing to {interested_topics}')
consumer.subscribe(list(interested_topics))
log.debug('Before polling...')
message = consumer.poll(timeout=functions.refresh_interval)
log.debug('After polling...')
if not message:
log.debug('Empty message received')
elif not message.error():
log.debug('Key:' + str(message.key()) + ' Value:' +
str(message.value()))
topic, key, value = message.topic(), \
message.key(), \
message.value()
try:
key = message.key().decode('utf-8')
except:
pass
try:
value = json.loads(value)
except:
pass
for function in callbacks[topic]:
jq_filter = functions.arguments(function).get('filter')
try:
if jq_filter and not pyjq.first(jq_filter, value):
continue
except:
log.error(
f'Could not filter message value with {jq_filter}')
data = self.function_data(function, topic, key, value)
log.debug('Invoking a function with data:' + str(data))
functions.gateway.post(functions._gateway_base +
f'/function/{function["name"]}',
data=data)
log.debug('Function: ' + f'/function/{function["name"]}' +
' Data:' + str(data))
# if auto commit not enabled, manually commit the messages
if not bool(os.getenv('ENABLE_AUTO_COMMIT', 'True')):
current_msg = message
consumer.commit(message=current_msg, async=False)
def test_first(self):
self.assertEqual(
pyjq.first('.[] | . + $foo', ['val1', 'val2'], vars=dict(foo='bar')),
'val1bar'
)
def step_impl(context):
assert context.mysql_group
mysql_group_id = context.mysql_group[0]["mysql_group_id"]
instances = get_mysql_info_in_group(context, mysql_group_id)
master_instance = pyjq.first('.[]|select(.mysql_instance_role=="STATUS_MYSQL_MASTER")', instances)
context.mysql_instance = master_instance
def condition(context, flag):
resp = api_post(context, context.version_3 + "/mysql/config/list", body)
config = pyjq.first('.data[]', resp)
if config["mysql_instance_option_current_value"] == option_value:
return True
def _current(self):
headers = {'Authorization': 'Bearer ' + self.config['token'], 'accept': 'application/json'}
url = 'https://api.nature.global/1/appliances'
res = requests.get(url, headers=headers)
q = '.[] | select(.id=="{}") | .settings'.format(self.config['appliance'])
return int(pyjq.first(q, res.json())['temp'])
