def run_assertion(method, description, args, expected):
description = method + ': ' + description
print
print "Running", description, method
method = {
'diff': diff,
'apply_diff': apply_diff,
'transform': transform_object_diff
}[method]
if method is transform_object_diff:
if len(args) == 3:
original, obja, objb = args
policy = None
else:
original, obja, objb, policy = args
diffa = diff(original, obja, policy)['v']
diffb = diff(original, objb, policy)['v']
tdiff = transform_object_diff(diffa, diffb, original, policy)
intermediate = apply_diff(original, diffb)
method = apply_diff
args = [intermediate, tdiff]
got = method(*args)
if not equals(got, expected):
print
print "Exception coming..."
print
print "\tdescription", description
print "\tgot", simplejson.dumps(got)
print "\texp", simplejson.dumps(expected)
return equals(got, expected)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("first")
parser.add_argument("second")
parser.add_argument("-p", "--patch", action="store_true", default=False)
parser.add_argument("-s", "--syntax", action="store", type=str, default="compact")
parser.add_argument("-i", "--indent", action="store", type=int, default=None)
args = parser.parse_args()
with open(args.first, "r") as f:
with open(args.second, "r") as g:
jf = json.load(f)
jg = json.load(g)
if args.patch:
x = jsondiff.patch(
jf,
jg,
marshal=True,
syntax=args.syntax
)
else:
x = jsondiff.diff(
jf,
jg,
marshal=True,
syntax=args.syntax
)
json.dump(x, sys.stdout, indent=args.indent)
def parse_payload(cls, desired, reported):
if desired is None:
delta = reported
elif reported is None:
delta = desired
else:
delta = jsondiff.diff(desired, reported)
return delta
def run(self, items: List[dict], **kwargs) -> List[dict]:
"""Run simple difference calculation on results based on a baseline."""
logger.debug('Performing simple local baseline analysis.')
if not kwargs.get('baseline'):
raise BadArguments('Cannot run simple analysis. No baseline found.')
for i in items:
i['diff'] = diff(kwargs['baseline']['stdout'], i['stdout'])
return items
def run(self, items: List[dict], **kwargs) -> List[dict]:
"""Run simple difference calculation on results based on a baseline."""
logger.debug("Performing simple local baseline analysis.")
if not kwargs.get("baseline"):
raise BadArguments("Cannot run simple analysis. No baseline found.")
for i in items:
i["diff"] = diff(kwargs["baseline"]["stdout"], i["stdout"])
return items
def test_diff():
data1 = {"age": 42,
"name": "Bob",
"food": "okonomiyaki"
}
data2 = {"name": "Casimir",
"Location": "Torcy",
"age": 42
}
assert diff(data1, data2) == {
'-food': 'okonomiyaki',
'name': 'Casimir',
'+Location': 'Torcy'}
def test_restart_heketi_pod(self):
"""Validate restarting heketi pod"""
# create heketi volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 1")
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'], raise_on_error=False)
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
# get heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
# delete heketi-pod (it restarts the pod)
oc_delete(self.ocp_master_node[0], 'pod', heketi_pod_name)
wait_for_resource_absence(self.ocp_master_node[0],
'pod', heketi_pod_name)
# get new heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node[0],
heketi_pod_name)
# check heketi server is running
self.assertTrue(
hello_heketi(self.heketi_client_node, self.heketi_server_url),
"Heketi server %s is not alive" % self.heketi_server_url
)
# compare the topology
new_topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertEqual(new_topo_info, topo_info, "topology info is not same,"
" difference - %s" % diff(topo_info, new_topo_info))
# create new volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=2, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 20")
heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def step_impl(context, response):
if response == 'empty':
json_data = {}
json_expected = {}
else:
json_data = json.loads(context.last_response.text)
json_expected = json.loads(response)
# remove the id from response and expected result
if isinstance(json_data, list):
[data.pop('id', None) for data in json_data]
[data.pop('created', None) for data in json_data]
[data.pop('id', None) for data in json_expected]
[data.pop('created', None) for data in json_expected]
else:
json_data.pop('id', None)
json_data.pop('created', None)
json_expected.pop('id', None)
json_expected.pop('created', None)
assert len(diff(json_data, json_expected)) == 0
def _web_post_changes_enerpi_sensors_json(dict_web_form, lines_config, dict_config, dest_filepath=None):
"""
Process changes in web editor of ENERPI SENSORS JSON File
:param dict_web_form: :OrderedDict: Posted Form with new JSON data
:param lines_config: :list: original file text lines (1) with original JSON
:param dict_config: :OrderedDict: original config dict of dicts
(like the one 'web_edit_enerpi_sensors_json' returns)
:param dest_filepath: :str: (optional) destination filepath for save configuration changes
:return: :tuple: (:dict: alert message, :list: text lines, :OrderedDict: updated dict_config))
"""
alerta = None
t, sub = TITLE_EDIT_JS_SENSORS, SUBTITLE_EDIT_JS_SENSORS
try:
ant_json = json.loads(dict_config[t][sub][0], encoding=ENCODING)
new_json = json.loads(dict_web_form[sub], encoding=ENCODING)
diff_json = diff(ant_json, new_json)
if diff_json:
if dest_filepath is not None:
log('New JSON Sensors config ("{}")\nwill be saved in "{}"'
.format(new_json, dest_filepath), 'warning', False)
with open(dest_filepath, 'w') as f:
json.dump(new_json, f, indent=1)
str_cambios = ('Configuration changes in ENERPI SENSORS:<br>{}<br> New config SAVED!'
.format('JSON DIFF- <strong>"{}"</strong>'.format(diff_json)))
log(str_cambios, 'debug', False)
alerta = {'alert_type': 'warning', 'texto_alerta': str_cambios}
if dest_filepath is not None:
lines_config = get_lines_file(dest_filepath)
ok, dict_config = _web_edit_enerpi_sensors_json(lines_config)
if not ok:
alerta['alert_type'] = 'error'
alerta['texto_alerta'] += dict_config['error']
alerta['texto_alerta'] += '\nNEW JSON SENSORS FILE NOT VALID!! FIX IT, PLEASE'
except json.decoder.JSONDecodeError:
msg_err = ('JSONDecodeError in web_post_changes_enerpi_sensors_json: {}'.format(dict_config[t][sub]))
alerta = {'alert_type': 'error', 'texto_alerta': msg_err}
return alerta, lines_config, dict_config
def main():
j = open('./directory.json').read()
directory = json.loads(j)
directory_master = get_directory_from_master_branch()
directory_diff = {}
directory_diff = json.loads(diff(directory_master, directory, syntax='explicit', dump=True))
spec_dir = './api.freifunk.net/specs/*.json'
spec_files = glob.glob(spec_dir)
for spec_file in spec_files:
spec_content = open(spec_file).read()
ff_api_specs[os.path.splitext(os.path.basename(spec_file))[0]] = json.loads(spec_content)
urls_to_load = []
invalid_urls = []
if "$insert" in directory_diff:
print("check inserted entries")
for x in directory_diff["$insert"]:
urls_to_load.append(directory_diff["$insert"][x])
if "$update" in directory_diff:
print("check updated entries")
for x in directory_diff["$update"]:
urls_to_load.append(directory_diff["$update"][x])
if urls_to_load == []:
print("check all files, as nothing else changed")
for x in directory_master:
urls_to_load.append(directory_master[x])
result = fetch_parallel(urls_to_load)
if result.empty():
print('Result: All URLs are valid :-)')
sys.exit(0)
else:
print('\nResult: Invalid URLs found :-(')
sys.exit(1)
def process_bounty_changes(old_bounty, new_bounty):
"""Process Bounty changes.
Args:
old_bounty (dashboard.models.Bounty): The old Bounty object.
new_bounty (dashboard.models.Bounty): The new Bounty object.
"""
from dashboard.utils import build_profile_pairs
profile_pairs = None
# process bounty sync requests
did_bsr = False
for bsr in BountySyncRequest.objects.filter(
processed=False, github_url=new_bounty.github_url):
did_bsr = True
bsr.processed = True
bsr.save()
# get json diff
json_diff = diff(old_bounty.raw_data,
new_bounty.raw_data) if old_bounty else None
# new bounty
if not old_bounty or (not old_bounty and new_bounty
and new_bounty.is_open) or (not old_bounty.is_open
and new_bounty.is_open):
is_greater_than_x_days_old = new_bounty.web3_created < (
timezone.now() - timezone.timedelta(hours=24))
if is_greater_than_x_days_old:
msg = 'attempting to create a new bounty ({new_bounty.standard_bounties_id}) when is_greater_than_x_days_old = True'
print(msg)
raise Exception(msg)
event_name = 'new_bounty'
elif old_bounty.num_fulfillments < new_bounty.num_fulfillments:
event_name = 'work_submitted'
elif old_bounty.is_open and not new_bounty.is_open:
if new_bounty.status == 'cancelled':
event_name = 'killed_bounty'
else:
event_name = 'work_done'
else:
event_name = 'unknown_event'
logging.error(
f'got an unknown event from bounty {old_bounty.pk} => {new_bounty.pk}: {json_diff}'
)
print(f"- {event_name} event; diff => {json_diff}")
# Build profile pairs list
if new_bounty.fulfillments.exists():
profile_pairs = build_profile_pairs(new_bounty)
# marketing
if event_name != 'unknown_event':
print("============ posting ==============")
did_post_to_twitter = maybe_market_to_twitter(new_bounty, event_name)
did_post_to_slack = maybe_market_to_slack(new_bounty, event_name)
did_post_to_github = maybe_market_to_github(new_bounty, event_name,
profile_pairs)
did_post_to_email = maybe_market_to_email(new_bounty, event_name)
print("============ done posting ==============")
# what happened
what_happened = {
'did_bsr': did_bsr,
'did_post_to_email': did_post_to_email,
'did_post_to_github': did_post_to_github,
'did_post_to_slack': did_post_to_slack,
'did_post_to_twitter': did_post_to_twitter,
}
print("changes processed: ")
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(what_happened)
else:
print('No notifications sent - Event Type Unknown = did_bsr: ',
did_bsr)
# elif str(diff_name) == '$update':
# self.reqless.append(self.diff_json[diff_name])
elif str(diff_name) == '$insert':
self.sqlless.append(self.diff_json[diff_name])
else:
if 'delete' in str(
self.diff_json[diff_name]) or 'insert' in str(
self.diff_json[diff_name]) or 'update' in str(
self.diff_json[diff_name]):
diff_json_match(self.diff_json[diff_name], self.sqlless,
self.reqless, self.matchinfo).find_diff()
else:
self.matchinfo.append(
{diff_name: self.diff_json[diff_name]})
return {
'查询缺少': self.sqlless,
'返回值缺少': self.reqless,
'匹配结果': self.matchinfo
}
if __name__ == '__main__':
a = {}
b = {'code': 500, 'data': None, 'message': '服务器内部处理错误', 'success': False}
matchinfo = diff(a, b, syntax='symmetric')
print(matchinfo)
# matchinfo = [{}, {'code': 500, 'data': None, 'message': '服务器内部处理错误', 'success': False}]
mmm = diff_json_match(matchinfo, [], [], []).find_diff()
print(mmm)
def test_json(self):
entrypoint = parser.from_JSON_string(TEST_JSON)
out = parser.to_JSON_string(entrypoint)
d = diff(TEST_JSON, out)
payload = {
'jsonrpc': '2.0',
'method': method,
'id': 'test',
'params': params
}
endpoint = 'http://' + self.Host + ':' + str(self.Port) + '/rpc'
# print(endpoint)
# print(json.dumps(payload))
response = requests.post(endpoint,
data=json.dumps(payload),
headers={
'Content-Type': 'application/json'
}).json()
# print(response)
if 'error' in response:
raise ValueError(response['error'])
return response
firstService = BlockExplorerSerivce('localhost', 22869)
secondService = BlockExplorerSerivce('localhost', 23869)
for i in range(1, 2418):
first = firstService.getBlockInfo(i)
second = secondService.getBlockInfo(i)
if first != second:
print("Difference at height: %d" % (i))
print(json.dumps(diff(first, second, syntax='symmetric'), indent=4))
break
def main():
log = logging.getLogger(__name__)
# es_cred = None
try:
with open('auth.yml') as f:
es_cred = yaml.load(f)['creds']
with open('clusters.yml') as f:
clusters = yaml.load(f)
except Exception as e:
# log error
raise e
cluster_name = 'monitoring_sat'
watcher_sets = wm.metadata.keys()
log.info("available clusters: %s" % repr(clusters.keys()))
log.info("contacting '%s' cluster" % cluster_name)
log.info("available watcher configs: %s" % repr(watcher_sets))
log.debug("cluster nodes: %s" % repr(clusters[cluster_name]))
# connect to the cluster and create a watcher API object
prod_es = es.Elasticsearch(hosts=clusters[cluster_name],
sniff_on_start=False,
sniff_on_connection_fail=False,
http_auth=(es_cred['user'], es_cred['pass']))
watch_es = es.client.xpack.watcher.WatcherClient(prod_es)
with open('watchers/cluster_state/cluster_state.json') as f:
clst_json = json.load(f)
try:
watcher_id = 'cluster_state'
watch_template = watch_es.get_watch(watcher_id)
wbody = watch_template['watch']
except es.exceptions.NotFoundError as e:
pprint(e)
wbody = clst_json
wp_result = watch_es.put_watch(watcher_id, wbody, active=True)
if (wp_result is not None):
status = ('Updated', 'Created')[wp_result['created']]
log.info("%s v%d : %s " %
(wp_result['_id'], wp_result['_version'], status))
else:
log.info('%s creation failed' % watcher_id)
sys.exit(0)
# pull the template watcher from the cluster
watch_template = watch_es.get_watch('filesystem_usage')
watch_body = watch_template['watch']
with open(
'watchers/filesystem_usage/filesystem_usage_watcher_template.json'
) as f:
fsuw_json = json.load(f)
#pprint(fsuw_json)
# jdiff = jsondiff.diff(fsuw_json, watch_body)
jdiff = jsondiff.diff(watch_body, fsuw_json)
pprint(jdiff)
try:
enodes_sm = watch_es.get_watch('elasticsearch_node_system_memory')
except es.exceptions.NotFoundError as e:
pprint(e)
sys.exit(0)
pprint(enodes_sm)
if (True):
sys.exit(0)
# clear out the metadata settings for the new watch
for _w_id in fs_metadata.keys():
watcher_id = "filesystem_usage_%s" % _w_id
wbody = watch_template['watch']
# clear out the metadata from the template and replace with cluster/fs specs
wbody['metadata'] = {}
wbody['metadata'] = fs_metadata[_w_id]
wp_result = watch_es.put_watch(watcher_id, wbody, active=True)
if (wp_result is not None):
status = ('Updated', 'Created')[wp_result['created']]
log.info("%s v%d : %s " %
(wp_result['_id'], wp_result['_version'], status))
else:
log.info('%s creation failed' % watcher_id)
# the template watcher gets activated sometimes after being queried;
# run a disable on it
watch_es.deactivate_watch('filesystem_usage_alert')
def perform(self):
for cluster in config.settings['clusters']:
if (self.region != 'all' and self.region != cluster['region']) or (self.env != 'all' and self.env != cluster['env']):
continue
print "*--------------------------------*"
print "Getting data for {} {}".format(cluster['region'], cluster['env'])
print "*--------------------------------*"
ecs = EcsHelper(cluster['region'])
ssm = SsmHelper(cluster['region'])
cluster['services'] = []
services = []
task_definitions = []
service_list_arns = ecs.list_services(cluster['name'], [], '')
for service_chunk in [service_list_arns[i:i + 10] for i in xrange(0, len(service_list_arns), 10)]:
for service in ecs.client.describe_services(cluster=cluster['name'], services=service_chunk)['services']:
name = "_".join((service['serviceName'].split('-')[3:-4]))
if self.services is not '' and name in self.services.split(','):
services.append(service)
elif self.services is '':
services.append(service)
for service in services:
task_definitions.append(ecs.client.describe_task_definition(
taskDefinition=service['taskDefinition']
))
for task_definition in task_definitions:
dict_ordered = None
for containerDefinition in task_definition['taskDefinition']['containerDefinitions']:
for param in containerDefinition['environment']:
if param['name'] == "VERSION" or param['name'] == "PARAM_VERSION":
dict_ordered = None
image_tag = containerDefinition['image'].split(':')[-1]
regionEnv = "{}.{}".format(base.get_landscape(cluster['region']), base.get_environment(cluster['env']))
print containerDefinition['name']
author = ssm.get_author(regionEnv, containerDefinition['name'], param['value'])
try:
author = ("".join(author).split('/')[-1:])[0]
except Exception as e:
author = None
github = GithubHelper().get_param_version_data(containerDefinition['name'], base.get_landscape(cluster['region']), cluster['env'])
version = regionEnv + "." + containerDefinition['name'].replace('-', '_') + "." + param['value']
ssm_version = ssm.get_param(version)
diff_output = diff(ssm_version, github['params'])
if self.diff_mode is True and author != self.ignore_author and (len(diff_output) > 0 or param['value'] != github['version'] or author != 'codeship_param_pusher'):
dict_ordered = OrderedDict([('name', containerDefinition['name']), ('region', regionEnv), ('image_tag', image_tag), ('running_param_version', param['value']), ('github_param_version', github['version']), ('author', author), ('diff', diff_output)])
break
elif self.diff_mode is False and author != self.ignore_author:
dict_ordered = OrderedDict([('name', containerDefinition['name']), ('region', regionEnv), ('image_tag', image_tag), ('running_param_version', param['value']), ('github_param_version', github['version']), ('author', author), ('diff', diff_output)])
break
if dict_ordered is not None: cluster['services'].append(dict_ordered)
print "\n\n"
self.output_data()
def _report_mismatch(self, run_data, upd_data):
log_error("run_data vs expected_data: {}".format(
str(jsondiff.diff(run_data, upd_data))[0:40]))
def test_inconsistent_objects(self, hge_ctx):
with open(self.dir() + "/test.yaml") as c:
test = yaml.safe_load(c)
# setup
st_code, resp = hge_ctx.v1q(json.loads(json.dumps(test['setup'])))
assert st_code == 200, resp
try:
# exec sql to cause inconsistentancy
sql_res = hge_ctx.sql(test['sql'])
# reload metadata
st_code, resp = hge_ctx.v1q(q=self.reload_metadata)
assert st_code == 200, resp
# fetch inconsistent objects
st_code, resp = hge_ctx.v1q(q=self.get_inconsistent_metadata)
assert st_code == 200, resp
incons_objs_test = test['inconsistent_objects']
incons_objs_resp = resp['inconsistent_objects']
assert resp['is_consistent'] == False, resp
assert incons_objs_resp == incons_objs_test, yaml.dump({
'response':
incons_objs_resp,
'expected':
incons_objs_test,
'diff':
jsondiff.diff(incons_objs_test, incons_objs_resp)
})
# export metadata
st_code, export = hge_ctx.v1q(q=self.export_metadata)
assert st_code == 200, export
# apply metadata
st_code, resp = hge_ctx.v1q(q={
"type": "replace_metadata",
"args": export
})
assert st_code == 400, resp
finally:
# drop inconsistent objects
st_code, resp = hge_ctx.v1q(q=self.drop_inconsistent_metadata)
assert st_code == 200, resp
# reload metadata
st_code, resp = hge_ctx.v1q(q=self.reload_metadata)
assert st_code == 200, resp
# fetch inconsistent objects
st_code, resp = hge_ctx.v1q(q=self.get_inconsistent_metadata)
assert st_code == 200, resp
assert resp['is_consistent'] == True, resp
assert len(resp['inconsistent_objects']) == 0, resp
# teardown
st_code, resp = hge_ctx.v1q(
json.loads(json.dumps(test['teardown'])))
assert st_code == 200, resp
def json_diff_report(expected, actual):
return jsondiff.diff(expected, actual, syntax="explicit")
def diffColLogs (clg1, clg2, frm, to):
print('Comparing nodes in col_logs... '+str(len(clg1['nodes']))+'-'+str(len(clg2['nodes'])))
result = {'globals':{}, 'nodes':[], 'links':[]}
node_mapping = {'inserted':{}, 'deleted':{}, 'same':{}}
node_name_id = {} # 1x1 mapping node name to new id
nm = ni = ndl = lm = li = ld = idx = 0
for orig_node in clg1['nodes']:
other_nd = LogDBUtils.findNodeByName(orig_node, clg2['nodes'])
if other_nd:
df = diff(orig_node, other_nd, syntax='symmetric')
# pprint.pprint(df)
# Add node to resulting dict assigning new node id
rnd = copy.deepcopy(other_nd)
rnd['id'] = idx
result['nodes'].append(rnd)
node_name_id[rnd['name']] = rnd['id']
node_mapping['same'][rnd['name']] = rnd
difrc = {'from':frm, 'to':to, 'diff':'modified'}
if not 'diffs' in rnd:
rnd['diffs'] = []
rnd['diffs'].append(difrc)
idx += 1
nm += 1
# print ('\n',orig_node['name']+' found\nORIG')
# pprint.pprint(orig_node)
# print('\nOTHER')
# pprint.pprint(other_nd)
# print('\nDIFF')
# pprint.pprint(df)
else:
# Add node to resulting dict assigning new node id
rnd = copy.deepcopy(orig_node)
rnd['id'] = idx
result['nodes'].append(rnd)
node_name_id[rnd['name']] = rnd['id']
node_mapping['deleted'][rnd['name']] = rnd
difrc = {'from':frm, 'to':to, 'diff':'deleted'}
if not 'diffs' in rnd:
rnd['diffs'] = []
rnd['diffs'].append(difrc)
idx += 1
ndl += 1
# print ('\n',orig_node['name']+' gone')
# Identify new nodes only
for other_node in clg2['nodes']:
if not LogDBUtils.findNodeByName(other_node, clg1['nodes']):
# Add node to resulting dict assigning new node id
rnd = copy.deepcopy(other_node)
rnd['id'] = idx
result['nodes'].append(rnd)
node_name_id[rnd['name']] = rnd['id']
node_mapping['inserted'][rnd['name']] = rnd
difrc = {'from':frm, 'to':to, 'diff':'inserted'}
if not 'diffs' in rnd:
rnd['diffs'] = []
rnd['diffs'].append(difrc)
idx += 1
ni += 1
# Build a new json combining both links
matched_links = []
def link_footprint(link):
return link['source_name']+'-'+link['target_name']
for lk in clg1['links']:
rlk = None
# Check if link still exists between these endpoint in new dict
if LogDBUtils.findLinkByEndpoints(lk, clg2):
if lk['source_name'] in node_name_id.keys() and\
lk['target_name'] in node_name_id.keys():
# Update link with new endpoint ids and add difference found
rlk = copy.deepcopy(lk)
rlk['source'] = node_name_id[lk['source_name']]
rlk['target'] = node_name_id[lk['target_name']]
difrc = {'from':frm, 'to':to, 'diff':'modified'}
if not 'diffs' in rlk:
rlk['diffs'] = []
rlk['diffs'].append(difrc)
# print ('existing link',link_footprint(rlk))
else: # Deleted link
rlk = copy.deepcopy(lk)
rlk['source'] = node_name_id[lk['source_name']]
rlk['target'] = node_name_id[lk['target_name']]
difrc = {'from':frm, 'to':to, 'diff':'deleted'}
if not 'diffs' in rlk:
rlk['diffs'] = []
rlk['diffs'].append(difrc)
# print ('deleted link',link_footprint(rlk))
if rlk:
result['links'].append(rlk)
matched_links.append(link_footprint(rlk))
# Look for new links
for lk in clg2['links']:
# Ignored, alread processed links
if link_footprint(lk) in matched_links:
continue
rlk = copy.deepcopy(lk)
rlk['source'] = node_name_id[lk['source_name']]
rlk['target'] = node_name_id[lk['target_name']]
difrc = {'from':frm, 'to':to, 'diff':'inserted'}
if not 'diffs' in rlk:
rlk['diffs'] = []
rlk['diffs'].append(difrc)
# print ('new link',link_footprint(lk) )
if rlk:
result['links'].append(rlk)
# pprint.pprint(result['links'])
# Merge all globals
funcs1 = len(clg1['globals']['functions'])
funcs2 = len(clg2['globals']['functions'])
protos1 = len(clg1['globals']['protocols'])
protos2 = len(clg2['globals']['protocols'])
trgts1 = len(clg1['globals']['targets'])
trgt2 = len(clg2['globals']['targets'])
result['globals']['functions'] = [x for x in set(clg1['globals']['functions']+clg2['globals']['functions'])]
result['globals']['protocols'] = [x for x in set(clg1['globals']['protocols']+clg2['globals']['protocols'])]
result['globals']['targets'] = [x for x in set(clg1['globals']['targets']+clg2['globals']['targets'])]
print('Identified '+
str(nm)+'/'+str(ndl)+'/'+str(ni)+
' nodes and '+
str(lm)+'/'+str(ld)+'/'+str(li)+
' links modified/deleted/inserted. '+
str(funcs1 - funcs2) + ' functions, '+
str(protos1 - protos2) + ' protocols, '+
str(trgts1 - trgt2) + ' targets.'
)
result['globals']['changes'] = {
'from':frm,
'to':to,
'nodes':{'new':ni, 'modified':nm,'deleted':ndl},
'links':{'new':li, 'modified':lm,'deleted':ld}
}
# pprint.pprint(result['globals'])
return result
def findDebugGroups(startnum, endnum, mmtracefile):
# dump that frame
#c:\utilities\apitrace-8.0.20190414-win64\bin\apitrace.exe dump mmconsole.trace --calls=23165-23562
args = [
the_apitrace, 'dump', '--calls=' + startnum + '-' + endnum,
'--color=never', mmtracefile
]
proc = subprocess.run(args, capture_output=True)
res = proc.stdout.decode("utf-8")
#find interesting debuggroups
# 30079 glPushDebugGroup(source = GL_DEBUG_SOURCE_APPLICATION, id = 1234, length = -1, message = "SphereRenderer::Render")
commands = res.split(os.linesep)
done = {}
groupstart = 0
groupend = 0
currgroup = ""
for c in commands:
m = re.search(r'^(\d+)\s+glPopDebugGroup', c)
if m:
groupend = m.group(1)
if len(groupstack) == 0:
print("found pop without push: " + c)
else:
groupstart, currgroup = groupstack.pop()
print("found end of debug group " + currgroup + " " +
groupstart + "-" + groupend)
if int(groupend) - int(groupstart) == 1:
print("no content")
else:
# dump both states
# c:\utilities\apitrace-msvc\x64\bin\apitrace.exe replay -D 167273 mmconsole.1.trace > before.json
args = [
the_apitrace, 'replay', '-D', groupstart, mmtracefile
]
proc = subprocess.run(args, capture_output=True)
text = safeString(proc.stdout.decode("ascii"))
before = json.loads(text)
args = [
the_apitrace, 'replay', '-D', groupend, mmtracefile
]
proc = subprocess.run(args, capture_output=True)
text = safeString(proc.stdout.decode("ascii"))
after = json.loads(text)
diffstr = jsondiff.diff(before, after, marshal=True)
print("found differences:")
json.dump(diffstr, sys.stdout, indent=2)
print("")
m = re.search(r'^(\d+)\s+glPushDebugGroup', c)
if m:
thestart = m.group(1)
m = re.search(r',\s+id\s+=\s+(\d+)', c)
if m:
m = re.search(r',\s+message\s+=\s+"(.*?)"', c)
if m:
thingy = m.group(1)
if thingy in done:
print("already looked at " + thingy)
else:
print("looking at " + thingy)
# do not look twice. the first frame seems to be
# different though, so I deactivated this
# done[thingy] = 1
groupstack.append([thestart, thingy])
def get_selected_collection(collection_id, connection, api_key):
"""
Input: Postman connection object, UUID of the collection chosen by the user, Postman API key of the user
Description: To fetch details about all the APIs present in a specfic collection and to detect changes if any
Returns the changes detected in the API schema
"""
boundary = ''
payload = ''
headers = {
'X-Api-Key': api_key,
'Content-type': 'multipart/form-data; boundary={}'.format(boundary)
}
connection.request("GET", "/collections/" + collection_id, payload,
headers)
response = connection.getresponse()
if response.status == 200:
data = json.loads(response.read())
# For each collection, a separate text file is created to store the details related to the collection
filepath = "./data/" + collection_id + ".txt"
os.makedirs(os.path.dirname(filepath), exist_ok=True)
if not os.path.exists(filepath):
with open(filepath, "w") as f:
f.write("{}")
f.close()
# Difference between the data received as part of the current API call and the data that previously existed in the .txt file
# The difference is computed twice to detect changes w.r.t to addition as well as deletion of key value pairs
with open(filepath, "r+") as f:
old_value = diff(data, json.load(f))
f.close()
with open(filepath, "r+") as f:
new_value = diff(json.load(f), data)
f.close()
# A list of changes in the existing API are determined
changes_detected = [
regex(str(value)) for value in [old_value, new_value]
]
# When changes are detected, the .txt file is updated according to the new API schema
if changes_detected:
with open(filepath, "w+") as f:
json.dump(data, f)
f.close()
# Formatting the changes detected to make it user-friendly
keys_old = "Old name of the query paramter: " + \
' '.join(changes_detected[0][0])
keys_new = "Updated name of the query parameter: " + \
' '.join(changes_detected[1][0])
keys_inserted = "Name of the query parameter newly added: " + \
' '.join(changes_detected[1][3])
keys_deleted = "Name of the query paramter that is deleted " + \
' '.join(changes_detected[0][2])
return keys_old + "\n" + keys_new + "\n" + keys_inserted + "\n" + keys_deleted
else:
raise Exception("Exited with status code " + str(response.status) +
'. ' +
str(json.loads(response.read())['error']['message']))
def test_ping_api(self):
ping_service = self.public_services_env.component(usage="ping")
openapi_def = ping_service.to_openapi()
canonical_def = get_canonical_json("ping_api.json")
self._fix_server_url(canonical_def)
self.assertFalse(jsondiff.diff(openapi_def, canonical_def))
import sys
import json
from pathlib import Path
from jsondiff import diff
if len(sys.argv) == 3:
old_file = Path(sys.argv[1])
new_file = Path(sys.argv[2])
if old_file.is_file() and new_file.is_file():
# with open(old_file) as json_data:
with old_file.open() as json_data:
old_json = json.load(json_data)
# with open(new_file) as json_data:
with new_file.open() as json_data:
new_json = json.load(json_data)
if diff(old_json, new_json):
sys.exit(1)
else:
sys.exit(0)
print(sys.argv)
print('json diff error: json_diff.py <new file> <old file>')
sys.exit(1)
def history_records_assemble(id, csv_dir, tar_dir, out_dir, assemble, do_diff):
if assemble:
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
os.makedirs(out_dir)
csvs = [c for c in os.listdir(csv_dir) if c.endswith(".csv")]
paths = []
# find all the files from the index csvs
for c in csvs:
tarname = c.rsplit(".", 1)[0] + ".tar.gz"
with codecs.open(os.path.join(csv_dir, c), "rb", "utf-8") as f:
reader = clcsv.UnicodeReader(f)
for row in reader:
if row[0] == id:
paths.append({
"csv" : c,
"tarname" : tarname,
"tarpath" : row[2],
"date" : row[1],
"fileid" : row[3]
})
# gather all the files in the target directory
with codecs.open(os.path.join(out_dir, "_index." + id + ".csv"), "wb", "utf-8") as g:
writer = clcsv.UnicodeWriter(g)
writer.writerow(["CSV", "Tar Name", "Tar Path", "Date", "File ID"])
for p in paths:
tarball = tarfile.open(os.path.join(tar_dir, p["tarname"]), "r:gz")
member = tarball.getmember(p["tarpath"])
handle = tarball.extractfile(member)
out = os.path.join(out_dir, p["date"] + "_" + p["fileid"] + ".json")
with codecs.open(out, "wb", "utf-8") as f:
shutil.copyfileobj(handle, f)
writer.writerow([p["csv"], p["tarname"], p["tarpath"], p["date"], p["fileid"]])
if do_diff:
difffile = os.path.join(out_dir, "_diff." + id + ".json")
if os.path.exists(difffile):
os.remove(difffile)
# order the files and diff them into a single summary file
# FIXME: note that this is not the standardised form of jsondiff, for some reason, but it
# will do for now.
changes = []
files = [f for f in os.listdir(out_dir) if f.endswith(".json")]
files.sort()
for i in range(len(files) - 1):
f1 = files[i]
f2 = files[i + 1]
with codecs.open(os.path.join(out_dir, f1), "rb", "utf-8") as r1, \
codecs.open(os.path.join(out_dir, f2), "rb", "utf-8") as r2:
j1 = json.loads(r1.read())
j2 = json.loads(r2.read())
d = diff(j1, j2)
d["_from"] = f1
d["_to"] = f2
d = _fix_symbols(d)
changes.append(d)
with codecs.open(difffile, "wb", "utf-8") as o:
o.write(json.dumps(changes, indent=2, sort_keys=True))
def test_partner_api(self):
partner_service = self.private_services_env.component(usage="partner")
openapi_def = partner_service.to_openapi()
canonical_def = get_canonical_json("partner_api.json")
self._fix_server_url(canonical_def)
self.assertFalse(jsondiff.diff(openapi_def, canonical_def))
def viz(str_num, date, line, matches, dict_num_day_ordinal,
dict_quarter_ordinal, dict_quarter_cardinal, dict_year_modifier,
dict_year_cardinal, dict_week_ordinal, dict_week_cardinal,
dict_month_modifier, dict_date, dict_day_cardinal, dict_month_ordinal,
dict_month_cardinal, dict_week_modifier, dict_count, dict_day_modifier,
dict_half_year, dict_year_ordinal, dict_quarter_modifier, id):
#print(len(matches))
dict_result = {}
dict_num = 0
date_d = {}
result = {}
split_index = []
#split_index.append(0)
for m in re.finditer(r' со | с | по | до |за | по | в |после ', line):
index = m.start()
split_index.append(index)
#split_on_date = re.split(r'с |по | до ', line)
if len(split_index) == 0:
split_index.append(0)
split_index.append(len(line))
#print("INTERVALS ", split_index)
num_split = 0
for split_loc in split_index:
num_split = num_split + 1
interval = []
result["id"] = str(id)
id = id + 1
r = {}
res_atr = []
out = {}
id_interval = 0
#r["DI_count"] = len(split_index)-1
# фильтровать повторяющиеся мэтчи
matches_arr = []
match_num = -1
for match in matches:
match_num = match_num + 1
matches_arr.append(match)
if match_num != 0:
match1 = matches_arr[match_num - 1].fact.as_json
match2 = matches_arr[match_num].fact.as_json
diff_match = jsondiff.diff(match1, match2)
if len(diff_match) == 0:
continue
for i in range(len(split_index)):
#for match in matches:
if match.span.start >= split_index[
i] and match.span.stop <= split_index[i + 1]:
#if match.span.start<:
dict_num = dict_num + 1
dict_match = {}
val = []
# interval_index = 1
# num_di = 0
# for split in split_index:
# num_di = num_di + 1
# if split < match.span.stop:
# interval_index = num_di
# break
#for m in re.finditer(r'с |по | до ', line):
# index, item = m.start(), m.group()
# print(index, m, item)
#dict_match["DI_count"] = len()
dict_match["start"] = match.span.start
dict_match["stop"] = match.span.stop
annotationText = ''
for tok in match.tokens:
annotationText = annotationText + " " + tok.value
dict_match["annotationText"] = annotationText
dict_match["word"] = match.tree.root.main.value
before = morph.parse(match.tokens[0].value)[0]
p = morph.parse(match.tokens[0].normalized)[0]
try:
d_list = list(match.fact.as_json.items())
dict_match["entity"] = list(
match.fact.as_json.items())[0][0]
dict_match["value"] = list(
match.fact.as_json.items())[0][1]
#print(isinstance(dict_match["value"], int))
#print("DAAAAAte")
if (re.search("ordinal", dict_match["entity"])) and (
p.tag.POS == 'NUMR') and (before.tag.POS
== "ADJF"):
dict_match["entity_cardinal"] = dict_match[
"entity"].replace("ordinal", "cardinal")
dict_match["value_cardinal"] = 1
if (re.search("ordinal", dict_match["entity"])):
if (re.search("year", dict_match["entity"])):
date_d["year{}".format(
dict_num)] = dict_match["value"]
if (re.search("month", dict_match["entity"])):
date_d["month{}".format(
dict_num)] = dict_match["value"]
dict_result[dict_num] = dict_match
#print("DATE",date)
#dict_match["date"] = date
except:
dict_match["entity"] = "error"
dict_match["value"] = "error"
continue
#dict_match["annotions"] = {dict_match["start"], dict_match["stop"] }
#result[dict_match["entity"]] = [dict_match["value"], dict_match["annotions"]]
#if (p.tag.POS == NUMR):
result_atr = {
#("DateInterval{}").format(id_interval):{
"value": dict_match["value"],
"annotations": {
"annotationStart": dict_match["start"],
"annotationEnd": dict_match["stop"],
"annotationText": dict_match["annotationText"]
},
#"id_interval":id_interval
#}
}
#print(result_atr, "______________")
if (dict_match["entity"]) in result:
if result[(dict_match["entity"])] != result_atr:
result[(dict_match["entity"]).replace(
"_", "_end_")] = [result_atr]
#result[(dict_match["entity"]).replace("_", "_start_")] = result[(dict_match["entity"])]
else:
result[(dict_match["entity"])] = [result_atr]
#print("____________", result)
r["DateInterval"] = [result]
#"DI_index": interval_index
#print(r)
#print("TAAAAARG", p.tag.POS)
#dict_result.update(dict_match)
#print(dict_date1)
#print(match.tree)
#json_d.value.append(match.tokens[0].value)
date.day_modifier.append(match.fact.day_modifier)
date.half_year.append(match.fact.half_year)
date.quarter_cardinal.append(match.fact.quarter_cardinal)
date.quarter_ordinal.append(match.fact.quarter_ordinal)
date.year_modifier.append(match.fact.year_modifier)
date.year_cardinal.append(match.fact.year_cardinal)
date.week_modifier.append(match.fact.week_modifier)
date.week_ordinal.append(match.fact.week_ordinal)
date.week_cardinal.append(match.fact.week_cardinal)
date.month_ordinal.append(match.fact.month_ordinal)
date.month_cardinal.append(match.fact.month_cardinal)
date.month_modifier.append(match.fact.month_modifier)
date.day_cardinal.append(match.fact.day_cardinal)
date.day_ordinal.append(match.fact.day_ordinal)
date.start_date.append(match.fact.start_date)
date.end_date.append(match.fact.end_date)
date.day_modifier.append(match.fact.day_modifier)
date.half_year.append(match.fact.half_year)
date.count.append(match.fact.count)
date.date.append(match.fact.date)
date.year_ordinal.append(match.fact.date)
date.quarter_modifier.append(match.fact.date)
# date.day.append(match.fact.day)
if len(str(skip_none(date.quarter_modifier))) > 0:
print("quarter_modifier: ", skip_none(date.quarter_modifier))
dict_quarter_modifier["{}".format(line)] = skip_none(
date.quarter_modifier)
if len(str(skip_none(date.year_ordinal))) > 0:
print("year_ordinal: ", skip_none(date.year_ordinal))
dict_year_ordinal["{}".format(line)] = skip_none(date.year_ordinal)
if len(str(skip_none(date.quarter_cardinal))) > 0:
print("quarter_cardinal: ", skip_none(date.quarter_cardinal))
dict_quarter_cardinal["{}".format(line)] = skip_none(
date.quarter_cardinal)
if len(str(skip_none(date.quarter_ordinal))) > 0:
print("quarter_ordinal: ", skip_none(date.quarter_ordinal))
dict_quarter_ordinal["{}".format(line)] = skip_none(
date.quarter_ordinal)
if len(str(skip_none(date.year_modifier))) > 0:
print("year_modifier: ", skip_none(date.year_modifier))
dict_year_modifier["{}".format(line)] = skip_none(
date.year_modifier)
if len(str(skip_none(date.year_cardinal))) > 0:
print("year_cardinal: ", skip_none(date.year_cardinal))
dict_year_cardinal["{}".format(line)] = skip_none(
date.year_cardinal)
if len(str(skip_none(date.date))) > 0:
print("date: ", skip_none(date.date))
dict_date["{}".format(line)] = skip_none(date.date)
if len(str(skip_none(date.week_ordinal))) > 0:
print("week_ordinal: ", skip_none(date.week_ordinal))
dict_week_ordinal["{}".format(line)] = skip_none(date.week_ordinal)
if len(str(skip_none(date.week_cardinal))) > 0:
print("week_cardinal: ", skip_none(date.week_cardinal))
dict_week_cardinal["{}".format(line)] = skip_none(
date.week_cardinal)
if len(str(skip_none(date.week_modifier))) > 0:
print("week_modifier: ", skip_none(date.week_modifier))
dict_week_modifier["{}".format(line)] = skip_none(
date.week_modifier)
if len(str(skip_none(date.month_ordinal))) > 0:
print("month_ordinal: ", skip_none(date.month_ordinal))
dict_month_ordinal["{}".format(line)] = skip_none(
date.month_ordinal)
if len(str(skip_none(date.month_cardinal))) > 0:
print("month_cardinal: ", skip_none(date.month_cardinal))
dict_month_cardinal["{}".format(line)] = skip_none(
date.month_cardinal)
if len(str(skip_none(date.month_modifier))) > 0:
print("month_modifier: ", skip_none(date.month_modifier))
dict_month_modifier["{}".format(line)] = skip_none(
date.month_modifier)
if len(str(skip_none(date.day_cardinal))) > 0:
print("day_cardinal: ", skip_none(date.day_cardinal))
dict_day_cardinal["{}".format(line)] = skip_none(date.day_cardinal)
if len(str(skip_none(date.day_ordinal))) > 0:
print("day_ordinal: ", skip_none(date.day_ordinal))
dict_num_day_ordinal["{}".format(line)] = skip_none(
date.day_ordinal)
if len(str(skip_none(date.start_date))) > 0:
print("start_date: ", skip_none(date.start_date))
dict_start_date["{}".format(line)] = skip_none(date.start_date)
if len(str(skip_none(date.end_date))) > 0:
print("end_date: ", skip_none(date.end_date))
dict_end_date["{}".format(line)] = skip_none(date.end_date)
if len(str(skip_none(date.day_modifier))) > 0:
print("day_modifier: ", skip_none(date.day_modifier))
dict_day_modifier["{}".format(line)] = skip_none(date.day_modifier)
if len(str(skip_none(date.half_year))) > 0:
print("half_year: ", skip_none(date.half_year))
dict_half_year["{}".format(line)] = skip_none(date.half_year)
if len(str(skip_none(date.count))) > 0:
print("count: ", skip_none(date.count))
dict_count["{}".format(line)] = skip_none(date.count)
#out[]
return dict_result, date, result, r, id
def main():
if (len(sys.argv) < 8):
sys.stderr.write(
'Usage: ./run_spark_hdfs.py <build_dir> <install_dir> <spark_master> <hdfs_namenode> <spark_deploy> <genomicsdb_version> <test_dir> [<build_type>]\n'
)
sys.stderr.write(
' Optional Argument 8 - build_type=Release|Coverage|...\n')
sys.exit(-1)
exe_path = sys.argv[2] + os.path.sep + 'bin'
spark_master = sys.argv[3]
namenode = sys.argv[4]
jar_dir = sys.argv[1] + os.path.sep + 'target'
spark_deploy = sys.argv[5]
genomicsdb_version = sys.argv[6]
test_dir = sys.argv[7]
if (len(sys.argv) == 9):
build_type = sys.argv[8]
else:
build_type = "default"
#Switch to tests directory
parent_dir = os.path.dirname(os.path.realpath(__file__))
os.chdir(parent_dir)
hostfile_path = parent_dir + os.path.sep + 'hostfile'
vid_path = parent_dir + os.path.sep
template_vcf_header_path = parent_dir + os.path.sep + 'inputs' + os.path.sep + 'template_vcf_header.vcf'
tmpdir = tempfile.mkdtemp()
ws_dir = tmpdir + os.path.sep + 'ws'
jacoco, jacoco_report_cmd = common.setup_jacoco(
os.path.abspath(sys.argv[1]), build_type)
loader_tests = [
{
"name":
"t0_1_2",
'golden_output':
'golden_outputs/t0_1_2_loading',
'callset_mapping_file':
'inputs/callsets/t0_1_2.json',
"column_partitions": [[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test0"
}],
[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test1"
}, {
"begin": 10000,
"workspace": "/tmp/ws",
"array": "test2"
}],
[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test3"
}, {
"begin": 3000,
"workspace": "/tmp/ws",
"array": "test4"
}, {
"begin": 6000,
"workspace": "/tmp/ws",
"array": "test5"
}, {
"begin": 9000,
"workspace": "/tmp/ws",
"array": "test6"
}, {
"begin": 12000,
"workspace": "/tmp/ws",
"array": "test7"
}]],
"query_params": [
{
"query_column_ranges": [12100, 12200],
"golden_output": {
"spark": "golden_outputs/spark_t0_1_2_vcf_at_12100",
}
},
{
"query_column_ranges": [0, 100000],
"golden_output": {
"spark": "golden_outputs/spark_t0_1_2_vcf_at_0",
}
},
{
"query_column_ranges": [12150, 100000],
"golden_output": {
"spark": "golden_outputs/spark_t0_1_2_vcf_at_12150",
}
},
]
},
{
"name":
"t0_overlapping",
'golden_output':
'golden_outputs/t0_overlapping',
'callset_mapping_file':
'inputs/callsets/t0_overlapping.json',
"column_partitions": [[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test0"
}],
[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test1"
}, {
"begin": 10000,
"workspace": "/tmp/ws",
"array": "test2"
}],
[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test3"
}, {
"begin": 3000,
"workspace": "/tmp/ws",
"array": "test4"
}, {
"begin": 6000,
"workspace": "/tmp/ws",
"array": "test5"
}, {
"begin": 9000,
"workspace": "/tmp/ws",
"array": "test6"
}, {
"begin": 12000,
"workspace": "/tmp/ws",
"array": "test7"
}]],
"query_params": [{
"query_column_ranges": [12202, 100000],
"golden_output": {
"spark": "golden_outputs/spark_t0_overlapping_at_12202",
}
}]
},
{
"name":
"t6_7_8",
'golden_output':
'golden_outputs/t6_7_8_loading',
'callset_mapping_file':
'inputs/callsets/t6_7_8.json',
"column_partitions": [[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test0"
}],
[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test1"
}, {
"begin": 500000,
"workspace": "/tmp/ws",
"array": "test2"
}, {
"begin": 1000000,
"workspace": "/tmp/ws",
"array": "test3"
}],
[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test4"
}, {
"begin": 250000,
"workspace": "/tmp/ws",
"array": "test5"
}, {
"begin": 500000,
"workspace": "/tmp/ws",
"array": "test6"
}, {
"begin": 750000,
"workspace": "/tmp/ws",
"array": "test7"
}, {
"begin": 1000000,
"workspace": "/tmp/ws",
"array": "test8"
}]],
"query_params": [{
"query_column_ranges": [0, 10000000],
"golden_output": {
"spark": "golden_outputs/spark_t6_7_8_vcf_at_0",
},
"query_block_size": 1000000,
"query_block_size_margin": 50000
}, {
"query_column_ranges": [8029500, 10000000],
"golden_output": {
"spark": "golden_outputs/spark_t6_7_8_vcf_at_8029500",
},
"query_block_size": 100000,
"query_block_size_margin": 5000
}, {
"query_column_ranges": [8029500, 8029500],
"golden_output": {
"spark":
"golden_outputs/spark_t6_7_8_vcf_at_8029500-8029500",
}
}]
},
{
"name":
"t0_1_2_combined",
'golden_output':
'golden_outputs/t0_1_2_combined',
'callset_mapping_file':
'inputs/callsets/t0_1_2_combined.json',
"column_partitions": [[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test0"
}],
[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test1"
}, {
"begin": 10000,
"workspace": "/tmp/ws",
"array": "test2"
}],
[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test3"
}, {
"begin": 3000,
"workspace": "/tmp/ws",
"array": "test4"
}, {
"begin": 6000,
"workspace": "/tmp/ws",
"array": "test5"
}, {
"begin": 9000,
"workspace": "/tmp/ws",
"array": "test6"
}, {
"begin": 12000,
"workspace": "/tmp/ws",
"array": "test7"
}]],
"query_params": [
{
"query_column_ranges": [0, 1000000],
"golden_output": {
"spark": "golden_outputs/spark_t0_1_2_combined",
},
"query_block_size": 100000,
"query_block_size_margin": 5000
},
]
},
{
"name":
"t0_haploid_triploid_1_2_3_triploid_deletion",
'golden_output':
'golden_outputs/t0_haploid_triploid_1_2_3_triploid_deletion_loading',
'callset_mapping_file':
'inputs/callsets/t0_haploid_triploid_1_2_3_triploid_deletion.json',
"vid_mapping_file":
"inputs/vid_DS_ID_phased_GT.json",
'size_per_column_partition':
1200,
'segment_size':
100,
"column_partitions": [[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test0"
}],
[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test1"
}, {
"begin": 10000,
"workspace": "/tmp/ws",
"array": "test2"
}],
[{
"begin": 0,
"workspace": "/tmp/ws",
"array": "test3"
}, {
"begin": 3000,
"workspace": "/tmp/ws",
"array": "test4"
}, {
"begin": 6000,
"workspace": "/tmp/ws",
"array": "test5"
}, {
"begin": 9000,
"workspace": "/tmp/ws",
"array": "test6"
}, {
"begin": 12000,
"workspace": "/tmp/ws",
"array": "test7"
}]],
"query_params": [{
"query_column_ranges": [0, 1000000],
'callset_mapping_file':
'inputs/callsets/t0_haploid_triploid_1_2_3_triploid_deletion.json',
"vid_mapping_file": "inputs/vid_DS_ID_phased_GT.json",
'segment_size': 100,
"golden_output": {
"spark":
"golden_outputs/spark_t0_haploid_triploid_1_2_3_triploid_deletion_java_vcf",
},
"query_block_size": 100000,
"query_block_size_margin": 5000
}, {
"query_column_ranges": [0, 1000000],
'callset_mapping_file':
'inputs/callsets/t0_haploid_triploid_1_2_3_triploid_deletion.json',
"vid_mapping_file": "inputs/vid_DS_ID_phased_GT.json",
'produce_GT_field': True,
'segment_size': 100,
"golden_output": {
"spark":
"golden_outputs/spark_t0_haploid_triploid_1_2_3_triploid_deletion_java_vcf_produce_GT",
},
"query_block_size": 100000,
"query_block_size_margin": 5000
}]
},
]
if ("://" in namenode):
pid = subprocess.Popen('hadoop fs -mkdir -p ' + namenode +
'/home/hadoop/.tiledb/',
shell=True,
stdout=subprocess.PIPE)
stdout_string = pid.communicate()[0]
if (pid.returncode != 0):
sys.stderr.write('Error creating hdfs:///home/hadoop/.tiledb/')
sys.exit(-1)
for test_params_dict in loader_tests:
test_name = test_params_dict['name']
for col_part in test_params_dict['column_partitions']:
test_loader_dict = create_loader_json(ws_dir, test_name,
test_params_dict, col_part,
test_dir)
if (test_name == "t0_1_2"):
test_loader_dict["compress_tiledb_array"] = True
if ("://" in namenode):
test_loader_dict = add_hdfs_to_loader_json(
test_loader_dict, namenode)
loader_json_filename = tmpdir + os.path.sep + test_name + '-loader.json'
with open(loader_json_filename, 'wb') as fptr:
json.dump(test_loader_dict,
fptr,
indent=4,
separators=(',', ': '))
fptr.close()
# invoke vcf2tiledb -r <rank> where <rank> goes from 0 to num partitions
# otherwise this only loads the first partition
for i in range(0, len(col_part)):
etl_cmd = exe_path + os.path.sep + 'vcf2tiledb -r ' + str(
i) + ' ' + loader_json_filename
pid = subprocess.Popen(etl_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_string, stderr_string = pid.communicate()
if (pid.returncode != 0):
sys.stderr.write('Loading failed for test: ' + test_name +
' rank ' + str(i) + '\n')
sys.stderr.write('Loading command: ' + etl_cmd + '\n')
sys.stderr.write('Loader file :' + str(test_loader_dict) +
'\n')
sys.stderr.write('Loading stdout: ' + stdout_string + '\n')
sys.stderr.write('Loading stderr: ' + stderr_string + '\n')
cleanup_and_exit(namenode, tmpdir, -1)
else:
sys.stdout.write('Loading passed for test: ' + test_name +
' rank ' + str(i) + '\n')
with open(loader_json_filename, 'wb') as fptr:
json.dump(test_loader_dict,
fptr,
indent=4,
separators=(',', ': '))
fptr.close()
for query_param_dict in test_params_dict['query_params']:
if ("://" in namenode):
test_query_dict = create_query_json(
namenode + ws_dir, test_name, query_param_dict,
test_dir)
else:
test_query_dict = create_query_json(
ws_dir, test_name, query_param_dict, test_dir)
test_query_dict[
'query_attributes'] = vcf_query_attributes_order
query_json_filename = tmpdir + os.path.sep + test_name + '-query.json'
with open(query_json_filename, 'wb') as fptr:
json.dump(test_query_dict,
fptr,
indent=4,
separators=(',', ': '))
fptr.close()
spark_cmd = 'spark-submit --class TestGenomicsDBSparkHDFS --master ' + spark_master + ' --deploy-mode ' + spark_deploy + ' --total-executor-cores 1 --executor-memory 512M --conf "spark.yarn.executor.memoryOverhead=3700" --conf "spark.executor.extraJavaOptions=' + jacoco + '" --conf "spark.driver.extraJavaOptions=' + jacoco + '" --jars ' + jar_dir + '/genomicsdb-' + genomicsdb_version + '-allinone.jar ' + jar_dir + '/genomicsdb-' + genomicsdb_version + '-examples.jar --loader ' + loader_json_filename + ' --query ' + query_json_filename + ' --template_vcf_header ' + template_vcf_header_path + ' --spark_master ' + spark_master + ' --jar_dir ' + jar_dir
if (test_name == "t6_7_8"):
spark_cmd = spark_cmd + ' --use-query-protobuf'
if (test_name == "t0_overlapping"):
spark_cmd = spark_cmd + ' --hostfile ' + hostfile_path
pid = subprocess.Popen(spark_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_string, stderr_string = pid.communicate()
if (pid.returncode != 0):
sys.stderr.write('Query test: ' + test_name +
' with query file ' +
query_json_filename + ' failed\n')
sys.stderr.write('Spark command was: ' + spark_cmd + '\n')
sys.stderr.write('Spark stdout was: ' + stdout_string +
'\n')
sys.stderr.write('Spark stderr was: ' + stderr_string +
'\n')
sys.stderr.write('Query file was: ' +
json.dumps(test_query_dict) + '\n')
cleanup_and_exit(namenode, tmpdir, -1)
stdout_list = stdout_string.splitlines(True)
stdout_list_filter = [
k for k in stdout_list if not k.startswith('##')
]
stdout_filter = "".join(stdout_list_filter)
md5sum_hash_str = str(hashlib.md5(stdout_filter).hexdigest())
if ('golden_output' in query_param_dict
and 'spark' in query_param_dict['golden_output']):
golden_stdout, golden_md5sum = get_file_content_and_md5sum(
query_param_dict['golden_output']['spark'])
if (golden_md5sum != md5sum_hash_str):
sys.stdout.write(
'Mismatch in query test: ' + test_name +
' with column ranges: ' +
str(query_param_dict['query_column_ranges']) +
' and loaded with ' + str(len(col_part)) +
' partitions\n')
print_diff(golden_stdout, stdout_filter)
sys.stderr.write('Spark command was: ' + spark_cmd +
'\n')
sys.stderr.write('Spark stdout was: ' + stdout_string +
'\n')
sys.stderr.write('Spark stderr was: ' + stderr_string +
'\n')
sys.stderr.write('Query file was: ' +
json.dumps(test_query_dict) + '\n')
cleanup_and_exit(namenode, tmpdir, -1)
else:
sys.stdout.write(
'Query test: ' + test_name +
' with column ranges: ' +
str(query_param_dict['query_column_ranges']) +
' and loaded with ' + str(len(col_part)) +
' partitions passed\n')
# add another spark run command to test datasourcev2 stuff
if ('vid_mapping_file' in query_param_dict):
vid_path_final = vid_path + query_param_dict[
'vid_mapping_file']
else:
vid_path_final = vid_path + "inputs" + os.path.sep + "vid.json"
spark_cmd_v2 = 'spark-submit --class TestGenomicsDBDataSourceV2 --master ' + spark_master + ' --deploy-mode ' + spark_deploy + ' --total-executor-cores 1 --executor-memory 512M --conf "spark.yarn.executor.memoryOverhead=3700" --conf "spark.executor.extraJavaOptions=' + jacoco + '" --conf "spark.driver.extraJavaOptions=' + jacoco + '" --jars ' + jar_dir + '/genomicsdb-' + genomicsdb_version + '-allinone.jar ' + jar_dir + '/genomicsdb-' + genomicsdb_version + '-examples.jar --loader ' + loader_json_filename + ' --query ' + query_json_filename + ' --vid ' + vid_path_final + ' --spark_master ' + spark_master
if (test_name == "t6_7_8"):
spark_cmd_v2 = spark_cmd_v2 + ' --use-query-protobuf'
if (test_name == "t0_overlapping"):
spark_cmd = spark_cmd_v2 + ' --hostfile ' + hostfile_path
pid = subprocess.Popen(spark_cmd_v2,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_string, stderr_string = pid.communicate()
if (pid.returncode != 0):
sys.stderr.write('Query test V2: ' + test_name +
' with query file ' +
query_json_filename + ' failed\n')
sys.stderr.write('Spark command was: ' + spark_cmd_v2 +
'\n')
sys.stderr.write('Spark stdout was: ' + stdout_string +
'\n')
sys.stderr.write('Spark stderr was: ' + stderr_string +
'\n')
sys.stderr.write('Query file was: ' +
json.dumps(test_query_dict) + '\n')
cleanup_and_exit(namenode, tmpdir, -1)
stdout_list = stdout_string.splitlines(True)
stdout_filter = "".join(stdout_list)
stdout_json = json.loads(stdout_filter)
if ('golden_output' in query_param_dict
and 'spark' in query_param_dict['golden_output']):
json_golden = get_json_from_file(
query_param_dict['golden_output']['spark'] + '_v2')
checkdiff = jsondiff.diff(stdout_json, json_golden)
if (not checkdiff):
sys.stdout.write(
'Query test V2: ' + test_name +
' with column ranges: ' +
str(query_param_dict['query_column_ranges']) +
' and loaded with ' + str(len(col_part)) +
' partitions passed\n')
else:
sys.stdout.write(
'Mismatch in query test V2: ' + test_name +
' with column ranges: ' +
str(query_param_dict['query_column_ranges']) +
' and loaded with ' + str(len(col_part)) +
' partitions\n')
print(checkdiff)
sys.stderr.write('Spark stdout was: ' + stdout_string +
'\n')
sys.stderr.write('Spark stderr was: ' + stderr_string +
'\n')
cleanup_and_exit(namenode, tmpdir, -1)
rc = common.report_jacoco_coverage(jacoco_report_cmd)
if (rc != 0):
cleanup_and_exit(namenode, tmpdir, -1)
cleanup_and_exit(namenode, tmpdir, 0)
# Check R package specs
if set(spec_files.keys()) != set(r_files.keys()):
for spec in set(spec_files.keys()) - set(r_files.keys()):
print('{} missing from R package'.format(spec), file=sys.stderr)
for spec in set(r_files.keys()) - set(spec_files.keys()):
print('{} found in R package but missing from specs/'.format(spec), file=sys.stderr)
sys.exit(1)
for spec_name in spec_files:
# check equality of specs
with open(spec_files[spec_name], 'r') as ip:
gold_spec = yaml.load(ip)
with open(py_files[spec_name], 'r') as ip:
py_spec = yaml.load(ip)
with open(r_files[spec_name], 'r') as ip:
r_spec = yaml.load(ip)
# Check python package
if jsondiff.diff(gold_spec, py_spec) != {}:
print('{} spec is different from python version'.format(spec_name), file=sys.stderr)
print(jsondiff.diff(gold_spec, py_spec), file=sys.stderr)
sys.exit(1)
# Check R package
if jsondiff.diff(gold_spec, r_spec) != {}:
print('{} spec is different from R version'.format(spec_name), file=sys.stderr)
print(jsondiff.diff(gold_spec, r_spec), file=sys.stderr)
sys.exit(1)
def get(self):
"""This get() calls the other functions for some reason(tm)."""
bucket_name = os.environ.get(
"BUCKET_NAME", app_identity.get_default_gcs_bucket_name()
)
self.response.headers["Content-Type"] = "text/plain"
self.response.write(
"Demo GCS Application running from Version: "
+ os.environ["CURRENT_VERSION_ID"]
+ "\n"
)
self.response.write("Using bucket name: " + bucket_name + "\n\n")
version = os.environ["CURRENT_VERSION_ID"].split(".")[0]
bucket = "/" + bucket_name
if version == "None":
filename = bucket + "/schedule"
updated_filename = bucket + "/updated_schedule"
else:
filename = bucket + "/schedule_" + version
updated_filename = bucket + "/updated_schedule_" + version
# This _etag is currently unused, could be used to reduce writes to
# only when the schedule is updated (and to notify of updates)
try:
filename_etag = self.stat_file(filename).etag
if DEBUG:
self.response.write("etag: %s \n" % filename_etag)
except gcs.NotFoundError:
if DEBUG:
print("P1")
[totalgames, jsondata] = self.fetch_upstream_schedule(URL)
if totalgames == 0:
pass
else:
self.response.write("Total Games: %s\n" % totalgames)
[teamdates] = self.parse_schedule(jsondata)
content = self.make_data_json(teamdates)
try:
old_content = self.read_file(filename)
except gcs.NotFoundError:
self.create_file(filename, content)
old_content = self.read_file(filename)
if old_content == content:
try:
last_updated = self.read_file(updated_filename)
self.response.write("Not updating schedule - it is current.\n")
except gcs.NotFoundError:
self.create_file(updated_filename, FOR_UPDATED)
last_updated = self.read_file(updated_filename)
self.response.write("Last updated: %s\n" % last_updated)
else:
print(
"Changes: %s" % (diff(json.loads(old_content), json.loads(content)))
)
self.response.write(
"Diff: %s" % diff(json.loads(old_content), json.loads(content))
)
self.create_file(filename, content)
self.create_file(updated_filename, FOR_UPDATED)
# Only send e-mails outside playoffs
# (potential spoilers - games are removed from the schedule)
if CURRENT_MONTH < 4 or CURRENT_MONTH > 6:
self.send_an_email(
diff(json.loads(old_content), json.loads(content)), True
)
self.response.set_status(202)
def test_a(self):
self.assertEqual({}, diff(1, 1))
self.assertEqual({}, diff(True, True))
self.assertEqual({}, diff('abc', 'abc'))
self.assertEqual({}, diff([1, 2], [1, 2]))
self.assertEqual({}, diff((1, 2), (1, 2)))
self.assertEqual({}, diff({1, 2}, {1, 2}))
self.assertEqual({}, diff({'a': 1, 'b': 2}, {'a': 1, 'b': 2}))
self.assertEqual({}, diff([], []))
self.assertEqual({}, diff(None, None))
self.assertEqual({}, diff({}, {}))
self.assertEqual({}, diff(set(), set()))
self.assertEqual(2, diff(1, 2))
self.assertEqual(False, diff(True, False))
self.assertEqual('def', diff('abc', 'def'))
self.assertEqual([3, 4], diff([1, 2], [3, 4]))
self.assertEqual((3, 4), diff((1, 2), (3, 4)))
self.assertEqual({3, 4}, diff({1, 2}, {3, 4}))
self.assertEqual({replace: {'c': 3, 'd': 4}}, diff({'a': 1, 'b': 2}, {'c': 3, 'd': 4}))
self.assertEqual({replace: {'c': 3, 'd': 4}}, diff([1, 2], {'c': 3, 'd': 4}))
self.assertEqual(123, diff({'a': 1, 'b': 2}, 123))
self.assertEqual({delete: ['b']}, diff({'a': 1, 'b': 2}, {'a': 1}))
self.assertEqual({'b': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'b': 3}))
self.assertEqual({'c': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'b': 2, 'c': 3}))
self.assertEqual({delete: ['b'], 'c': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'c': 3}))
self.assertEqual({add: {3}}, diff({1, 2}, {1, 2, 3}))
self.assertEqual({add: {3}, discard: {4}}, diff({1, 2, 4}, {1, 2, 3}))
self.assertEqual({discard: {4}}, diff({1, 2, 4}, {1, 2}))
self.assertEqual({insert: [(1, 'b')]}, diff(['a', 'c'], ['a', 'b', 'c']))
self.assertEqual({insert: [(1, 'b')], delete: [3, 0]}, diff(['x', 'a', 'c', 'x'], ['a', 'b', 'c']))
self.assertEqual(
{insert: [(2, 'b')], delete: [4, 0], 1: {'v': 20}},
diff(['x', 'a', {'v': 11}, 'c', 'x'], ['a', {'v': 20}, 'b', 'c'])
)
self.assertEqual(
{insert: [(2, 'b')], delete: [4, 0], 1: {'v': 20}},
diff(['x', 'a', {'u': 10, 'v': 11}, 'c', 'x'], ['a', {'u': 10, 'v': 20}, 'b', 'c'])
)
def commit(snoRoot, sessionID, DryRun=False):
#with open("ConfigDB", "r") as f:
# originalsno_dict = json.loads(f.read())
get_lock = ConfigDB.acquire_lock()
if 'Error' in get_lock.keys():
return get_lock
else:
originalsnoRoot = get_lock['ConfigDB']
if DryRun:
'''
original_xml = (pybindIETFXMLEncoder.serialise(originalsnoRoot))
new_xml = (pybindIETFXMLEncoder.serialise(snoRoot))
diff_xml = XMLDiff(original_xml, new_xml)
'''
diff_xml = calculate_diff(originalsnoRoot, snoRoot)
ConfigDB.release_lock()
return diff_xml
sno_dict = json.loads(pybindJSON.dumps(snoRoot))
originalsno_dict = json.loads(pybindJSON.dumps(originalsnoRoot))
difference = diff(originalsno_dict, sno_dict)
#print ((originalsnoRoot.get()))
device_config = {}
if 'devices' in difference.keys(
) and 'device' in difference['devices'].keys():
for device in difference['devices']['device'].keys():
if 'config' in difference['devices']['device'][device]:
config_dict = difference['devices']['device'][device]['config']
#print ((originalsnoRoot.devices.device[device].config.get()))
#print (config_dict)
if device not in originalsnoRoot.devices.device.keys():
return {
"Error":
"Device {} not present in DB. First add the device".
format(device)
}
original_config = originalsnoRoot.devices.device[device].config
new_config = snoRoot.devices.device[device].config
'''
print (original_config)
print (new_config)
XMLDiff1 = XMLDiff(original_config, new_config)
print (XMLDiff1+"\n")
configXML_tree = list( etree.fromstring(XMLDiff1) )[0]
configXML = etree.tostring(configXML_tree).decode()
configXML = "<config>\n" + configXML + "</config>"
'''
XMLDIFF = calculate_diff(
original_config,
new_config,
snoObject=originalsnoRoot.devices.device[device].config)
configXML_tree = list(etree.fromstring(XMLDIFF))[0]
configXML = etree.tostring(configXML_tree).decode()
configXML = "<config>\n" + configXML + "</config>"
print(configXML)
rev_XMLDIFF = calculate_diff(
new_config,
original_config,
snoObject=originalsnoRoot.devices.device[device].config)
rev_XML_tree = list(etree.fromstring(rev_XMLDIFF))[0]
rev_XML = etree.tostring(rev_XML_tree).decode()
rev_XML = "<config>\n" + rev_XML + "</config>"
device_config[device] = {}
device_config[device]['config'] = configXML
device_config[device]['rev_config'] = rev_XML
if device_config:
try:
NetworkTransaction(device_config, originalsnoRoot)
except Exception as e:
return ({"Error": "Failed to commit. " + str(e)})
return ({"ConfigDB": ConfigDB.write(snoRoot, sessionID)})
def get_diff(expected, result):
return diff(json.loads(expected), json.loads(result))
def process_bounty_changes(old_bounty, new_bounty):
"""Process Bounty changes.
Args:
old_bounty (dashboard.models.Bounty): The old Bounty object.
new_bounty (dashboard.models.Bounty): The new Bounty object.
"""
from dashboard.utils import build_profile_pairs
profile_pairs = None
# check for maintainer blocks
is_blocked = any([(ele.lower() in new_bounty.github_url.lower())
for ele in BlockedURLFilter.objects.values_list(
'expression', flat=True)])
if is_blocked:
raise UnsupportedRepoException(
"This repo is not bountyable at the request of the maintainer.")
# process bounty sync requests
did_bsr = False
for bsr in BountySyncRequest.objects.filter(
processed=False, github_url=new_bounty.github_url).nocache():
did_bsr = True
bsr.processed = True
bsr.save()
# get json diff
json_diff = diff(old_bounty.raw_data, new_bounty.raw_data) if (
old_bounty and new_bounty) else None
# new bounty
if not old_bounty or (not old_bounty and new_bounty
and new_bounty.is_open) or (not old_bounty.is_open
and new_bounty
and new_bounty.is_open):
is_greater_than_x_days_old = new_bounty.web3_created < (
timezone.now() - timezone.timedelta(hours=24))
if is_greater_than_x_days_old and not settings.IS_DEBUG_ENV:
msg = f"attempting to create a new bounty ({new_bounty.standard_bounties_id}) when is_greater_than_x_days_old = True"
print(msg)
raise Exception(msg)
event_name = 'new_bounty'
elif old_bounty.num_fulfillments < new_bounty.num_fulfillments:
event_name = 'work_submitted'
elif old_bounty.is_open and not new_bounty.is_open:
if new_bounty.status in ['cancelled', 'expired']:
event_name = 'killed_bounty'
else:
event_name = 'work_done'
elif old_bounty.value_in_token < new_bounty.value_in_token:
event_name = 'increased_bounty'
else:
event_name = 'unknown_event'
logger.info(
f'got an unknown event from bounty {old_bounty.pk} => {new_bounty.pk}: {json_diff}'
)
print(f"- {event_name} event; diff => {json_diff}")
# record a useraction for this
record_user_action(event_name, old_bounty, new_bounty)
record_bounty_activity(event_name, old_bounty, new_bounty)
# Build profile pairs list
if new_bounty.fulfillments.exists():
profile_pairs = build_profile_pairs(new_bounty)
# Send an Email if this is a LowBall bounty
try:
if (not old_bounty
or old_bounty.value_in_usdt != new_bounty.value_in_usdt):
if is_lowball_bounty(new_bounty.value_in_usdt
) and new_bounty.network == 'mainnet':
notify_of_lowball_bounty(new_bounty)
except Exception as e:
logger.error(f'{e} during check for Lowball Bounty')
# marketing
if event_name != 'unknown_event':
print("============ posting ==============")
did_post_to_slack = maybe_market_to_slack(new_bounty, event_name)
did_post_to_user_slack = maybe_market_to_user_slack(
new_bounty, event_name)
did_post_to_github = maybe_market_to_github(new_bounty, event_name,
profile_pairs)
did_post_to_email = maybe_market_to_email(new_bounty, event_name)
print("============ done posting ==============")
# what happened
what_happened = {
'did_bsr': did_bsr,
'did_post_to_email': did_post_to_email,
'did_post_to_github': did_post_to_github,
'did_post_to_slack': did_post_to_slack,
'did_post_to_user_slack': did_post_to_user_slack,
'did_post_to_twitter': False,
}
print("changes processed: ")
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(what_happened)
else:
print('No notifications sent - Event Type Unknown = did_bsr: ',
did_bsr)
def test_replace_all_classes():
current = {
"azure.ai.formrecognizer": {
"class_nodes": {}
}
}
stable = {
"azure.ai.formrecognizer": {
"class_nodes": {
"class_name": {
"methods": {
"one": {
"parameters": {
"testing": {
"default": None,
"param_type": "positional_or_keyword"
}
},
"is_async": True
},
"two": {
"parameters": {
"testing2": {
"default": None,
"param_type": "positional_or_keyword"
}
},
"is_async": True
},
}
},
"class_name2": {
"methods": {
"one": {
"parameters": {
"testing": {
"default": None,
"param_type": "positional_or_keyword"
}
},
"is_async": True
},
"two": {
"parameters": {
"testing2": {
"default": None,
"param_type": "positional_or_keyword"
}
},
"is_async": True
},
}
},
}
}
}
EXPECTED = [
"(RemovedOrRenamedClass): The model or publicly exposed class 'azure.ai.formrecognizer.class_name' was deleted or renamed in the current version",
"(RemovedOrRenamedClass): The model or publicly exposed class 'azure.ai.formrecognizer.class_name2' was deleted or renamed in the current version"
]
diff = jsondiff.diff(stable, current)
bc = BreakingChangesTracker(stable, current, diff, "azure-storage-queue")
bc.run_checks()
assert len(bc.breaking_changes) == len(EXPECTED)
for message in bc.breaking_changes:
assert message in EXPECTED
def diff(self, kind, inmodel, dbmodel, hide_defaults=True):
"""Create a diff between input and existing model.
Args:
kind (str): kind of object to diff.
inmodel (model): Description
dbmodel (model): Description
hide_defaults (bool, optional):
hide values that are defaulted into db
Returns:
dict: Diff
"""
# Check for empty inputs
if inmodel is None:
return None
if dbmodel is None:
dbmodel = {}
else:
# Transform django object to dictionary.
dbmodel = dbmodel.__dict__
context = {'section', 'course', 'semester', 'textbook'}
whats = {}
for k, v in inmodel.iteritems():
if k not in context:
continue
try:
whats[k] = str(v)
except (django.utils.encoding.DjangoUnicodeDecodeError,
UnicodeEncodeError):
whats[k] = '<{}: [Bad Unicode data]'.format(k)
# Remove db specific content from model.
blacklist = context | {
'_state',
'id',
'section_id',
'course_id',
'_course_cache',
'semester_id',
'_semester',
'vector',
}
def prune(d):
return {k: v for k, v in d.iteritems() if k not in blacklist}
dbmodel = prune(dbmodel)
inmodel = prune(inmodel)
if 'course' in dbmodel:
dbmodel['course'] = str(dbmodel['course'])
# Remove null values from dictionaries.
dbmodel = {k: v for k, v in dbmodel.iteritems() if v is not None}
# Move contents of default dictionary to first-level of dictionary.
if 'defaults' in inmodel:
defaults = inmodel['defaults']
del inmodel['defaults']
inmodel.update(defaults)
# Diff the in-model and db-model
diffed = json.loads(
jsondiff.diff(dbmodel, inmodel, syntax='symmetric', dump=True))
# Remove db defaulted values from diff output.
if hide_defaults and '$delete' in diffed:
self.remove_defaulted_keys(kind, diffed['$delete'])
if len(diffed['$delete']) == 0:
del diffed['$delete']
# Add `what` and `context` tag to diff output.
if len(diffed) > 0:
if isinstance(diffed, list) and len(diffed[0]) == 0:
diffed = {'$new': diffed[1]}
elif isinstance(diffed, dict):
diffed.update({'$what': inmodel})
diffed.update({'$context': whats})
self.json_streamer.write(diffed)
return diffed
#!/usr/bin/env python
import json
from jsondiff import diff
with open('hw.json') as f:
json_data = f.read()
json_prev = json.loads(json_data)
with open('hw_new.json') as f:
json_data = f.read()
json_new = json.loads('hw_new.json')
diff(json_prev, json_new)
def diff_json(a, b):
a_json = json.loads(a.encode("utf-8"))
b_json = json.loads(b.encode("utf-8"))
raw_diff = diff(a_json, b_json, syntax='symmetric')
good_diff = {}
for element in raw_diff:
if str(element) == "$insert":
for block in raw_diff[element]:
good_diff[block[0]] = ["",block[1]]
elif str(element) == "$delete":
for block in raw_diff[element]:
good_diff[block[0]] = [block[1],""]
else:
block_diff = raw_diff[element]['value']
old = {}
new = {}
old_dict = {}
new_dict = {}
try:
for value in block_diff:
old[value] = block_diff[value][0]
new[value] = block_diff[value][1]
old_dict['type'] = b_json[element]['type']
new_dict['type'] = b_json[element]['type']
old_dict['value'] = old
new_dict['value'] = new
good_diff[element] = [old_dict, new_dict]
except TypeError:
old_dict['type'] = "Changed Blocktype"
new_dict['type'] = "Changed Blocktype"
old = {}
new = {}
old_str = ""
for (key, val) in block_diff[0].items():
if isinstance(val, str):
old_str += key + ": " + val.encode('ascii', 'xmlcharrefreplace') + "\n"
elif isinstance(val, int):
old_str += key + ": " + str(val).encode('ascii', 'xmlcharrefreplace') + "\n"
else:
old_str += key + ": Empty\n"
old[raw_diff[element]['type'][0]] = old_str
new_str = ""
for (key, val) in block_diff[1].items():
if isinstance(val, str):
new_str += key + ": " + val.encode('ascii', 'xmlcharrefreplace') + "\n"
elif isinstance(val, int):
new_str += key + ": " + str(val).encode('ascii', 'xmlcharrefreplace') + "\n"
else:
new_str += key + ": Empty\n"
new[raw_diff[element]['type'][1]] = new_str
old_dict['value'] = old
new_dict['value'] = new
good_diff[element] = [old_dict, new_dict]
print good_diff[element]
except:
if 'type' in raw_diff[element]:
old_dict['type'] = raw_diff[element]['type']
new_dict['type'] = raw_diff[element]['type']
old = {}
new = {}
old['Complicated Changes'] = block_diff
new['Complicated Changes'] = block_diff
old_dict['value'] = old
new_dict['value'] = new
good_diff[element] = [old_dict, new_dict]
return good_diff
def uploadLogoFile():
# This is the path to the upload directory
duset_name = ""
# These are the extension that we are accepting to be uploaded
ALLOWED_EXTENSIONS = ['png', 'jpg', 'jpeg', 'gif', 'bmp']
re.compile("active", re.IGNORECASE)
# Get the name of the uploaded file
file = request.files.get('logo')
duset_id = request.form.get('duset_id')
if duset_id is None:
raise Exception("duset_id not found in request")
if deploymentUnitSetDB.GetDeploymentUnitSetById(duset_id, False) is None:
raise Exception("No such DU SET exists")
filename = None
if file is not None:
filename = ('.' in file.filename and str(
file.filename.rsplit('.', 1)[1]).lower() in ALLOWED_EXTENSIONS)
else:
raise Exception("File not found in request.")
if filename not in [True]:
raise Exception(
"Invalid file .Please select file 'png', 'jpg', 'jpeg', 'gif'")
# Check if the file is one of the allowed types/extensions
if file and filename:
# Make the filename safe, remove unsupported chars
filename = secure_filename(file.filename)
file_extension = filename.split(".")[-1]
result1 = deploymentUnitSetDB.GetDeploymentUnitSetById(duset_id, False)
duset_name = '_'.join(result1["name"].split())
file_path = str(logo_full_path + '/' + duset_name + '_' + duset_id +
'.' + file_extension)
thumbnail_file_path = (str(logo_full_path + '/' + duset_name + '_' +
duset_id + '_thumbnail.' + file_extension))
current_logo_file = str(logo_path + '/' + duset_name + '_' + duset_id +
'.' + file_extension)
current_thumbnail_file = str(logo_path + '/' + duset_name + '_' +
duset_id + '_thumbnail.' + file_extension)
db_logo_file = result1.get("logo")
# db_thumbnail_file=result1.get("thumbnail_logo")
file.save(file_path)
if not os.path.isfile(file_path):
raise Exception("Failed to save logo to path :" + file_path)
if FileUtils.thumbnail(file_path, thumbnail_file_path) is None:
current_thumbnail_file = current_logo_file
if not os.path.isfile(thumbnail_file_path):
current_thumbnail_file = current_logo_file
if db_logo_file is None:
# updated=tooldb.add_tool_logo(current_logo_file,current_thumbnail_file,duset_id)
updated = deploymentUnitSetDB.add_du_set_logo(
current_logo_file, current_thumbnail_file, duset_id)
else:
sub = diff(db_logo_file, current_logo_file)
if sub:
print "Du Set API:compare: logo " + db_logo_file + " is required to be updated as the logo data has changed."
# updated=tooldb.add_tool_logo(current_logo_file,current_thumbnail_file,duset_id)
updated = deploymentUnitSetDB.add_du_set_logo(
current_logo_file, current_thumbnail_file, duset_id)
return jsonify(
json.loads(
dumps({
"result":
"success",
"message":
"File and Deployment Unit Set was updated successfully and is readable"
}))), 200
def run_compare(module, count, test):
class CustomFilter:
def filter_jmespath(self, test, legal_json_diff):
plugins_filter = {
"ntp": "peers.{delete: delete, insert: insert}",
"vlan": "vlans.{delete: delete, insert: insert}",
"as_path": "activeIpAsPathLists.{delete: delete, insert: insert}",
"lldp": "lldpNeighbors.{delete: delete, insert: insert}",
}
if plugins_filter.get(test):
final_diff = search(plugins_filter.get(test), legal_json_diff)
elif test == "interface":
final_diff = CustomFilter().filter_iface_counters(
legal_json_diff
)
elif test == "acl":
final_diff = CustomFilter().filter_acls_counters(
legal_json_diff
)
else:
final_diff = legal_json_diff
return final_diff
def filter_iface_counters(self, legal_json_diff):
return_dict = {"interfaces": {}}
for ifaces in legal_json_diff.values():
for iface_name, iface_values in ifaces.items():
if iface_values.get("interfaceStatus"):
return_dict["interfaces"][
iface_name
] = legal_json_diff["interfaces"][iface_name][
"interfaceStatus"
]
if iface_values.get("memberInterfaces"):
if iface_values["memberInterfaces"].get(
"delete"
) or iface_values["memberInterfaces"].get("insert"):
return_dict["interfaces"][
iface_name
] = legal_json_diff["interfaces"][iface_name][
"memberInterfaces"
]
if iface_name == "insert" or iface_name == "delete":
return_dict["interfaces"][
iface_name
] = legal_json_diff["interfaces"][iface_name]
return return_dict
def filter_acls_counters(self, legal_json_diff):
for acls in legal_json_diff.values():
for acl_change in acls.keys():
if acl_change != 'insert' or acl_change != 'delete' or acl_change is not None:
del legal_json_diff['aclList'][acl_change]
return legal_json_diff
def replace(string, test):
substitutions = {
"'": '"',
"insert": '"insert"',
"delete": '"delete"',
"True": "true",
"False": "false",
"(": "[",
")": "]",
" u'": "\"",
"{u'": "{\"",
"[u'": "[\"",
" u\"": "\"",
"{u\"": "{\"",
"[u\"": "[\"",
}
skip_list = ["vrf"]
substrings = sorted(substitutions, key=len, reverse=True)
regex = re.compile("|".join(map(re.escape, substrings)))
sub_applied = regex.sub(
lambda match: substitutions[match.group(0)], string
)
if test not in skip_list:
# ('{', '28: ') (' ', '187: ')
for integer in re.findall(r"(\s|{)(\d+:\s)", sub_applied):
int_replacement = integer[1][:-2]
merged_integer = integer[0] + integer[1]
if integer[0] == '{':
sub_applied = sub_applied.replace(
# double {{ required by format to exscape {
merged_integer, '{{"{0}": '.format(int_replacement)
)
else:
sub_applied = sub_applied.replace(
merged_integer, ' "{0}": '.format(int_replacement)
)
return sub_applied
before_file = module.params.get("files")[0]
after_file = module.params.get("files")[1]
filter_flag = module.params.get("filter")
host = module.params.get("hostname")
root_path = module.params.get("root_path")
if len(before_file) == len(after_file):
try:
before = open(
"{root_path}/tests/before/{test}/{host}/{before_file}.json".format(
root_path=root_path,
test=test,
host=host,
before_file=str(before_file[count]),
),
"r",
)
except FileNotFoundError as error:
module.fail_json(msg=error)
try:
after = open(
"{root_path}/tests/after/{test}/{host}/{after_file}.json".format(
root_path=root_path,
test=test,
host=host,
after_file=str(after_file[count]),
),
"r",
)
except FileNotFoundError as error:
module.fail_json(msg=error)
destination = "{root_path}/tests/diff/{test}/{host}/".format(
root_path=root_path,
test=test,
host=host,
)
if not os.path.exists(destination):
os.makedirs(destination)
json_diff = str(diff(before, after, load=True, syntax="symmetric"))
legal_json_diff = replace(json_diff, test)
try:
if not filter_flag:
final_diff = json.loads(legal_json_diff)
if filter_flag:
final_diff = CustomFilter().filter_jmespath(
test, json.loads(legal_json_diff)
)
except ValueError as error:
module.fail_json(msg="Diff file not legal:\n{0}".format(legal_json_diff))
diff_file_id = str(round(time.time())) + '_' + str(
(int(before_file[count]) - int(after_file[count])) * -1
)
try:
with open("{0}{1}.json".format(destination, diff_file_id), "w") as file:
json.dump(final_diff, file, ensure_ascii=False, indent=4)
except IOError:
module.fail_json(msg="Something went wrong when writing to the file")
return final_diff
def get_json_value(self, element_name, task_dfn, expected_val):
self.assertFalse(
diff(json.loads(run(element_name, task_dfn)),
json.loads(expected_val)))
def processDataset(self, dataset, resources):
if not dataset:
raise Exception(
_("Fehler: Kein Datensatz zum Erstellen in datasetuploader.processDataset()"
))
if self.limit:
logging.debug(_("Datensatz: %s/%s"), self.dataset_count,
self.limit)
if self.dataset_count >= self.limit:
logging.info(
_("Datensatz wird übersprungen. Limit von %s erreicht."),
self.limit)
return None
logging.info(_("Bearbeite Datensatz: '%s'"), dataset)
logging.debug(_("Resourcen: %s"), resources)
node_id = dataset.getValue(Dataset.NODE_ID)
if node_id:
# update existing dataset by node_id
node_id = self.updateDataset(dataset)
elif dataset.getValue(Dataset.DATASET_ID):
# update by package_id
package_id = dataset.getValue(Dataset.DATASET_ID)
if not node_id:
remote_url = config.x_api_find_node_id.format(package_id)
r = requests.get(remote_url)
node_search = json.load(r.text)
if node_search[0]['nid']:
node_id = node_search[0]['nid']
dataset.set(Dataset.NODE_ID, node_id)
node_id = self.updateDataset(dataset)
else:
logging.error(
_("Datensatz mit der Package-ID '%s' wurde nicht gefunden"
), package_id)
else:
# create new dataset
self.dataset_count += 1
node_id = dkanhandler.create(dataset)
logging.debug(_("NEUE Dataset-ID: %s"), node_id)
if not node_id:
raise Exception(
_("Fehler beim Erstellen oder beim Update des Datensatzes"))
elif node_id == '-':
return None
raw_dataset = dkanhandler.getDatasetDetails(node_id)
# Summary with changes
raw_dataset2 = dkanhelpers.HttpHelper.read_dkan_node(node_id)
logging.debug(_(" == Datensatz-Änderung: == "))
logging.debug(diff(raw_dataset, raw_dataset2))
# add or update resources
if not self._ignore_resources:
self.processResources(raw_dataset, resources)
return node_id
def test_replace_all_params():
current = {
"azure.ai.formrecognizer": {
"function_nodes": {
"my_function_name": {
"parameters": {},
"is_async": False
}
},
"class_nodes": {
"class_name": {
"methods": {
"one": {
"parameters": {},
"is_async": True
},
"two": {
"parameters": {},
"is_async": True
},
}
}
}
}
}
stable = {
"azure.ai.formrecognizer": {
"function_nodes": {
"my_function_name": {
"is_async": False,
"parameters": {
"testing": {
"default": None,
"param_type": "positional_or_keyword"
},
"testing2": {
"default": None,
"param_type": "positional_or_keyword"
}
}
}
},
"class_nodes": {
"class_name": {
"methods": {
"one": {
"parameters": {
"testing": {
"default": None,
"param_type": "positional_or_keyword"
}
},
"is_async": True
},
"two": {
"parameters": {
"testing2": {
"default": None,
"param_type": "positional_or_keyword"
}
},
"is_async": True
},
}
}
}
}
}
EXPECTED = [
"(RemovedOrRenamedPositionalParam): The 'azure.ai.formrecognizer.class_name method 'one' had its 'positional_or_keyword' parameter 'testing' deleted or renamed in the current version",
"(RemovedOrRenamedPositionalParam): The 'azure.ai.formrecognizer.class_name method 'two' had its 'positional_or_keyword' parameter 'testing2' deleted or renamed in the current version",
"(RemovedOrRenamedPositionalParam): The function 'azure.ai.formrecognizer.my_function_name' had its 'positional_or_keyword' parameter 'testing' deleted or renamed in the current version",
"(RemovedOrRenamedPositionalParam): The function 'azure.ai.formrecognizer.my_function_name' had its 'positional_or_keyword' parameter 'testing2' deleted or renamed in the current version"
]
diff = jsondiff.diff(stable, current)
bc = BreakingChangesTracker(stable, current, diff, "azure-storage-queue")
bc.run_checks()
assert len(bc.breaking_changes) == len(EXPECTED)
for message in bc.breaking_changes:
assert message in EXPECTED
def test_dump(self, scenario):
a, b = scenario
diff(a, b, syntax='compact', dump=True)
diff(a, b, syntax='explicit', dump=True)
diff(a, b, syntax='symmetric', dump=True)
def json_deep_equals(expected, actual):
result = jsondiff.diff(expected, actual)
return result == {}
with open(v + '/' + v, "w") as f:
json.dump(json_data, f, indent=1)
# Cleanup
for file in glob(v + '/' + '*.json'):
remove(file)
if path.exists(v + '/' + v):
rename(v + '/' + v, v + '/' + v + '.json')
# Compare
print("Comparing")
with open(v + '/' + 'old_' + v, 'r') as o, open(v + '/' + v + '.json',
'r') as n:
old = json.load(o)
new = json.load(n)
c = diff(old, new)
if c:
print("New changes found!")
with open(v + '/' + 'changes', "w") as f:
json.dump(c, f, indent=1)
changes = True
else:
print("No changes found!")
if path.exists(v + '/' + 'changes'):
remove(v + '/' + 'changes')
changes = False
# commit, push and send
if changes is True:
# Notify
rom = str(v.replace('_', ' '))
def get_changes(previous_id, revision_id):
changes = []
revision = PageRevision.objects.get(pk=revision_id)
page_2 = revision.content_json
prev_revision = PageRevision.objects.get(pk=previous_id)
page_1 = prev_revision.content_json
p1_dict = json.loads(page_1)['body']
p2_dict = json.loads(page_2)['body']
a_json = json.loads(p1_dict.encode("utf-8"))
b_json = json.loads(p2_dict.encode("utf-8"))
raw_diff = diff(a_json, b_json, syntax='symmetric')
for element in raw_diff:
if str(element) == "$insert":
fields = []
for block in raw_diff[element]:
fields.append({
"value": "#" + str(block[0]) + ": " + block[1]['type'],
"short": True
})
changes.append(
{
"title": "Added Blocks",
"fields": fields,
"color": "good"
}
)
elif str(element) == "$delete":
fields = []
for block in raw_diff[element]:
fields.append({
"value": "#" + str(block[0]) + ": " + block[1]['type'],
"short": True
})
changes.append(
{
"title": "Removed Blocks",
"fields": fields,
"color": "danger"
}
)
else:
block_diff = raw_diff[element]['value']
fields = []
try:
for value in block_diff:
fields.append(
{
"value": str(value),
"short": True
})
block_type = str(b_json[element]['type'])
except:
block_type = str(raw_diff[element]['type'][0])+" to: "+str(raw_diff[element]['type'][1])
fields.append(
{
"value": "Complicated Changes: Details in Link"
})
changes.append(
{
"title": "Changed Block #" + str(element) + ": " + block_type,
"fields": fields,
"color": "warning"
}
)
if not changes:
changes.append(
{
"title": "No fields were changed",
"color": "good"
})
return changes
