def update(self, body):
# Use JSONPath to extract out what we need
passphrase = parse('config.ssh.passphrase').find(body)
public_key = parse('config.ssh.publicKey').find(body)
if passphrase:
ssh = self.cluster['config'].setdefault('ssh', {})
ssh['passphrase'] = passphrase[0].value
if public_key:
public_key = public_key[0].value
if not _validate_key(public_key):
raise RestException('Invalid key format', 400)
ssh = self.cluster['config'].setdefault('ssh', {})
ssh['publicKey'] = public_key
self.cluster = self.model('cluster', 'cumulus').save(self.cluster)
# Don't return the access object
del self.cluster['access']
# Don't return the log
del self.cluster['log']
# Don't return the passphrase
if parse('config.ssh.passphrase').find(self.cluster):
del self.cluster['config']['ssh']['passphrase']
return self.cluster
def _process_experimental(self, doc):
facility_used = parse('experiment.experimentalEnvironment.facilityUsed').find(doc)[0].value
experiments = parse('experiment.experiments').find(doc)[0].value
experiment_model = self.model('experimental', 'molecules')
experiments_list = []
for experiment in experiments:
spectrum_type = experiment['spectrumType']
experimental_technique = experiment['experimentalTechnique']
id = experiment['id']
molecular_formula = experiment['molecularFormula']
instenisty_units = parse('measuredSpectrum.unitsY').find(experiment)[0].value
frequency_units = parse('measuredSpectrum.unitsX').find(experiment)[0].value
data_points = parse('measuredSpectrum.dataPoints').find(experiment)[0].value
frequencies = data_points[::2]
intensities = data_points[1::2]
measured_spectrum = {
'frequencies': {
'units': frequency_units,
'values': frequencies
},
'intensities': {
'units': instenisty_units,
'values': intensities
}
}
experiments_list.append(experiment_model.create(
facility_used, spectrum_type, experimental_technique, id,
molecular_formula, measured_spectrum))
return experiments_list
def extract_phones_urls_from_all_files(outer_folder=
'/Users/mayankkejriwal/datasets/memex/user-worker-dig3-full-phase4-2017/'):
"""
Will write out strict phones in outer_folder+'strict_phones.txt' and relaxed_phones in outer_folder+'relaxed_phones.txt'
Note that while we do 'set' checking for each individual json there can be many repeated phones across the file.
Each line will be tab-delimited and represent the phones per json.
:param outer_folder:
:return:
"""
jpath_relaxed = parse('fields.phone.relaxed[*].name')
jpath_strict = parse('fields.phone.strict[*].name')
listOfFiles = glob.glob(outer_folder + 'gz/*.txt.gz')
out_strict = codecs.open(outer_folder+'strict_phones_urls.txt', 'w', 'utf-8')
out_relaxed = codecs.open(outer_folder + 'relaxed_phones_urls.txt', 'w', 'utf-8')
for fi in listOfFiles:
print 'processing file: ',fi
with gzip.open(fi, 'rb') as f:
for line in f:
j = json.loads(re.split('\t', line[0:-1])[1])
strict_phones = list(set([match.value for match in jpath_strict.find(j)]))
if len(strict_phones) != 0:
strict_phones = '\t'.join(strict_phones)
out_strict.write(j['url']+'\t'+strict_phones)
out_strict.write('\n')
relaxed_phones = list(set([match.value for match in jpath_relaxed.find(j)]))
if len(relaxed_phones) != 0:
relaxed_phones = '\t'.join(relaxed_phones)
out_relaxed.write(j['url']+'\t'+relaxed_phones)
out_relaxed.write('\n')
out_strict.close()
out_relaxed.close()
def process_class(class_name, args, listPath=None):
"process on one class"
args1 = list(args)
while True:
response_json = call_cli(args1)
if (listPath):
jsonpath_expr = parse(listPath[0])
node_list = jsonpath_expr.find(response_json)
for node in node_list:
response_json = node.value
if (len(listPath )> 1 and len(response_json) > 0):
jsonpath_expr = parse(listPath[1])
for first_node_list in response_json:
node_list2 = jsonpath_expr.find(first_node_list)
for node2 in node_list2:
process_object(class_name, node2.value)
else:
process_object(class_name, node.value)
else:
process_object(class_name, response_json)
if ('NextToken' in response_json):
args1 = list(args)
args1.append("--starting-token")
args1.append(u2s(response_json['NextToken']))
else:
break
def get_primary_score(exp):
primary = parse('$.details[0].value')
primary_with_phrase = parse('$.details[0].details[0].details[0].value')
if has_phrase_rescore(exp):
return get_score_from_path(exp, primary_with_phrase)
else:
return get_score_from_path(exp, primary)
def cross_references(self):
"""
Extract all source and destination cross references i.e.
parameters with on 'SourceEnd' and 'TargetEnd' parameters
:return: pd.DataFrame
"""
tag_dependency_list_query = 'DependencyList.DependencyListObj.Category.Dependency[*]'
tag_dependency_json_list = parse(tag_dependency_list_query).find(self._cm_json)
tag_xref_list_query = '*..Attribute[*]'
source_end_list = []
target_end_list = []
for dependent_json in tag_dependency_json_list:
for param in parse(tag_xref_list_query).find(dependent_json):
v = param.value.values()
if 'SourceEnd' in v:
source_end_list.append(v[1])
continue
if 'TargetEnd' in v:
target_end_list.append(v[1])
continue
continue
df = pd.DataFrame({'SourceEnd': source_end_list, 'TargetEnd': target_end_list})
df['SourceTagname'] = df.SourceEnd.apply(lambda source_end: source_end.split('.')[0])
df['TargetTagname'] = df.TargetEnd.apply(lambda target_end: target_end.split('.')[0])
df.drop_duplicates(inplace=True)
return self._drop_empty_columns(dataframe=df)
def _update(url, request):
request_body = json.loads(request.body)
passphrase = parse('config.ssh.passphrase').find(request_body)
public_key = parse('config.ssh.publicKey').find(request_body)
status = request_body['status'] == 'created'
self._update = passphrase and public_key and status
return httmock.response(200, None, {}, request=request)
def validate_args(kwargs):
required = ['cluster.config.paraview.installDir', 'sessionKey']
for r in required:
if not parse(r).find(kwargs):
raise Exception('Required parameter %s not provide to taskflow.'
% r)
if not parse('dataDir') and not parse('input.file.id'):
raise Exception('\'dataDir\' or \'input.file.id\' must be provided.')
def submit_paraview_job(task, cluster, job, *args, **kwargs):
task.taskflow.logger.info('Submitting job to cluster.')
girder_token = task.taskflow.girder_token
params = {}
if 'dataDir' in kwargs:
params['dataDir'] = kwargs['dataDir']
if 'fileName' in kwargs:
params['fileName'] = kwargs['fileName']
if 'simulationJobId' in kwargs:
params['simulationJobId'] = kwargs['simulationJobId']
if 'sessionKey' in kwargs:
params['sessionKey'] = kwargs['sessionKey']
# Save the sessionKey so we can clean up the proxy entry
task.taskflow.set_metadata('sessionKey', kwargs['sessionKey'])
parallel_environment \
= parse('config.parallelEnvironment').find(cluster)
if parallel_environment:
parallel_environment = parallel_environment[0].value
params['parallelEnvironment'] = parallel_environment
params['numberOfSlots'] = 1
job_output_dir = get_cluster_job_output_dir(cluster)
if job_output_dir:
params['jobOutputDir'] = job_output_dir
paraview_install_dir \
= parse('config.paraview.installDir').find(cluster)
if paraview_install_dir:
paraview_install_dir = paraview_install_dir[0].value
params['paraviewInstallDir'] = paraview_install_dir
# Does the cluster have GPUs?
params['gpu'] = has_gpus(cluster) or kwargs.get('numberOfGpusPerNode', 0) > 0
job['params'] = params
# Create proxy entry
if cluster['type'] == 'ec2':
create_proxy_entry(task, cluster, job)
# Before we submit the job upload any file we may have been given
upload_input(task, cluster, job, *args, **kwargs)
submit_job(cluster, job, log_write_url=None,
girder_token=girder_token, monitor=False)
monitor_paraview_job.delay(cluster, job, *args, **kwargs)
def get_assetstore_id(girder_token, cluster):
if "assetstoreId" not in cluster:
headers = {"Girder-Token": girder_token}
url_base = get_assetstore_url_base(cluster)
create_url = "%s/%s" % (cumulus.config.girder.baseUrl, url_base)
body = {
"name": cluster["_id"],
"host": cluster["config"]["host"],
"machine": cluster["config"]["host"],
"authKey": cluster["_id"],
}
user = parse("config.ssh.user").find(cluster)
if user:
body["user"] = user[0].value
r = requests.post(create_url, json=body, headers=headers)
check_status(r)
cluster["assetstoreId"] = r.json()["_id"]
cluster_url = "%s/clusters/%s" % (cumulus.config.girder.baseUrl, cluster["_id"])
body = {"assetstoreId": cluster["assetstoreId"]}
r = requests.patch(cluster_url, json=body, headers=headers)
check_status(r)
return cluster["assetstoreId"]
def test_text_fields(pattern):
owl_entities = set()
if 'classes' in pattern.keys(): owl_entities.update(set(pattern['classes'].keys()))
if 'relations' in pattern.keys(): owl_entities.update(set(pattern['relations'].keys()))
expr = parse('*..text')
text_fields = [match for match in expr.find(pattern)]
stat=True
if text_fields:
for field in text_fields:
# Test for even number single quotes
val = field.value
m = re.findall("'", val)
if divmod(len(m), 2)[1]:
warnings.warn("text field '%s' has an odd number of single quotes." % val)
stat = False
# Test that single quoted strings are OWL entities in dict.
m = re.findall("'(.+?)'", val)
quoted = set(m)
if not owl_entities.issuperset(quoted):
warnings.warn("%s has values (%s) not found in owl entity dictionaries t (%s): "
% (field.full_path, str(quoted.difference(owl_entities)), str(owl_entities)))
stat = False
else:
warnings.warn("Pattern has no text fields")
return stat
def test_vars(pattern):
"""Tests whether variable names in any field with key 'vars'
is in the vars list for the pattern"""
if 'vars' in pattern.keys():
vars = set(pattern['vars'].keys())
else:
warnings.warn("Pattern has no vars")
return True ## If this is to be compulsory, should be spec'd as such in json_schema
if 'data_vars' in pattern.keys():
vars.update(set(pattern['data_vars'].keys()))
if 'substitutions' in pattern.keys():
subvars = [X['out'] for X in pattern['substitutions']]
vars.update(set(subvars))
expr = parse('*..vars')
var_fields = [match for match in expr.find(pattern)]
stat = True
if var_fields:
for field in var_fields:
val = set(field.value)
if not vars.issuperset(val):
warnings.warn("%s has values (%s) not found in pattern variable list (%s): "
% (field.full_path, str(val.difference(vars)), str(vars)))
stat = False
else:
warnings.warn("Pattern has no var fields")
return stat
def __init__(self, name, trait_cfg):
self.cfg = trait_cfg
self.name = name
type_name = trait_cfg.get('type', 'string')
if 'fields' not in trait_cfg:
raise EventDefinitionException(
_("Required field in trait definition not specified: "
"'%s'") % 'fields',
self.cfg)
fields = trait_cfg['fields']
if not isinstance(fields, six.string_types):
# NOTE(mdragon): if not a string, we assume a list.
if len(fields) == 1:
fields = fields[0]
else:
fields = '|'.join('(%s)' % path for path in fields)
try:
self.fields = jsonpath_rw.parse(fields)
except Exception as e:
raise EventDefinitionException(
_("Parse error in JSONPath specification "
"'%(jsonpath)s' for %(trait)s: %(err)s")
% dict(jsonpath=fields, trait=name, err=e), self.cfg)
self.trait_type = type_name
if self.trait_type is None:
raise EventDefinitionException(
_("Invalid trait type '%(type)s' for trait %(trait)s")
% dict(type=type_name, trait=name), self.cfg)
def validate(self):
profile_id = parse('profileId').find(self.volume)[0].value
profile = self.model('aws', 'cumulus').load(profile_id,
user=getCurrentUser())
if not profile:
raise ValidationException('Invalid profile id')
valid_fs = ['ext2', 'ext3', 'ext4']
if 'fs' in self.volume and self.volume['fs'] not in valid_fs:
raise ValidationException('Unsupported file system type', 'fs')
try:
int(self.volume['size'])
except ValueError:
raise ValidationException('size number in an integer', 'size')
# Name should be unique
user = getCurrentUser()
query = {
'name': self.volume['name'],
'userId': user['_id']
}
if '_id' in self.volume:
query['_id'] = {'$ne': self.volume['_id']}
volume = self.model('volume', 'cumulus').findOne(query)
if volume:
raise ValidationException('A volume with that name already exists',
'name')
return self.volume
def main(argv=None):
'''this is called if run from command line'''
try:
argv = argv or sys.argv
parser = argparse.ArgumentParser()
parser.add_argument('path',
type=str,
help='see python module jsonpath_rw')
args = parser.parse_args()
path = args.path
expr = parse(path)
for line in sys.stdin:
try:
(url, repn) = line.split('\t')
obj = json.loads(repn)
vals = expr.find(obj)
val = vals[0].value
print >> sys.stdout, "%s\t%s" % (url, json.dumps(val))
except Exception as e:
print >> sys.stderr, "inner %r" % e
# Failure to parse one line: continue to next
pass
except Exception as e:
print >> sys.stderr, "outer %r" % e
# Any other failure: die with no output
pass
def edit(json, path, value, convert=False, multi=False):
'''Updates the value at the JSONPath endpoint.
:param json JSONConfigParser: The JSON representation to act on.
:param path str: A string representation of a JSONpath endpoint
:param value: The value to be placed at the endpoint
:param convert str: A string representing the steps to convert the value to
it's final form.
'''
expr = parse(path)
matches = expr.find(json)
if len(matches) > 1 and not multi:
raise AttributeError(
'Multiple endpoints found for {}. '
'Please specify the multi flag if this is intended.'
''.format(str(path))
)
if convert:
converter = build_converter(convert)
value = converter(value)
for m in matches:
set_on_path(json, str(m.full_path), value)
def fetch_values(self):
""" fetch the data from providers and select the final values
with jsonpath rules """
# use grid-proxy for authentication
ckey, cert = get_key_cert()
handler = HTTPSClientAuthHandler(ckey, cert)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
# request list of possible values
params = {}
encoded_data = urllib.urlencode(params, doseq=True)
service = self.cfg
url = service['url'] + encoded_data
print(str(url))
req = urllib2.Request(url)
# ensure we get json (sitedb is messed up and randomly returns xml)
if service['jsonpath_selector']:
req.add_header('Accept', 'application/json')
#print req.get_full_url()
stream = urllib2.urlopen(req)
if service['jsonpath_selector']:
response = json.load(stream)
jsonpath_expr = parse(service['jsonpath_selector'])
results = jsonpath_expr.find(response)
stream.close()
return ({'value': v.value} for v in results)
return []
def validate_response_body(cls, test_output_so_far, test_config_data, config_response_body, response_body):
if type(config_response_body) == dict:
for k,v in config_response_body.items():
if TapiExprEvaluator.is_tapi_expr(v):
if not TapiExprEvaluator.get_response_tapi_expr(v, test_output_so_far, test_config_data, response_body):
return False, 'Script {0} validation failed'.format(v)
elif TapiExprEvaluator.is_jsonpath_expr(k):
json_expr = parse(k)
try:
matches = json_expr.find(json.loads(response_body))
except:
return False, 'Json path expression {0} did not find any match against {1}'.format(k, response_body)
if v == '*':
return True, None
if not (len(matches) and matches[0].value == v):
return False, 'Json path expression {0} did not validate against {1}'.format(k, v)
else:
if TapiExprEvaluator.is_tapi_expr(config_response_body):
if not TapiExprEvaluator.get_response_tapi_expr(config_response_body, test_output_so_far, test_config_data, response_body):
return False, 'Script {0} validation failed'.format(config_response_body)
return True, None
def validate(self, cluster):
if not cluster['name']:
raise ValidationException('Name must not be empty.', 'name')
if not cluster['type']:
raise ValidationException('Type must not be empty.', 'type')
scheduler_type = parse('config.scheduler.type').find(cluster)
if scheduler_type:
scheduler_type = scheduler_type[0].value
else:
scheduler_type = QueueType.SGE
config = cluster.setdefault('config', {})
scheduler = config.setdefault('scheduler', {})
scheduler['type'] = scheduler_type
if not queue.is_valid_type(scheduler_type):
raise ValidationException('Unsupported scheduler.', 'type')
# If inserting, ensure no other clusters have the same name field amd
# type
if '_id' not in cluster:
query = {
'name': cluster['name'],
'userId': getCurrentUser()['_id'],
'type': cluster['type']
}
if self.findOne(query):
raise ValidationException('A cluster with that name already '
'exists', 'name')
adapter = get_cluster_adapter(cluster)
return adapter.validate()
def series_key(self, sdmxobj):
# pull down dataset key
dataset_dim = parse(
'$.structure.dimensions.dataSet[*]').find(sdmxobj._elem)
full_key_ids = [d.value['id'] for d in dataset_dim]
full_key_values = [d.value['values'][0]['id'] for d in dataset_dim]
key_idx = [int(i) for i in sdmxobj._elem.value['_key'].split(':')]
struct_dim = parse('$.structure.dimensions.series').find(
sdmxobj._elem)[0].value
series_key_ids = [d['id'] for d in struct_dim]
series_key_values = [d['values'][i]['id'] for i, d in
zip(key_idx, struct_dim)]
full_key_ids.extend(series_key_ids)
full_key_values.extend(series_key_values)
SeriesKeyTuple = namedtuple_factory('SeriesKey', full_key_ids)
return SeriesKeyTuple._make(full_key_values)
def get_value(doc, key):
if not key or not isinstance(doc, dict):
raise ValueError()
jsonpath_expr = parse(key)
matches = jsonpath_expr.find(doc)
value = None if len(matches) < 1 else matches[0].value
return value
def meets_condition(response, condition):
if condition.get('status_code') and response.status_code != condition['status_code']:
return False
if condition.get('json') and response.json() != condition['json']:
return False
if condition.get('text') and response.text != condition['text']:
return False
if condition.get('jsonpath'):
for jsonpath in condition['jsonpath']:
if isinstance(jsonpath['expression'], string_types):
expression = jsonpath_rw.parse(jsonpath['expression'])
else:
expression = jsonpath['expression']
value = jsonpath['value']
results = expression.find(response.json())
if not results:
return False
elif len(results) == 1:
if results[0].value != value:
return False
else:
if [result.value for result in results] != value:
return False
if condition.get('callback'):
if condition['callback'](response) == False:
return False
return True
def __init__(self, node):
"""
`node` is deserialized JSON i.e. it is a native python structure
"""
import jsonpath_rw
self.node = jsonpath_rw.parse('`this`').find(node)[0]
def create(self, params):
body = getBodyJson()
self.requireParams(['name', 'type', 'size', 'profileId'], body)
if not VolumeType.is_valid_type(body['type']):
raise RestException('Invalid volume type.', code=400)
profile_id = parse('profileId').find(body)
if not profile_id:
raise RestException('A profile id must be provided', 400)
profile_id = profile_id[0].value
profile, secret_key = _get_profile(profile_id)
if not profile:
raise RestException('Invalid profile', 400)
if 'zone' in body:
zone = body['zone']
else:
zone = profile['availabilityZone']
volume = self._create_ebs(body, zone)
cherrypy.response.status = 201
cherrypy.response.headers['Location'] = '/volumes/%s' % volume['_id']
return self._model.filter(volume, getCurrentUser())
def path(self, path):
from jsonpath_rw import parse
value = self.value if isinstance(self.value, dict) else json.loads(self.value)
jsonpath_expr = parse(path)
result = [match.value for match in jsonpath_expr.find(value)]
result = result[0] if len(result) == 1 else result
return result
def get_elements (self, json_string, expr):
"""
Возвращает список элементов из _json_string_, соответствующих [http://goessner.net/articles/JsonPath/|JSONPath] выражению.
*Args:*\n
_json_string_ - json-строка;\n
_expr_ - JSONPath выражение;
*Return:*\n
Список найденных элементов. Если элементы не найдены, то возвращается ``None``
*Example:*\n
| *Settings* | *Value* |
| Library | JsonValidator |
| Library | OperatingSystem |
| *Test Cases* | *Action* | *Argument* | *Argument* |
| Get json elements | ${json_example}= | OperatingSystem.Get File | ${CURDIR}${/}json_example.json |
| | ${json_elements}= | Get elements | ${json_example} | $.store.book[*].author |
=>\n
| [u'Nigel Rees', u'Evelyn Waugh', u'Herman Melville', u'J. R. R. Tolkien']
"""
load_input_json=self.string_to_json (json_string)
# парсинг jsonpath
jsonpath_expr=parse(expr)
# список возвращаемых элементов
value_list=[]
for match in jsonpath_expr.find(load_input_json):
value_list.append(match.value)
if not value_list:
return None
else:
return value_list
def parse_json( uuid, date, json_string ):
print "Parsing Json: %s" % json_string
weights = [match.value for match in parse( '[*].weight' ).find( json_string )]
dates = [match.value for match in parse( '[*].date' ).find( json_string )]
ids = [match.value for match in parse( '[*].id' ).find( json_string )]
print "weightparse=%s" % weights
length = len( weights )
result = list()
for idx,weight in enumerate( weights ):
if date == str( dates[idx] ).replace("-",""):
result.append((weight, dates[idx], ids[idx]))
if len(result) >1 :
print( "WARN:%s HAS %s weights for %s" % (uuid, len(result), date))
return result
def parse(self, response):
# x_path test
if self.scraper.checker_type == '4':
self.log("No 404. Item kept.", log.INFO)
return
if self.scraper.detail_page_content_type == 'J':
json_resp = json.loads(response.body_as_unicode())
try:
jsonpath_expr = parse(self.scraper.checker_x_path)
except JsonPathLexerError:
raise CloseSpider("Invalid checker JSONPath!")
test_select = [match.value for match in jsonpath_expr.find(json_resp)]
#self.log(unicode(test_select), log.INFO)
else:
try:
test_select = response.xpath(self.scraper.checker_x_path).extract()
except ValueError:
self.log('Invalid checker XPath!', log.ERROR)
return
if len(test_select) > 0 and self.scraper.checker_x_path_result == '':
self.log("Elements for XPath found on page (no result string defined).", log.INFO)
if self.conf['DO_ACTION']:
self._del_ref_object()
return
elif len(test_select) > 0 and test_select[0] == self.scraper.checker_x_path_result:
self.log("XPath result string '" + self.scraper.checker_x_path_result + "' found on page.", log.INFO)
if self.conf['DO_ACTION']:
self._del_ref_object()
return
else:
self.log("XPath result string not found. Item kept.", log.INFO)
return
def lookup(obj, ref, qlang):
def itemval(e):
if isinstance(e, etree._Element):
return e.text
else:
return e
if qlang==QL.XPath:
if isinstance(ref, tuple):
lval = []
for iref in list(ref):
out = obj.xpath(iref)
if isinstance(out, list):
out = [itemval(e) for e in out]
else:
out = itemval(out)
lval.append(out)
return tuple(lval)
else:
out = obj.xpath(ref)
if isinstance(out, list):
return [itemval(e) for e in out]
else:
return itemvalue(out)
elif qlang==QL.JSONPath:
out = [o.value for o in parse(ref).find(obj)]
return out
elif qlang==QL.CSV:
return obj.get(str(ref))
def parse(self, response):
# x_path test
checker = response.request.meta['checker']
rpt = response.request.meta['rpt']
if checker.checker_type == '4':
self.log("No 404 (%s)." % unicode(checker), log.INFO)
return
if rpt.content_type == 'J':
json_resp = json.loads(response.body_as_unicode())
try:
jsonpath_expr = parse(checker.checker_x_path)
except JsonPathLexerError:
raise CloseSpider("Invalid checker JSONPath (%s)!" % unicode(checker))
test_select = [match.value for match in jsonpath_expr.find(json_resp)]
#self.log(unicode(test_select), log.INFO)
else:
try:
test_select = response.xpath(checker.checker_x_path).extract()
except ValueError:
self.log("Invalid checker XPath (%s)!" % unicode(checker), log.ERROR)
return
if len(test_select) > 0 and checker.checker_x_path_result == '':
self.log("Elements for XPath found on page (no result string defined) (%s). Delete reason." % unicode(checker), log.INFO)
if self.conf['DO_ACTION']:
self._del_ref_object()
return
elif len(test_select) > 0 and test_select[0] == checker.checker_x_path_result:
self.log("XPath result string '%s' found on page (%s). Delete reason." % (checker.checker_x_path_result, unicode(checker)), log.INFO)
if self.conf['DO_ACTION']:
self._del_ref_object()
return
else:
self.log("XPath result string not found (%s)." % unicode(checker), log.INFO)
return
def _load_ref(self, ref_uri, session):
"""Resolve "$ref" URI ``uri``"""
if self.verbose:
print('Resolving $ref URI {}'.format(ref_uri), file=sys.stderr)
parsed_ref_uri = self._parse_ref_uri(ref_uri)
ref_file = parsed_ref_uri.netloc + parsed_ref_uri.path
if not ref_file: # must be relative to current doc
pass # is already in cache
elif ref_file not in self.cache:
self.cache[ref_uri] = self._load_for_cache(parsed_ref_uri, session)
ref_json = self.cache[ref_uri]
expr = jsonpath_rw.parse('$' +
'.'.join(parsed_ref_uri.fragment.split('/')))
for match in expr.find(ref_json):
return match.value # return first match only
# If we reach here, resolution failed
raise RefResolutionException(
'Could not resolve reference URI "{}"'.format(ref_uri))
def get_data_for_key(self, row):
depend_data = self.data.get_depend_key(row)
print "depend_data", depend_data
if depend_data != None:
response_data = self.run_dependent()
print "response_data", type(response_data)
#设置规则，在层级关系中查找需要的数据
try:
json_exe = parse(depend_data)
#查找内容
madle = json_exe.find(response_data)
print "madle", madle
if madle != None:
return [math.value for math in madle][0]
else:
print "nothing is find"
except Exception, e:
raise e
def get_data_for_key(self, row):
"""
获取响应数据中的依赖返回值
:param row: 行数
:return: 依赖返回值
"""
depend_data = self.data.get_depend_key(row)
response_data = self.run_dependent()
if "validateCookie" in depend_data:
cookie = OperationCookie(response_data)
cookie.write_cookie()
code = OperationJson('../data_config/cookie.json')
return code.get_data(depend_data)[8:]
else:
json_rules = parse(depend_data)
result = json_rules.find(json.loads(response_data.text))
return [math.value for math in result][0]
def _validate_zone(self, client, doc):
try:
client = get_ec2_client(doc)
client.describe_availability_zones(
ZoneNames=[doc['availabilityZone']])
except ClientError as ce:
code = parse('Error.Code').find(ce.response)
if code:
code = code[0].value
else:
raise
if code == ClientErrorCode.InvalidParameterValue:
raise ValidationException(
'Invalid zone', 'availabilityZone')
except EndpointConnectionError as ece:
raise ValidationException(ece.message)
def _build_request_entity_selectors(self,
path_item: PathItem) -> typing.Dict:
res = {}
for method_name in self.methods:
method: Operation = getattr(path_item, method_name)
operation = (
ApiOperation.UNKNOWN if method is None else ApiOperation(
get_x(method, "x-hmt-operation", ApiOperation.UNKNOWN)))
if operation == ApiOperation.UPSERT or operation == ApiOperation.INSERT:
request_body = typing.cast(RequestBody, method.requestBody)
if "application/json" in request_body.content:
schema = request_body.content["application/json"].schema
if schema is not None:
entity_path = self._find_entity(
convert_from_openapi(schema), "$")
if res is not None:
res[method_name] = parse(entity_path)
return res
def _extract_envelope(envelope_schema, json_payload):
"""Extract envelope key/values from the original payload
Args:
schema (dict): Parsing schema
envelope_schema (dict): Envelope keys to be extracted
json_payload (dict): The parsed json data
Returns:
dict: Key/values extracted from the log to be used as the envelope
"""
if not isinstance(json_payload, dict):
json_payload = json.loads(json_payload)
LOGGER.debug('Parsing envelope keys')
envelope_keys = envelope_schema.keys()
envelope_jsonpath = jsonpath_rw.parse("$." + ",".join(envelope_keys))
envelope_matches = [match.value for match in envelope_jsonpath.find(json_payload)]
return dict(zip(envelope_keys, envelope_matches))
def __init__(self, filename):
self.items = []
data = load(open(filename))
expr = parse("dependencies[*].vulnerabilities.`parent`")
for item in expr.find(data):
title = f"Vulnerable dependency {item.value['fileName']}"
description = f"{item.value.get('description', '')}"
_severity, steps_to_reproduce = self.steps_to_reproduce(item)
severity = cwe_to_severity(_severity)
file_path = item.value['filePath']
self.items.append({
"title": title,
"description": description,
"severity": severity,
"file_path": file_path,
"steps_to_reproduce": steps_to_reproduce
})
def find_json(json_dict, json_path, condition=[]):
'''
查找符合条件的json列表
:param json_dict: json字典结构
:param json_path: json_path
:param condition: 筛选条件 [==|=], !=, >, <, >=, <= ,
:return:
'''
jsonpath_expr = parse(json_path)
male = jsonpath_expr.find(json_dict)
data = [match.value for match in male]
new_data = []
EXPRESS = ['==', '!=', '<=', '>=', '>', '<']
for d_temp in data:
#d = unicode_dict_list_to_str(d_temp)
d = d_temp
temp = 0
for i in condition:
for e in EXPRESS:
flag = False
if e in i and e+'=' not in i:
k, v = i.split(e)
#执行条件筛选
if isinstance(d, (dict)):
b = d[k]
if isinstance(d[k], (unicode)):
value = d[k].encode('utf-8')
if isinstance(value, basestring):
exec ("if \"%s\" %s %s: flag=True" % (d[k].encode('utf-8'), e, v))
else:
exec ("if %s %s %s: flag=True" % (d[k].encode('utf-8'), e, v))
else:
exec ("if d['%s'] %s %s: flag=True" % (k, e, v))
if flag:
#条件有几个temp就加几次,temp数(符合条件数)和条件数相等才能添加到new_data里面
temp += 1
break
if not flag:
break
#如果连之前的条件都不满足就没有必要再比下去了直接break
if temp == len(condition):
new_data.append(d)
return utils.pretty_dict2(new_data), new_data
def main(arguments):
"""Program entry point"""
arguments = docopt(__doc__, version="0")
# Load spec from file
with open(arguments["SPEC_FILE"], "r") as spec_file:
spec = json.loads(spec_file.read())
# Create default dir structure
for i in ["resources", "responses"]:
os.makedirs(os.path.join(arguments["OUT_DIR"], i), exist_ok=True)
# Generate resources
for component_name, component_spec in spec["components"]["schemas"].items(
):
with open(
os.path.join(arguments["OUT_DIR"], "resources",
component_name + ".json"),
"w",
) as out_file:
out_file.write(
json.dumps(
generate_resource_example(component_spec["properties"],
[component_name])))
# Pull out responses
match_expr = parse(
"paths.*.*.(response|(responses.*)).content.*.(example|(examples.*.value))"
)
for match in match_expr.find(spec):
if "patch" in str(match.full_path):
# PATCHes are not FHIR resources, so we should not be validating them
continue
with open(
os.path.join(
arguments["OUT_DIR"],
"responses",
str(match.full_path).replace("/", "_") + ".json",
),
"w",
) as out_file:
out_file.write(json.dumps(match.value))
def on_message(mosq, userdata, msg):
"""
This is invoked when we get our own message back. Verify that it
is actually our message and if so, we've completed a round-trip.
"""
#print "on_message", msg.topic, str(msg.payload)
global message
global status
payload = msg.payload
if module_jsonpath_rw and module_json:
if args.mqtt_jsonpath is not None:
try:
jspayload = json.loads(payload)
jspath = parse(args.mqtt_jsonpath)
extractpayload = [match.value for match in jspath.find(jspayload)]
payload = extractpayload[0]
except:
payload = ''
pass
print "on_message", msg.topic, str(payload)
elapsed = (time.time() - userdata['start_time'])
userdata['have_response'] = True
status = 2
if args.short_output == True:
message = "value=%s | response_time=%.2f value=%s" % (str(payload), elapsed, str(payload))
else:
message = "message from %s at %s in %.2fs | response_time=%.2f value=%s" % (args.check_subscription, args.mqtt_host, elapsed, elapsed, str(payload))
try:
if (args.mqtt_operator == 'lt' or args.mqtt_operator == 'lessthan') and float(payload) < float(args.mqtt_value):
status = 0
if (args.mqtt_operator == 'gt' or args.mqtt_operator == 'greaterthan') and float(payload) > float(args.mqtt_value):
status = 0
if (args.mqtt_operator == 'eq' or args.mqtt_operator == 'equal') and str(payload) == args.mqtt_value:
status = 0
if (args.mqtt_operator == 'ct' or args.mqtt_operator == 'contains') and str(payload).find(args.mqtt_value) != -1:
status = 0
except:
pass
def main(*argv):
parser = argparse.ArgumentParser(
description=
'Search JSON files (or stdin) according to a JSONPath expression.',
formatter_class=argparse.RawTextHelpFormatter,
epilog="""
Quick JSONPath reference (see more at https://github.com/kennknowles/python-jsonpath-rw)
atomics:
$ - root object
`this` - current object
operators:
path1.path2 - same as xpath /
path1|path2 - union
path1..path2 - somewhere in between
fields:
fieldname - field with name
* - any field
[_start_?:_end_?] - array slice
[*] - any array index
""")
parser.add_argument('expression', help='A JSONPath expression.')
parser.add_argument('files',
metavar='file',
nargs='*',
help='Files to search (if none, searches stdin)')
args = parser.parse_args(argv[1:])
expr = parse(args.expression)
glob_patterns = args.files
if len(glob_patterns) == 0:
# stdin mode
print_matches(find_matches_for_file(expr, sys.stdin))
else:
# file paths mode
for pattern in glob_patterns:
for filename in glob.glob(pattern):
with open(filename) as f:
print_matches(find_matches_for_file(expr, f))
def get_value(expression,
body=None,
value_type="string",
get_tag=False,
expression_instead_none=False):
if isinstance(body, str):
body = loads(body)
if not expression:
return ''
positions = search(r'\$\{(?:(.*))\}', expression)
if positions is not None:
p1 = positions.regs[-1][0]
p2 = positions.regs[-1][1]
else:
p1 = 0
p2 = len(expression)
target_str = str(expression[p1:p2])
if get_tag:
return target_str
full_value = None
try:
if isinstance(body, dict) and target_str.split()[0] in body:
if value_type.lower() == "string":
full_value = expression[0:max(abs(p1 - 2), 0)] + body[
target_str.split()[0]] + expression[p2 +
1:len(expression)]
else:
full_value = body.get(target_str.split()[0])
elif isinstance(body, dict):
jsonpath_expression = parse(target_str)
jsonpath_match = jsonpath_expression.find(body)
if jsonpath_match:
full_value = jsonpath_match[0].value
elif isinstance(body, (str, bytes)):
search_result = search(expression, body)
if search_result.groups():
full_value = search_result.group(0)
elif expression_instead_none:
full_value = expression
except Exception as e:
log.exception(e)
finally:
return full_value
def __enter__(self):
# Do we need to get the session id for this user
if not self._newt_session_id:
headers = {'Girder-Token': self._girder_token}
url = '%s/newt/sessionId' % cumulus.config.girder.baseUrl
r = requests.get(url, headers=headers)
check_status(r)
session_id = parse('sessionId').find(r.json())
if not session_id:
raise Exception('No NEWT session ID present')
self._session = requests.Session()
self._newt_session_id = session_id[0].value
self._session.cookies.set('newt_sessionid', self._newt_session_id)
return self
def get_datasources(self, item):
# Query datasources.
_finder = parse("$..datasource")
_datasources = _finder.find(item)
# Compute unique list of datasources.
datasources = []
for _ds in _datasources:
if not _ds.value:
continue
if isinstance(_ds.value, Munch):
value = dict(_ds.value)
else:
value = str(_ds.value)
if value not in datasources:
datasources.append(value)
return datasources
def notify(request):
if request.method == 'POST':
rulesPath = os.path.join(os.getcwd(), "rules")
requestRule = request.body
bFinRule = False
matchRule = ''
for (dirpath, dirnames, filenames) in walk(rulesPath):
for filename in filenames:
if filename.endswith(".rle"):
filePath = os.path.join(rulesPath, filename)
f = open(filePath, 'r')
fdata = ""
while 1:
line = f.readline()
if not line: break
fdata += line
f.close()
# compare simple strings
if requestRule.replace(' ', '') == fdata.replace(' ', ''):
bFinRule = True
matchRule = filename
break
else: # compare complex rule
try:
complexRule = parse(fdata)
js = json.loads(requestRule)
m = [match.value for match in complexRule.find(js)]
if not len(m) == 0:
bFinRule = True
matchRule = filename
break
except Exception, e:
pass
else:
pass
finally:
pass
def step_impl(context, jsonpath, thing, value):
if not context.content_json:
## Apparently no JSON at all...
assert True is False
else:
jsonpath_expr = jsonpath_rw.parse(jsonpath)
res = jsonpath_expr.find(context.content_json)
if not res[0]:
assert True is False
else:
if thing == "string":
assert res[0].value == value
elif thing == "integer":
assert res[0].value == int(value)
elif thing == "float":
assert res[0].value == float(value)
else:
## Not a thing we know how to deal with yet.
assert True is False
def find_calc_types(self, params):
fields = ['access', 'properties.calculationTypes']
query = {}
if 'moleculeId' in params:
query['moleculeId'] = ObjectId(params['moleculeId'])
calcs = self._model.find(query, fields=fields)
allTypes = []
for types in calcs:
calc_types = parse('properties.calculationTypes').find(types)
if calc_types:
calc_types = calc_types[0].value
allTypes.extend(calc_types)
typeSet = set(allTypes)
return list(typeSet)
def convert(cur_format, new_format, content_data, field_name):
content_data_loaded = json.loads(str(content_data))
if isinstance(content_data_loaded, float) or isinstance(
content_data_loaded, int):
new_value = convert_map[cur_format][new_format](content_data_loaded)
new_content_data = json.dumps(new_value)
else:
pattern = parse(field_name)
match = pattern.find(content_data_loaded)
if match:
cur_value = match[0].value
new_value = convert_map[cur_format][new_format](cur_value)
new_content_data = update_json(content_data_loaded,
get_path(match[0]), new_value)
new_content_data = json.dumps(new_content_data)
else:
raise Exception('No match found')
return new_content_data
def parse_fields(self, field, message):
fval = self.cfg.get(field)
if not fval:
return
if isinstance(fval, six.integer_types):
return fval
try:
parts = jsonpath_rw.parse(fval)
except Exception as e:
raise MeterDefinitionException(
_LE("Parse error in JSONPath specification "
"'%(jsonpath)s': %(err)s") % dict(jsonpath=parts, err=e),
self.cfg)
values = [
match.value for match in parts.find(message)
if match.value is not None
]
if values:
return values[0]
def headmark(self):
dt = ''
r = get(
'https://api.xuangubao.cn/api/pc/msgs?%slimit=50&subjids=9,469,35,10'
% dt)
jo = r.json()
jsonpath_expr = parse('$.NewMsgs[*]')
res = OrderedDict()
for i, match in enumerate(jsonpath_expr.find(jo)):
jo = match.value
ts, title, summary = jo['CreatedAt'], jo['Title'], jo['Summary']
stocks = jo.get('Stocks', )
res['[%s]%s' % (ts, title)] = {
'title': title,
'ts': ts,
'summary': summary,
'stocks': stocks
}
return res
def _handle_function(input_data, func):
function = getattr(functions, func.name, None)
assert function is not None, ("{} is not an available function").format(
func.name)
values = []
for value in func.values:
expr = parse(value)
parsed_value = [m.value for m in expr.find(input_data)][0]
values.append(parsed_value)
assert len(func.values) == len(values), "Unable parse some values"
args = values + func.args
_required_no_of_args = inspect.getfullargspec(function).args
assert len(_required_no_of_args) == len(
args), "Number of arguments mismatch. {} != {}".format(
len(_required_no_of_args), len(args))
return function(*args)
def get_table_from_jpath(jobj, jpath, minrow, mincol):
if jobj is None:
return []
if '_id' in jobj:
cdr_id = jobj['_id']
elif 'cdr_id' in jobj:
cdr_id = jobj['_id']
else:
raise Exception('cdr_id not found in {}'.format(jobj))
my_parser = parse(jpath)
res = []
for match in my_parser.find(jobj):
val = match.value
if val is not None:
row, col = get_table_dim(val)
if row >= minrow or col >= mincol:
val['cdr_id'] = cdr_id
res.append(val)
return res
def capture_box_score_json_experiment(self):
uri = "http://api.nfldata.apiphany.com/nfl/v2/json/BoxScores/2015PRE/4?key=%s" % srkey
print uri
resp = requests.get(uri)
if resp.status_code != 200:
raise Exception(resp.status_code)
#clean it up? load and then return to string
parsed = json.loads(resp.text)
tostr = json.dumps(parsed)
# attempt raw parse
jsonpath_expr = parse(tostr)
items = resp.json()
fname = datetime.now().strftime('%Mminute-%Hhour')
f = open('%s.json' % fname, 'w')
f.write(str(items))
f.close()
def get_response_data(self):
'''
取出依赖的多个字段
:return:
'''
depend_fields_list = self.get_response_field()
depend_case_ids = self.get_case_id()
fields_list = []
for i, depend_case_id in enumerate(depend_case_ids):
response_data = self.send_depend_request(depend_case_id)
fields = []
if depend_fields_list[i]:
for depend_field in depend_fields_list[i]:
depend_field = parse(depend_field)
madle = depend_field.find(response_data)
res = [match.value for match in madle][0]
fields.append(res)
fields_list.append(fields)
return fields_list
def count_tables(jobj, jpath, minrow, mincol):
# print 'here we are!'
if jobj is None:
return 0
if '_id' in jobj:
cdr_id = jobj['_id']
elif 'cdr_id' in jobj:
cdr_id = jobj['cdr_id']
else:
raise Exception('cdr_id not found in {}'.format(jobj))
count = 0
my_parser = parse(jpath)
for match in my_parser.find(jobj):
val = match.value
if val is not None:
row, col = get_table_dim(val)
if row >= minrow or col >= mincol:
count += 1
return count
def analyze_record_delta(self, file_name, test_event):
"""Provide some additional context on why this test failed. This will
perform some analysis of the test record to determine which keys are
missing or which unnecessary keys are causing the test to fail. Any
errors are appended to a list of errors so they can be printed at
the end of the test run.
Args:
file_name (str): Name of file containing the test event
test_event (dict): Actual record data being tested
"""
base_message = ('Invalid test event in file \'{}.json\' with description '
'\'{}\'.'.format(file_name, test_event['description']))
if not self.check_log_declared_in_sources(base_message, test_event):
return
log_type = test_event['log']
if log_type not in self.cli_config['logs']:
message = ('{} Log (\'{}\') declared in test event does not exist in '
'logs.json'.format(base_message, log_type))
self.status_messages.append(StatusMessage(StatusMessage.FAILURE, message))
return
config_log_info = self.cli_config['logs'][log_type]
schema_keys = config_log_info['schema']
envelope_keys = config_log_info.get('configuration', {}).get('envelope_keys')
if envelope_keys:
if self.report_envelope_key_error(base_message, envelope_keys, test_event['data']):
return
# Check is a json path is used for nested records
json_path = config_log_info.get('configuration', {}).get('json_path')
if json_path:
records_jsonpath = jsonpath_rw.parse(json_path)
for match in records_jsonpath.find(test_event['data']):
self.report_record_delta(base_message, log_type, schema_keys, match.value)
return
self.report_record_delta(base_message, log_type, schema_keys, test_event['data'])
def send_requests(s, testdata):
'''封装requests请求'''
method = testdata["请求方式"]
url = testdata["url"]
header = eval(testdata["header"])
data = eval(testdata["请求数据"])
test_nub = testdata['ID']
expect = eval(testdata['预期结果'])
depend_case = testdata['依赖id']
depend_data = testdata['依赖数据']
if depend_case == "":
print("*******正在执行用例：----- %s ----**********" % test_nub)
print("请求方式：%s, 请求url:%s" % (method, url))
print("预期结果: %s" % expect)
res = s.request(method=method, url=url, headers=header, data=json.dumps(data))
print("请求响应结果: %s" % res.text)
else:
dependdata = dependent_data(depend_case)
depend_response_data = dependdata.run_dependent(depend_case)
print(depend_response_data)
jsonpath_expr = parse(depend_data)
madle = jsonpath_expr.find(depend_response_data)
depend_response_data_new = [match.value for match in madle][0]
depend_data_new = testdata['依赖数据所属字段']
header[depend_data_new] = depend_response_data_new
print("*******正在执行用例：----- %s ----**********" % test_nub)
print("请求方式：%s, 请求url:%s" % (method, url))
print("预期结果: %s" % expect)
print("依赖ID: %s" % depend_case)
print("依赖数据: %s" % depend_data)
print("依赖数据的值: %s" % depend_response_data_new)
print(type(header))
print(header)
print("===============================")
print(json.dumps(data))
print("===============================")
print(method)
print("===============================")
print(url)
print("===============================")
res1 = s.request(method=method, url=url, headers=header, data=json.dumps(data))
print("请求响应结果: %s" % res1.text)
return res1
def extract_json(self, text, path):
try:
if not isinstance(text, string_types):
logging.warning("[extract_json] unknow text type %s" %
type(text))
return []
data_json = json.loads(text)
logging.debug("[extract_json] Parse json text successfully.")
jsonpath_expr = parse(path)
extract = [match.value for match in jsonpath_expr.find(data_json)]
# 转换为string list
for idx, e in enumerate(extract):
if not isinstance(e, string_types):
extract[idx] = json.dumps(e)
return extract
except Exception:
logging.exception("extract_json error: %s" % path)
raise
def run(host, keyword, resultname, basicQuery, repeatQuery, jsonPath):
result = ""
if jsonPath != "":
jsonpath_expr = parse(jsonPath)
with open(os.path.join(keyword), 'r') as f:
query = host + basicQuery
tempQ = query
req = requests.get(query)
ret = req.text
ret = json.loads(ret)
if req.status_code == 200:
result += (tempQ + " : OK\n")
result += ("검색결과 갯수 : ")
result += str(ret["response"]["rows"])
result += "\n"
else:
result += (tempQ + " : Not OK\n")
repeatQuery = repeatQuery.split("&")
basicQuery = basicQuery.split("&")
while (True):
line = f.readline()
if not line: break
for basicQ in basicQuery:
for repeatQ in repeatQuery:
temp = basicQ.split("=")
temp = temp[0]
if repeatQ == temp:
tempQ = query.replace(basicQ, (repeatQ + "=" + line))
req = requests.get(tempQ)
ret = req.text
ret = json.loads(ret)
if req.status_code == 200:
result += (tempQ + " : OK\n")
result += ("검색결과 갯수 : ")
result += str(ret["response"]["rows"])
result += "\n"
else:
result += (tempQ + " : Not OK\n")
print result
with open(os.path.join(BASE_DIR, resultname), 'w+') as result_file:
result_file.write(result)
def _on_response_filter_changed(self, entry: Gtk.SearchEntry):
filter_text = entry.get_text()
if filter_text == '':
self._set_response_text()
ct = get_content_type(self.last_response)
try:
if ct == 'application/json':
path_expr = jsonpath_rw.parse(filter_text)
j = self.last_response.json()
match_text = json.dumps(
[match.value for match in path_expr.find(j)],
indent=4) or 'No matches found'
self.response_text.get_buffer().set_text(match_text)
elif ct in {'text/xml', 'application/xml'}:
root = etree.fromstring(self.last_response.text)
matches = root.xpath(filter_text)
matches_root = etree.Element('matches')
for m in matches:
matches_root.append(m)
matches_html = etree.tostring(matches_root,
encoding='unicode',
pretty_print=True)
self.response_text.get_buffer().set_text(matches_html)
elif ct == 'text/html':
root = html.fromstring(self.last_response.text)
matches = root.xpath(filter_text)
matches_root = etree.Element('matches')
for m in matches:
matches_root.append(m)
matches_html = etree.tostring(matches_root,
encoding='unicode',
pretty_print=True)
self.response_text.get_buffer().set_text(matches_html)
else:
log.warning(
'Got unexpected content type %s when filtering response.',
ct)
except Exception as e:
log.debug('Failed to filter response json %s', e)
