def __init__(self, schema):
self.schema = schema
self.blocks = jp.match("$..blocks[*]", self.schema)
self.sub_blocks = jp.match(
"$..[add_block, edit_block, add_or_edit_block, remove_block]",
self.schema)
self.blocks_by_id = {
block["id"]: block
for block in self.blocks + self.sub_blocks
}
self.block_ids = list(self.blocks_by_id.keys())
self.calculated_summary_block_ids = {
block["id"]
for block in self.blocks_by_id.values()
if block["type"] == "CalculatedSummary"
}
self.sections = jp.match("$.sections[*]", self.schema)
self.sections_by_id = {
section["id"]: section
for section in self.sections
}
self.section_ids = list(self.sections_by_id.keys())
self.groups = jp.match("$..groups[*]", self.schema)
self.groups_by_id = {group["id"]: group for group in self.groups}
self.group_ids = list(self.groups_by_id.keys())
self.list_names = jp.match(
'$..blocks[?(@.type=="ListCollector")].for_list', self.schema)
self._answers_with_context = {}
def __init__(self, schema):
self.schema = schema
self.blocks = jp.match("$..blocks[*]", self.schema)
self.sub_blocks = jp.match(
"$..[add_block, edit_block, add_or_edit_block, remove_block]",
self.schema)
self.blocks_by_id = {
block["id"]: block
for block in self.blocks + self.sub_blocks
}
self.block_ids = list(self.blocks_by_id.keys())
self.sections = jp.match("$.sections[*]", self.schema)
self.sections_by_id = {
section["id"]: section
for section in self.sections
}
self.section_ids = list(self.sections_by_id.keys())
self.groups = jp.match("$..groups[*]", self.schema)
self.groups_by_id = {group["id"]: group for group in self.groups}
self.group_ids = list(self.groups_by_id.keys())
self.list_names = jp.match(
'$..blocks[?(@.type=="ListCollector")].for_list', self.schema)
def result_all(data):
title = jp.match('data.children[*].data.title', data)
url = jp.match('data.children[*].data.url', data)
selftext = jp.match('data.children[*].data.selftext', data)
for titles, urls, selftexts in zip(title, url, selftext):
print(titles + '\n')
print(selftexts.encode('utf-8') + '\n')
print(urls)
def log_comparison(plan_index, global_index, parameter_name):
pth1 = "$.planCostShares[" + str(plan_index) + "]." + parameter_name
pth2 = "$.globalcostshares[" + str(global_index) + "]." + parameter_name
val1, val2 = str(jp.match(pth1,
plan_data)), str(jp.match(pth2, sec_plan_data))
if val1 == val2:
logging.info("Both files values of " + parameter_name + " is same")
else:
logging.error(
"\t " + parameter_name +
" values of file 1 is %s and file 2 is %s", val1, val2)
def test(self, response):
#print(response.text)
html = json.loads(response.text)
rate = jsonpath_rw_ext.match('$..rate', html)
title = jsonpath_rw_ext.match('$..title', html)
id = jsonpath_rw_ext.match('$..id', html)
for i in range(0, len(rate)):
item = FilmIntroductionItem()
item['movie_rate'] = rate[i]
item['movie_name'] = title[i]
item['movie_id'] = id[i]
yield item
def find_index(file_json_data, planshare_costsharelevel, planshare_costsharetype, planshare_providertier):
globalCostShares_json_path = "$.globalcostshares[*]"
sec_index = 0
for values in jp.match(globalCostShares_json_path, file_json_data):
costsharelevel = jp.match("$.globalcostshares[" + str(sec_index) + "].costShareLevel", file_json_data)[0]
costsharetype = jp.match("$.globalcostshares[" + str(sec_index) + "].costShareType", file_json_data)[0]
providertier = jp.match("$.globalcostshares[" + str(sec_index) + "].providerTier", file_json_data)[0]
if costsharelevel == planshare_costsharelevel and costsharetype == planshare_costsharetype:
if providertier == planshare_providertier:
return sec_index
sec_index = sec_index + 1
return -1
def parse(self, response):
result = json.loads(response.text)
for image in result.get('list'):
item = ImageItem()
gid = image.get('grpmd5')
item['id'] = image.get('id')
item['url'] = image.get('qhimg_url')
item['title'] = image.get('title')
item['thumb'] = image.get('qhimg_thumb')
item['gid'] = gid
yield item
img_path = 'D:\\images.so_webspider\\images' + os.path.sep + image.get(
'title')
if not os.path.exists(img_path):
os.mkdir(img_path)
try:
t = requests.get(
'http://images.so.com/z?a=jsondetailbygidv2&currsn=0&identity=list&ch=beauty&gid={gid}'
.format(gid=gid),
headers=self.header)
if codes.ok == t.status_code:
img_url = json.loads(t.text)
imgs = jsonpath_rw_ext.match('$..qhimg_url', img_url)
for img in imgs:
image = requests.get(img,
headers=self.header_download)
file_path = img_path + os.path.sep + '{img}'.format(
img=img[24:])
with open(file_path, 'wb') as f:
f.write(image.content)
print('Downloaded image path is %s' %
file_path)
except Exception as e:
print(e)
def get_list_of_execution_by_cycle(self, cycle_id, project_id, version_id):
end_point = 'executions/search/cycle/'
canonical_path = f"GET&{self.RELATIVE_PATH}{end_point}{cycle_id}&projectId={project_id}&versionId={version_id}"
token = self.jwt.generate_jwt(canonical_path)
headers = {
'Authorization': f'JWT {token}',
'Content-Type': 'text/plain',
'zapiAccessKey': self.access_key
}
# Get List of Executions By Cycle
print("Getting list of execution ID's by Cycle")
raw_result = requests.get(self.ZAPI_CLOUD_URL + self.RELATIVE_PATH +
end_point + cycle_id + '?versionId=' +
version_id + '&projectId=' + project_id,
headers=headers)
assert raw_result.status_code == 200
if tools.is_json(raw_result.text):
# JSON RESPONSE: convert response to JSON
json_result = json.loads(raw_result.text)
# PRINT RESPONSE: pretty print with 4 indent
# print(json.dumps(json_result, indent=4, sort_keys=True))
# Getting Execution ID Lists
execution_ids = jp.match("$.searchObjectList[*].execution.id",
json_result)
print(f"Execution ID's are {execution_ids}")
return execution_ids
def numeric_answer_ranges(self):
numeric_answer_ranges = {}
for answer in jp.match("$..answers[*]", self.schema):
numeric_answer_ranges[answer["id"]] = get_numeric_range_values(
answer, numeric_answer_ranges)
return numeric_answer_ranges
def step_impl(context,status):
response = requests.request(_method, _url)
global _result
_result = response.json()
archiveResponse.saveJson(str(_result))
global _asteroids
_asteroids = jp.match("$.near_earth_objects.*.[*]", _result)
assert response.status_code is int(status)
def step_impl(context):
global response
response = requests.request(
"GET", settings['url']['root'] +
settings['url']['path']['asteroids']['septemberAsteroids'])
global data
data = response.json()
data = jp.match("$.near_earth_objects.*.[*]", data)
assert response.status_code is 200
def get_blocks(self, **filters):
conditions = []
for key, value in filters.items():
conditions.append(f'@.{key}=="{value}"')
if conditions:
final_condition = " & ".join(conditions)
return jp.match(f"$..blocks[?({final_condition})]", self.schema)
return self.blocks
def get_shopinfo(self, response):
#使用jsonpath_rw_ext对数据进行提取
html = json.loads(response.text)
branchName = jsonpath_rw_ext.match('$..branchName', html)
categoryId = jsonpath_rw_ext.match('$..categoryId', html)
categoryName = jsonpath_rw_ext.match('$..categoryName', html)
id = jsonpath_rw_ext.match('$..id', html)
matchText = jsonpath_rw_ext.match('$..matchText', html)
name = jsonpath_rw_ext.match('$..name', html)
priceText = jsonpath_rw_ext.match('$..priceText', html)
recommendReasonData = jsonpath_rw_ext.match('$..recommendReasonData',
html)
reviewCount = jsonpath_rw_ext.match('$..reviewCount', html)
shopPower = jsonpath_rw_ext.match('$..shopPower', html)
for i in range(0, len(name)):
item = ShopInfoItem()
item['branch_name'] = branchName[i]
item['category_id'] = str(categoryId[i])
item['category_name'] = categoryName[i]
item['shop_id'] = id[i]
item['match_text'] = matchText[i]
item['name'] = name[i]
item['avg_cost'] = priceText[i]
#解决部分店铺没有recommendReason字段的问题
recommendReason = re.sub("{'iconHeight': 0, 'iconWidth': 0,", '',
str(recommendReasonData[i]))
recommendReason = re.sub(
"'recommendReasonType': 0, 'recommendReasonUserId': 0}", '',
recommendReason)
recommendReason = re.sub("'recommendReason': '", '',
recommendReason)
recommendReason = re.sub("',", '', recommendReason)
item['reason'] = recommendReason
item['review_count'] = str(reviewCount[i])
item['shop_mark'] = str(shopPower[i])
item['created_at'] = datetime.datetime.now()
item['updated_at'] = datetime.datetime.now()
item['content_id'] = 0
yield item
def get_other_blocks(self, block_id_to_filter, **filters):
conditions = []
for key, value in filters.items():
conditions.append(f'@.{key}=="{value}"')
if conditions:
final_condition = " & ".join(conditions)
return jp.match(
f'$..blocks[?(@.id!="{block_id_to_filter}" & {final_condition})]',
self.schema,
)
return self.blocks
def get_from_JSON(self, json_file, json_path):
"""
Search for a value in the given json file.
If multiple matches found it returns multiple matches
"""
data = json.dumps(json_file) # Dict to string
data = json.loads(data) # string to json - pretty cumbersome
value = jp.match(json_path, data)
if value == []: # Check if empty
raise Exception("No value found for the given JSON path: {}".format(json_path))
print("Actual value: {} {}".format(value[0], type(value[0])))
return value[0]
def get_price_from(page, province_code, city_code, begin_date, end_date):
# 根据不同的页数, 省市的code构造请求
referer = referer_pattern.format(page=page,
province=province_code,
city=city_code,
begin=begin_date,
end=end_date)
headers = HEADERS
headers.update({'referer': referer})
query = QUERY
query.update({
'page': str(page),
"provinceCode": province_code,
"cityCode": city_code,
"startTime": begin_date,
"endTime": end_date
})
r = requests.get(url, headers=headers, params=query)
data = r.json()
pages, brands, types, provinces, cities, counties, priceComps, times, bulks, sacks, guides = [], [], [], [], [], [], [], [], [], [], []
print(data)
try:
price_list = jp.match('$..price_list[*]', data)
# print(price_list)
for price in price_list:
pages.append(page)
brands.append(price['brand'])
types.append(price['type'])
provinces.append(price['province'])
cities.append(price['city'])
counties.append(price['county'])
priceComps.append(price['PriceComp'])
times.append(price['time'])
bulks.append(price['bulk'])
sacks.append(price['sack'])
guides.append(price['guide'])
tuples = list(
zip(pages, brands, types, provinces, cities, counties, priceComps,
times, bulks, sacks, guides))
output_df = pd.DataFrame(tuples,
columns=[
'页数', '品牌', '水泥品种', '省份', '城市', '区域',
'生产厂家', '日期', '散装价', '袋装价', '参考价'
])
# trained = pd.DataFrame(tuples,
# columns=['brand', 'type', 'province', 'city', 'county', 'PriceComp', 'date',
# 'start_priceSack', 'end_priceSack', 'bulk', 'sack', '', 'guide'])
# coll.insert_many(price_list)
return True, output_df
except Exception:
return False, ''
def _lookup(
json_path: str,
working_dir: str = settings.working_dir(),
translations_json_location: str = settings.TRANSLATIONS_JSON_LOCATION,
) -> Any:
"""Return jsonpath value or empty list if JSON node doesn't exist."""
json_data = fetch_source_data(working_dir, translations_json_location)
value = jp.match(
json_path,
json_data,
)
value_set = set(value)
return list(value_set)
async def on_post(self, req, res, *, anything):
print("Inside POST method")
postedJson = await req.media()
actualJson = json.dumps(postedJson, indent=4)
matches = jp.match('$..GlossDiv..Title', postedJson)
if len(matches) <= 0:
res.text = "No match found for the query"
elif len(matches) == 1:
if matches[0] == 'abc':
res.media = outputJson("abc.json")
elif matches[0] == 'xyz':
res.media = outputJson("xyz.json")
else:
res.media = outputJson("abcxyz.json")
elif len(matches) > 1:
res.text = "More than one value matched for the query"
def lambda_handler(event, context):
client = FarosClient.from_event(event)
# First, fetch the query from the input key.
query_key = event['params'].get('key')
if query_key is None:
raise RuntimeError("missing 'key' parameter")
res = requests.get(QUERY_BASE_URL + query_key)
if res.status_code == 404: # Nicer error if the query did not exist.
raise RuntimeError('no such query: {!r}'.format(query_key))
res.raise_for_status() # Any other error type.
query = res.json()['query']
# Then, execute it and apply any JSONPath expression.
data = client.graphql_execute(query['doc'])
json_path = query.get('dataPath')
if json_path:
data = jsonpath_rw_ext.match(json_path, data)
return data
def update_json_config(self, json_added, json_path, options, list_column,
config_text):
"""
Write JSON file configuration
"""
data = literal_eval(config_text)
if (options != "replace"):
json_target = jsonpath_rw_ext.match(json_path, data)
if isinstance(json_target[0], dict):
if len(list_column) == 1:
json_target[0][list_column[0]] = json_added
json_final = json_target[0]
else:
return False, json.dumps(data, indent=4)
else:
for json_target_value in json_target[0]:
json_added.append(json_target_value)
json_final = json_added
else:
json_final = json_added
jsonpath_expr = parse(json_path)
matches = jsonpath_expr.find(data)
if len(matches) == 0:
return make_response(
json.dumps({
'success': False,
'message': 'JSON path not found.'
}))
for match in matches:
data = ClientConfiguration.update_json(
data, ClientConfiguration.get_path(match), json_final)
return make_response(
json.dumps({
'success': True,
'data': json.dumps(data, indent=4)
}))
# get path to vm, break apart
fullPath = jp.match1("elements[*].Path", json_data)
fullPath = fullPath[0:fullPath.rfind("/")]
pathsplit = fullPath.split("/")
print("Full path: {}".format(fullPath))
print("Parent Folder: {}".format(pathsplit[len(pathsplit) - 1]))
# show IP address and cidr/netmask
print("Default IpAddress: {}".format(
jp.match1("elements[*].Object.Guest.IpAddress", json_data)))
hasNet = jp.match1("elements[*].Object.Guest.Net", json_data)
if hasNet:
for IpAddress in jp.match(
"elements[*].Object.Guest.Net[*].IpConfig.IpAddress[*]",
json_data):
cidr = IpAddress['PrefixLength']
if isIPv4(IpAddress['IpAddress']):
print(" IPv4 {}/{} netmask {}".format(IpAddress['IpAddress'],
cidr,
cidr_to_netmask(cidr)))
else:
print(" IPv6 {}/{}".format(IpAddress['IpAddress'], cidr))
else:
print("There are no network elements in Guest, VM might be powered off")
# memory/cpu
#print("MaxMemoryUsage: {}".format( jp.match1("elements[*].Object.Runtime.MaxMemoryUsage",json_data) ) )
print("MemorySizeMB: {}".format(
def process(instance, arg):
return jp.match(arg._expr, instance.payload)
pth2 = "$.globalcostshares[" + str(global_index) + "]." + parameter_name
val1, val2 = str(jp.match(pth1,
plan_data)), str(jp.match(pth2, sec_plan_data))
if val1 == val2:
logging.info("Both files values of " + parameter_name + " is same")
else:
logging.error(
"\t " + parameter_name +
" values of file 1 is %s and file 2 is %s", val1, val2)
planCostShares_json_path = "$.planCostShares[*]"
globalCostShares_json_path = "$.globalcostshares[*]"
index = 0
for values in jp.match(planCostShares_json_path, plan_data):
logging.info("\n\nProduct dict at position " + str((index + 1)) +
"\n==========================================\n")
ps_costsharelevel = jp.match(
"$.planCostShares[" + str(index) + "].costShareLevel", plan_data)[0]
ps_costsharetype = jp.match(
"$.planCostShares[" + str(index) + "].costShareType", plan_data)[0]
ps_providertier = jp.match(
"$.planCostShares[" + str(index) + "].providerTier", plan_data)[0]
gindex = pcu.get_global_share_values_index(sec_plan_data,
ps_costsharelevel,
ps_costsharetype,
ps_providertier)
logging.info("Found the match at the index : " + str(gindex))
if gindex > -1:
logging.error(
def step_impl(context):
data = context.response.json()
context.data = jp.match("$.near_earth_objects.*.[*]", data)
# After we declare these variables "into context" we have them for the rest of the test
assert context.response.status_code is 200
async def matches(self, getter, session: Union[ClientSession, None]):
got = await self.get(session=session)
return jp.match(getter, got)
def contains_jsonpath(self, jsonpath):
return jp.match(jsonpath, self.json_data)
root = html.fromstring(js_comment_clean(data))
def dom_recurse(node, tags):
temp_obj = []
i = 0
for childnode in node:
child_obj = []
if childnode.tag not in tags:
attr = {}
for key, value in childnode.attrib.iteritems():
attr[key] = value
child_obj.append({
"child": dom_recurse(childnode, tags),
"tag": childnode.tag,
"text": childnode.text,
"attr": attr
})
i = i + 1
temp_obj.append(child_obj[0])
return temp_obj
print json.dumps(dom_recurse(root, ['']))
print jp.match('$.[0].child.[*].text', dom_recurse(root, ['']))
async def matches(self, getter) -> list:
got: dict = await self.to_json()
matches: list = jp.match(getter, got)
return matches
def match(self, instance):
return jp.match(self._expr, instance.payload)
def return_first_value_by_jsonpath(self, jsonpath):
for v in jp.match(jsonpath, self.json_data):
return v
return None
