def get_all_charities_from_jsons():
'''
Uses json utils to read json files and place results in an array
'''
# Grab charities from FoodBanks.json
charities = read_json_file("./FoodBanks.json")
# Grab charities from HomelessServices.json
charities += read_json_file("./HomelessServices.json")
return charities
def __init__(self, lang: Text):
from sagas.conf import resource_path
import json_utils
from os import path
file = resource_path(f'analspa_{lang}.json')
self.root = json_utils.read_json_file(file) if path.exists(
file) else {}
def load_rss_seed(json_file='./data/rss/westore_new.json'):
import json_utils
rss_doc = json_utils.read_json_file(json_file)
# rss_doc['title']
product_set = []
for entry in rss_doc['entry']:
# print(entry["title@zh"], entry['published'])
data = {}
data['productName'] = entry["title@zh"]
data['createdDate'] = now_jdbc()
data['lastModifiedDate'] = now_jdbc()
data['fromDate'] = to_jdbc(entry['published'])
data['isVirtual'] = 'N'
data['productTypeId'] = "FINISHED_GOOD"
data['productId'] = extract_id(entry['id'])
data['productPricePurposeId'] = "PURCHASE"
data['productPriceTypeId'] = "DEFAULT_PRICE"
data['productStoreGroupId'] = "Test_group"
data['priceDetailText'] = entry['summary']
attrs = get_product_attrs(entry)
if "价格:" in attrs:
data['price'] = float(attrs['价格:'].strip().replace('元', ''))
data['currencyUomId'] = 'CNY'
if "image" in attrs:
data['largeImageUrl'] = attrs['image']
product_set.append(data)
# print(json.dumps(product_set, indent=2, ensure_ascii=False))
return product_set
def practice(self, lang_tr='fr', use_latest=False):
"""
$ python -m sagas.nlu.nlu_tools practice fr
:param lang:
:param lang_tr:
:param dataf:
:return:
"""
from sagas.nlu.tts_utils import say_lang
import json_utils
lang = 'en'
dataf=corpus_resources[lang_tr]
rows=[]
if use_latest:
rows = json_utils.read_json_file('./out/latest_%s.json' % lang_tr)
else:
rows=load_corpus(dataf)
json_utils.write_json_to_file('./out/latest_%s.json' % lang_tr, rows.tolist())
for r in rows:
sents = str(r[0])
tr_lang = str(r[1].strip())
say_lang(sents, lang)
print('♥', sents)
say_lang(tr_lang, lang_tr)
print('♡', tr_lang)
def word_info(self, word):
"""
$ python -m sagas.ru.ru_procs word_info мальчик
:param word:
:return:
"""
import json_utils
ipa_xs = self.epi.xsampa_list(word)
ipa = self.epi.transliterate(word)
rev_map = json_utils.read_json_file(self.target_file_rev)
words_map = json_utils.read_json_file(self.target_file)
key = rev_map[word]
print(word, ipa, ''.join(ipa_xs), '☌', key)
print('\t', words_map[key])
def word_info_df(self, word):
import sagas
import json_utils
rev_map = json_utils.read_json_file(self.target_file_rev)
words_map = json_utils.read_json_file(self.target_file)
tuples = []
ipa_xs = self.epi.xsampa_list(word)
ipa = self.epi.transliterate(word)
key = rev_map[word]
tuples.append((word, ipa, ''.join(ipa_xs), key))
for w in words_map[key]:
ipa_xs = self.epi.xsampa_list(w)
ipa = self.epi.transliterate(w)
tuples.append((w, ipa, ''.join(ipa_xs), key))
return sagas.to_df(tuples, ['word', 'ipa', 'xsampa_list', 'key'])
def update_by_overrides(self) -> None:
overrides = json_utils.read_json_file(self.overrides_file)
for k, v in overrides.items():
if '.' in k:
parts = k.split('.')
self.conf[parts[0]][parts[1]] = v
else:
self.conf[k] = v
def genres_loader():
genres_root = read_json_file(root_dir + "data/movie_genres_f.json")
genres_map = {}
for genre in genres_root['genres']:
genres_map[genre['id']] = genre['name']
for genre in genres_root['genres']:
store_movie_genres(genre)
def view_json_data(self, file_name):
data=json_utils.read_json_file(file_name)
index=1
for item in data["rasa_nlu_data"]["common_examples"]:
print("{}. {} ({})".format(index, item["text"], item["intent"]))
children=item['entities']
if len(children)>0:
for c in children:
print("\t{}-{}: {}({})".format(c["start"], c["end"], c["value"], c["entity"]))
index=index+1
def movies_loader(file):
data_root=read_json_file(file)
print(data_root["page"], "/", data_root["total_pages"])
movies=data_root["results"]
for movie in movies:
result = create_or_store("SaMovie", movie)
set_movie_genres(movie["id"], movie['genre_ids'])
print(result.get("title"))
def get_state_name_from_dict(state_dict_json_file, state):
'''
Gets the state dict from the provided json file and returns
the value corresponding to the provided state.
'''
state_dict = read_json_file(state_dict_json_file)
state_name = ''
if state in state_dict:
state_name = state_dict[state]
return state_name
def lookup_nor(self, word):
"""
$ python -m sagas.ko.kwn_procs lookup_nor '미생물'
:param word:
:return:
"""
import json_utils
if not self.idx_nor:
self.idx_nor = json_utils.read_json_file(
f"{self.index_dir}/ko_nor.json")
return self.idx_nor[word] if word in self.idx_nor else []
def lookup_tra(self, word):
"""
$ python -m sagas.ko.kwn_procs lookup_tra mi-saeng-mur
:param word:
:return:
"""
import json_utils
if not self.idx_tra:
self.idx_tra = json_utils.read_json_file(
f"{self.index_dir}/ko_tra.json")
return self.idx_tra[word] if word in self.idx_tra else []
def similar(self, feat_name, sents):
"""
$ python -m sagas.nlu.expanders similar samples 'Charge périodique'
:param feat_name:
:param sents:
:return:
"""
arranger = json_utils.read_json_file("./data/feats/%s.json" %
feat_name)
doc_vecs = np.load("./data/feats/%s.npy" % feat_name) # load
bm = BertManager()
# 'Charge périodique'
bm.get_similar(arranger, doc_vecs, sents)
def build_input_pairs(additions):
import clipboard
voc_dicts = f'{cf.conf_dir}/langs/voc/ru-voc-dicts.json'
voc_words = json_utils.read_json_file(voc_dicts)
print('total voc words', len(voc_words))
rs = []
for w in voc_words:
for en_w in w['en']:
ru_w = ' '.join(w['ru']).replace(',', '')
rs.append('%s %s' % (norm_key(en_w), ru_w))
print('input pairs contains voc words %d, and ips words %d'%(len(rs), len(additions)))
clipboard.copy('\n'.join(rs+additions))
def __init__(self):
import glob
import json_utils
from sagas.conf import resource_files, resource_path
self.intents = []
files = [resource_path(f) for f in resource_files('ruleset_*.json')]
# for f in glob.glob(f'{cf.conf_dir}/stack/conf/ruleset_*.json'):
for f in files:
rules = json_utils.read_json_file(f)
# for rule in rules:
# self.intents.append({rule['intent']: {'triggers': rule['action']}})
self.intents.extend(rules)
def load_resources(self):
"""
$ python -m sagas.graph.rss_hub load_resources
:return:
"""
from io_utils import list_with_suffix
schema_map = self.get_schema_map()
client = helper.reset(schema_map)
files = list_with_suffix('data/rss', '.json')
for file in tqdm(files):
feed_json = json_utils.read_json_file(file)
# feed_json = self.get_feed_json(file)
_ = set_json(client, feed_json)
def check_json(self, input_file):
data=json_utils.read_json_file(input_file)
dataset=NluData()
for item in data["rasa_nlu_data"]["common_examples"]:
# print("{}. {} ({})".format(index, item["text"], item["intent"]))
fact=FactData(item["text"], item["intent"])
children=item['entities']
if len(children)>0:
for c in children:
# print("\t{}-{}: {}({})".format(c["start"], c["end"], c["value"], c["entity"]))
entity=EntityData(c["start"], c["end"], c["value"], c["entity"], c["value"])
fact.entities.append(entity)
dataset.examples.append(fact)
print(json.dumps(dataset.json, indent=2, sort_keys=False, ensure_ascii=False))
def __init__(self):
from sagas.conf.runtime import runtime
from cachetools import LRUCache
conf_file = f"{runtime_dir()}/sagas_conf.json"
overrides_file = f"{runtime_dir()}/sagas_overrides.json"
self.conf = json_utils.read_json_file(conf_file)
self.overrides_file = overrides_file
if runtime.is_docker():
self.update_by_overrides()
# self.cache = LRUCache(maxsize=1024)
self.loaded_classes = {}
def build_voc():
voc_file=f'{cf.conf_dir}/langs/voc/ru-voc.json'
voc_dicts=f'{cf.conf_dir}/langs/voc/ru-voc-dicts.json'
words=json_utils.read_json_file(voc_file)
all_dicts=build_dicts()
print('filter by voc-words ...')
skips=[]
rs=filter_set(all_dicts, words, 'ru', skips)
json_utils.write_json_to_file(voc_dicts, rs)
# print('done.')
print('absents %d, see these words in file ru-voc-absents.json'%len(skips))
json_utils.write_json_to_file(f'{cf.conf_dir}/langs/voc/ru-voc-absents.json', skips)
return rs, skips
async def add(ctx, arg, *args):
if ctx.message.author.top_role.name in config.get('allow_permission', []):
if args and arg[0] == '!':
shitpost_keys = read_json_file('shitpost.json').get(
'shitpost', {}).keys()
if arg not in shitpost_keys:
add_shitpost_record(arg, ' '.join(args))
await ctx.send(f'Added command: {arg}')
else:
await ctx.send('Command already exist!')
else:
await ctx.send(
'Command does not start with ! or either does not have content.'
)
else:
await ctx.send('Incorrect permission!')
def check_modified(self):
"""
$ python -m saai.nlu_mod_procs check_modified
:return:
"""
import os
if not os.path.exists(self.timestamps_file):
return mods
before = json_utils.read_json_file(self.timestamps_file)
current = self.get_timestamps()
modified = []
for item in current:
if current[item] - before[item] > 1:
modified.append(item)
return modified
def test_json_utils2(self):
'''
Testing write_json_file and read_json_file by writing
empty dict to file
'''
test_dict = {}
temp_test_file = './temp_json_test_file.json'
# Write test_dict to temp_test_file
json_utils.write_json_file(temp_test_file, test_dict)
# Read temp_json_test_file.json to get dict
dict_in_temp_test_file = json_utils.read_json_file(temp_test_file)
# Make sure json reader returns original test_dict from file
self.assertEqual(dict_in_temp_test_file, test_dict)
# Clean up and delete temp test file
os.remove(temp_test_file)
def fill_data_seeds(self, recreate=False):
"""
$ python -m sagas.modules.life.data_source fill_data_seeds
$ python -m sagas.modules.life.data_source fill_data_seeds True # recreate tables
:return:
"""
import json_utils
import pkg_resources
if recreate:
self.setup()
path = pkg_resources.resource_filename(__name__,
'knowledge_base_data.json')
dataset = json_utils.read_json_file(path)
print(dataset.keys())
# fill dataset
self.initial_table(restaurant, dataset['restaurant'])
self.initial_table(hotel, dataset['hotel'])
def test_json_scraper(self):
'''
Testing restful api scraper to see if it returns expected value
from the FightPoverty api
'''
# Will store response in temp file
temp_test_file = './temp_test_file.json'
# Sample request
request = 'http://api.fightpoverty.online/api/county'
# Scrape response into temp_test_file
scraper.restful_api_scraper(request, temp_test_file)
# Read response from temp_test_file
response = json_utils.read_json_file(temp_test_file)
# Make sure it returns value as expected
self.assertEqual(response['num_results'], 291)
# Clean up and delete temp test file
os.remove(temp_test_file)
def setup_lang_feeds(self):
"""
$ python -m sagas.graph.graph_manager setup_lang_feeds
:return:
"""
print('.. setup schema')
client = helper.reset('''
name: string @index(exact, term) .
nsubj: string @index(exact, term) .
dobj: string @index(exact) .
pobj: string @index(exact) .
attr: string @index(exact) .
sents: string @index(fulltext) @lang .
lemmas: string @index(term) .
verbs: string @index(term) .
zh_SBV: string @index(term) @lang .
zh_VOB: string @index(term) @lang .
''')
files = list_with_suffix('data/graph', '_feed.json')
for file in tqdm(files):
feed_json = json_utils.read_json_file(file)
_ = helper.set_json(client, feed_json)
def run_ab3dmot(
classname: str,
pose_dir: str,
dets_dump_dir: str,
tracks_dump_dir: str,
min_conf: float = 0.3,
match_algorithm: str = 'h', #hungarian
match_threshold: float = 4,
match_distance: float = 'iou',
p: np.ndarray = np.eye(10),
thr_estimate: float = 0.8,
thr_prune: float = 0.1,
ps: float = 0.9) -> None:
"""
#path to argoverse tracking dataset test set, we will add our predicted labels into per_sweep_annotations_amodal/
#inside this folder
Filtering occurs in the city frame, not the egovehicle frame.
Args:
- classname: string, either 'VEHICLE' or 'PEDESTRIAN'
- pose_dir: string
- dets_dump_dir: string
- tracks_dump_dir: string
- max_age: integer
- min_hits: integer
Returns:
- None
"""
dl = SimpleArgoverseTrackingDataLoader(data_dir=pose_dir,
labels_dir=dets_dump_dir)
am = ArgoverseMap()
for log_id in tqdm(dl.sdb.get_valid_logs()):
print(log_id)
city_name = dl.get_city_name(log_id)
labels_folder = dets_dump_dir + "/" + log_id + "/per_sweep_annotations_amodal/"
lis = os.listdir(labels_folder)
lidar_timestamps = [
int(file.split(".")[0].split("_")[-1]) for file in lis
]
lidar_timestamps.sort()
previous_frame_bbox = []
ab3dmot = AB3DMOT(thr_estimate=thr_estimate,
thr_prune=thr_prune,
ps=ps)
print(labels_folder)
tracked_labels_copy = []
for j, current_lidar_timestamp in enumerate(lidar_timestamps):
dets = dl.get_labels_at_lidar_timestamp(log_id,
current_lidar_timestamp)
dets_copy = dets
transforms = []
city_SE3_egovehicle = dl.get_city_to_egovehicle_se3(
log_id, current_lidar_timestamp)
egovehicle_SE3_city = city_SE3_egovehicle.inverse()
transformed_labels = []
conf = []
for l_idx, l in enumerate(dets):
if l['label_class'] != classname:
# will revisit in other tracking pass
continue
if l["score"] < min_conf:
# print('Skipping det with confidence ', l["score"])
continue
det_obj = json_label_dict_to_obj_record(l)
det_corners_egovehicle_fr = det_obj.as_3d_bbox()
transforms += [city_SE3_egovehicle]
if city_SE3_egovehicle is None:
print('Was None')
# convert detection from egovehicle frame to city frame
det_corners_city_fr = city_SE3_egovehicle.transform_point_cloud(
det_corners_egovehicle_fr)
ego_xyz = np.mean(det_corners_city_fr, axis=0)
# Check the driveable/roi area
#da = am.remove_non_driveable_area_points(np.array([ego_xyz]), city_name=city_name)
# if len(da) == 0 and l['label_class'] == 'VEHICLE':
# continue
# roi = am.remove_non_roi_points(np.array([ego_xyz]), city_name=city_name)
# if len(roi) == 0:
# continue
yaw = yaw_from_bbox_corners(det_corners_city_fr)
transformed_labels += [[
ego_xyz[0], ego_xyz[1], ego_xyz[2], yaw, l["length"],
l["width"], l["height"]
]]
conf += [l["score"]]
if len(transformed_labels) > 0:
transformed_labels = np.array(transformed_labels)
else:
transformed_labels = np.empty((0, 7))
dets_all = {
"dets": transformed_labels,
"info": np.zeros(transformed_labels.shape),
"conf": conf
}
# perform measurement update in the city frame.
dets_with_object_id = ab3dmot.update(dets_all, match_distance,
match_threshold,
match_algorithm, p)
tracked_labels = []
for det in dets_with_object_id:
# move city frame tracks back to ego-vehicle frame
xyz_city = np.array(
[det[0].item(), det[1].item(),
det[2].item()]).reshape(1, 3)
city_yaw_object = det[3]
city_se2_object = SE2(rotation=rotmat2d(city_yaw_object),
translation=xyz_city.squeeze()[:2])
city_se2_egovehicle, city_yaw_ego = get_B_SE2_A(
city_SE3_egovehicle)
ego_se2_city = city_se2_egovehicle.inverse()
egovehicle_se2_object = ego_se2_city.right_multiply_with_se2(
city_se2_object)
# recreate all 8 points
# transform them
# compute yaw from 8 points once more
egovehicle_SE3_city = city_SE3_egovehicle.inverse()
xyz_ego = egovehicle_SE3_city.transform_point_cloud(
xyz_city).squeeze()
# update for new yaw
# transform all 8 points at once, then compute yaw on the fly
ego_yaw_obj = se2_to_yaw(egovehicle_se2_object)
qx, qy, qz, qw = yaw_to_quaternion3d(ego_yaw_obj)
tracked_labels.append({
"center": {
"x": xyz_ego[0],
"y": xyz_ego[1],
"z": xyz_ego[2]
},
"rotation": {
"x": qx,
"y": qy,
"z": qz,
"w": qw
},
"length":
det[4],
"width":
det[5],
"height":
det[6],
"track_label_uuid":
uuid_gen.get_uuid(det[7]),
"timestamp":
current_lidar_timestamp,
"label_class":
classname
})
tracked_labels_copy = copy.deepcopy(tracked_labels)
label_dir = os.path.join(tracks_dump_dir, log_id,
"per_sweep_annotations_amodal")
check_mkdir(label_dir)
json_fname = f"tracked_object_labels_{current_lidar_timestamp}.json"
json_fpath = os.path.join(label_dir, json_fname)
if Path(json_fpath).exists():
# accumulate tracks of another class together
prev_tracked_labels = read_json_file(json_fpath)
tracked_labels.extend(prev_tracked_labels)
save_json_dict(json_fpath, tracked_labels)
sys.path.insert(0, "../python_utils")
# pylint: disable=import-error, wrong-import-position, wrong-import-order
from mysql_utils import connect_to_mysql_db
from json_utils import read_json_file
# Connect to SQL db
(CNX, CUR) = connect_to_mysql_db(None)
CUR.execute("SELECT * FROM county")
COUNTIES = CUR.fetchall()
ALL_COUNTY_ARRAY = read_json_file('./counties.json')
with open('county_codes.tsv', 'w') as tsvfile:
WRITER = csv.writer(tsvfile, delimiter='\t')
WRITER.writerow(["id", "rate"])
for county in COUNTIES:
(county_id, county_name, county_state,
poverty_per, poverty_pop, fp_mult) = county
# Search through COUNTY_ARRAY for county in db
fips_code = ''
for fips_county in ALL_COUNTY_ARRAY:
(fips_name, fips_per, fips_pop, not_used,
fips_state, fips_county_code) = fips_county
def load_json_data(self, schema_head, file_name, reset=True):
import json_utils
if reset:
self.setup_schema_head(schema_head)
ps = json_utils.read_json_file(file_name)
self.gm.add_object(ps)
if label not in self.classes:
continue
elif word not in self.emo_dict.keys():
self.emo_dict[word] = {}
self.emo_dict[word][label] = score
# add missing scores
for word in self.emo_dict.keys():
for c in self.classes:
if c not in self.emo_dict[word].keys():
self.emo_dict[word][c] = 0
"""
save the dictionary as a json file
"""
def save_as_json(self):
dump_path = "../models/lexicon.json"
with open(dump_path, "w") as jsonfile:
json.dump(self.emo_dict, jsonfile)
d = EmotionDictionaryBuilder("../models/emotion_lex.txt")
d.build_dict()
d.save_as_json()
# test json
json_data = read_json_file("../models/lexicon.json")
print(json_data)
