def get_all_charities_from_jsons(): ''' Uses json utils to read json files and place results in an array ''' # Grab charities from FoodBanks.json charities = read_json_file("./FoodBanks.json") # Grab charities from HomelessServices.json charities += read_json_file("./HomelessServices.json") return charities
def __init__(self, lang: Text): from sagas.conf import resource_path import json_utils from os import path file = resource_path(f'analspa_{lang}.json') self.root = json_utils.read_json_file(file) if path.exists( file) else {}
def load_rss_seed(json_file='./data/rss/westore_new.json'): import json_utils rss_doc = json_utils.read_json_file(json_file) # rss_doc['title'] product_set = [] for entry in rss_doc['entry']: # print(entry["title@zh"], entry['published']) data = {} data['productName'] = entry["title@zh"] data['createdDate'] = now_jdbc() data['lastModifiedDate'] = now_jdbc() data['fromDate'] = to_jdbc(entry['published']) data['isVirtual'] = 'N' data['productTypeId'] = "FINISHED_GOOD" data['productId'] = extract_id(entry['id']) data['productPricePurposeId'] = "PURCHASE" data['productPriceTypeId'] = "DEFAULT_PRICE" data['productStoreGroupId'] = "Test_group" data['priceDetailText'] = entry['summary'] attrs = get_product_attrs(entry) if "价格:" in attrs: data['price'] = float(attrs['价格:'].strip().replace('元', '')) data['currencyUomId'] = 'CNY' if "image" in attrs: data['largeImageUrl'] = attrs['image'] product_set.append(data) # print(json.dumps(product_set, indent=2, ensure_ascii=False)) return product_set
def practice(self, lang_tr='fr', use_latest=False): """ $ python -m sagas.nlu.nlu_tools practice fr :param lang: :param lang_tr: :param dataf: :return: """ from sagas.nlu.tts_utils import say_lang import json_utils lang = 'en' dataf=corpus_resources[lang_tr] rows=[] if use_latest: rows = json_utils.read_json_file('./out/latest_%s.json' % lang_tr) else: rows=load_corpus(dataf) json_utils.write_json_to_file('./out/latest_%s.json' % lang_tr, rows.tolist()) for r in rows: sents = str(r[0]) tr_lang = str(r[1].strip()) say_lang(sents, lang) print('♥', sents) say_lang(tr_lang, lang_tr) print('♡', tr_lang)
def word_info(self, word): """ $ python -m sagas.ru.ru_procs word_info мальчик :param word: :return: """ import json_utils ipa_xs = self.epi.xsampa_list(word) ipa = self.epi.transliterate(word) rev_map = json_utils.read_json_file(self.target_file_rev) words_map = json_utils.read_json_file(self.target_file) key = rev_map[word] print(word, ipa, ''.join(ipa_xs), '☌', key) print('\t', words_map[key])
def word_info_df(self, word): import sagas import json_utils rev_map = json_utils.read_json_file(self.target_file_rev) words_map = json_utils.read_json_file(self.target_file) tuples = [] ipa_xs = self.epi.xsampa_list(word) ipa = self.epi.transliterate(word) key = rev_map[word] tuples.append((word, ipa, ''.join(ipa_xs), key)) for w in words_map[key]: ipa_xs = self.epi.xsampa_list(w) ipa = self.epi.transliterate(w) tuples.append((w, ipa, ''.join(ipa_xs), key)) return sagas.to_df(tuples, ['word', 'ipa', 'xsampa_list', 'key'])
def update_by_overrides(self) -> None: overrides = json_utils.read_json_file(self.overrides_file) for k, v in overrides.items(): if '.' in k: parts = k.split('.') self.conf[parts[0]][parts[1]] = v else: self.conf[k] = v
def genres_loader(): genres_root = read_json_file(root_dir + "data/movie_genres_f.json") genres_map = {} for genre in genres_root['genres']: genres_map[genre['id']] = genre['name'] for genre in genres_root['genres']: store_movie_genres(genre)
def view_json_data(self, file_name): data=json_utils.read_json_file(file_name) index=1 for item in data["rasa_nlu_data"]["common_examples"]: print("{}. {} ({})".format(index, item["text"], item["intent"])) children=item['entities'] if len(children)>0: for c in children: print("\t{}-{}: {}({})".format(c["start"], c["end"], c["value"], c["entity"])) index=index+1
def movies_loader(file): data_root=read_json_file(file) print(data_root["page"], "/", data_root["total_pages"]) movies=data_root["results"] for movie in movies: result = create_or_store("SaMovie", movie) set_movie_genres(movie["id"], movie['genre_ids']) print(result.get("title"))
def get_state_name_from_dict(state_dict_json_file, state): ''' Gets the state dict from the provided json file and returns the value corresponding to the provided state. ''' state_dict = read_json_file(state_dict_json_file) state_name = '' if state in state_dict: state_name = state_dict[state] return state_name
def lookup_nor(self, word): """ $ python -m sagas.ko.kwn_procs lookup_nor '미생물' :param word: :return: """ import json_utils if not self.idx_nor: self.idx_nor = json_utils.read_json_file( f"{self.index_dir}/ko_nor.json") return self.idx_nor[word] if word in self.idx_nor else []
def lookup_tra(self, word): """ $ python -m sagas.ko.kwn_procs lookup_tra mi-saeng-mur :param word: :return: """ import json_utils if not self.idx_tra: self.idx_tra = json_utils.read_json_file( f"{self.index_dir}/ko_tra.json") return self.idx_tra[word] if word in self.idx_tra else []
def similar(self, feat_name, sents): """ $ python -m sagas.nlu.expanders similar samples 'Charge périodique' :param feat_name: :param sents: :return: """ arranger = json_utils.read_json_file("./data/feats/%s.json" % feat_name) doc_vecs = np.load("./data/feats/%s.npy" % feat_name) # load bm = BertManager() # 'Charge périodique' bm.get_similar(arranger, doc_vecs, sents)
def build_input_pairs(additions): import clipboard voc_dicts = f'{cf.conf_dir}/langs/voc/ru-voc-dicts.json' voc_words = json_utils.read_json_file(voc_dicts) print('total voc words', len(voc_words)) rs = [] for w in voc_words: for en_w in w['en']: ru_w = ' '.join(w['ru']).replace(',', '') rs.append('%s %s' % (norm_key(en_w), ru_w)) print('input pairs contains voc words %d, and ips words %d'%(len(rs), len(additions))) clipboard.copy('\n'.join(rs+additions))
def __init__(self): import glob import json_utils from sagas.conf import resource_files, resource_path self.intents = [] files = [resource_path(f) for f in resource_files('ruleset_*.json')] # for f in glob.glob(f'{cf.conf_dir}/stack/conf/ruleset_*.json'): for f in files: rules = json_utils.read_json_file(f) # for rule in rules: # self.intents.append({rule['intent']: {'triggers': rule['action']}}) self.intents.extend(rules)
def load_resources(self): """ $ python -m sagas.graph.rss_hub load_resources :return: """ from io_utils import list_with_suffix schema_map = self.get_schema_map() client = helper.reset(schema_map) files = list_with_suffix('data/rss', '.json') for file in tqdm(files): feed_json = json_utils.read_json_file(file) # feed_json = self.get_feed_json(file) _ = set_json(client, feed_json)
def check_json(self, input_file): data=json_utils.read_json_file(input_file) dataset=NluData() for item in data["rasa_nlu_data"]["common_examples"]: # print("{}. {} ({})".format(index, item["text"], item["intent"])) fact=FactData(item["text"], item["intent"]) children=item['entities'] if len(children)>0: for c in children: # print("\t{}-{}: {}({})".format(c["start"], c["end"], c["value"], c["entity"])) entity=EntityData(c["start"], c["end"], c["value"], c["entity"], c["value"]) fact.entities.append(entity) dataset.examples.append(fact) print(json.dumps(dataset.json, indent=2, sort_keys=False, ensure_ascii=False))
def __init__(self): from sagas.conf.runtime import runtime from cachetools import LRUCache conf_file = f"{runtime_dir()}/sagas_conf.json" overrides_file = f"{runtime_dir()}/sagas_overrides.json" self.conf = json_utils.read_json_file(conf_file) self.overrides_file = overrides_file if runtime.is_docker(): self.update_by_overrides() # self.cache = LRUCache(maxsize=1024) self.loaded_classes = {}
def build_voc(): voc_file=f'{cf.conf_dir}/langs/voc/ru-voc.json' voc_dicts=f'{cf.conf_dir}/langs/voc/ru-voc-dicts.json' words=json_utils.read_json_file(voc_file) all_dicts=build_dicts() print('filter by voc-words ...') skips=[] rs=filter_set(all_dicts, words, 'ru', skips) json_utils.write_json_to_file(voc_dicts, rs) # print('done.') print('absents %d, see these words in file ru-voc-absents.json'%len(skips)) json_utils.write_json_to_file(f'{cf.conf_dir}/langs/voc/ru-voc-absents.json', skips) return rs, skips
async def add(ctx, arg, *args): if ctx.message.author.top_role.name in config.get('allow_permission', []): if args and arg[0] == '!': shitpost_keys = read_json_file('shitpost.json').get( 'shitpost', {}).keys() if arg not in shitpost_keys: add_shitpost_record(arg, ' '.join(args)) await ctx.send(f'Added command: {arg}') else: await ctx.send('Command already exist!') else: await ctx.send( 'Command does not start with ! or either does not have content.' ) else: await ctx.send('Incorrect permission!')
def check_modified(self): """ $ python -m saai.nlu_mod_procs check_modified :return: """ import os if not os.path.exists(self.timestamps_file): return mods before = json_utils.read_json_file(self.timestamps_file) current = self.get_timestamps() modified = [] for item in current: if current[item] - before[item] > 1: modified.append(item) return modified
def test_json_utils2(self): ''' Testing write_json_file and read_json_file by writing empty dict to file ''' test_dict = {} temp_test_file = './temp_json_test_file.json' # Write test_dict to temp_test_file json_utils.write_json_file(temp_test_file, test_dict) # Read temp_json_test_file.json to get dict dict_in_temp_test_file = json_utils.read_json_file(temp_test_file) # Make sure json reader returns original test_dict from file self.assertEqual(dict_in_temp_test_file, test_dict) # Clean up and delete temp test file os.remove(temp_test_file)
def fill_data_seeds(self, recreate=False): """ $ python -m sagas.modules.life.data_source fill_data_seeds $ python -m sagas.modules.life.data_source fill_data_seeds True # recreate tables :return: """ import json_utils import pkg_resources if recreate: self.setup() path = pkg_resources.resource_filename(__name__, 'knowledge_base_data.json') dataset = json_utils.read_json_file(path) print(dataset.keys()) # fill dataset self.initial_table(restaurant, dataset['restaurant']) self.initial_table(hotel, dataset['hotel'])
def test_json_scraper(self): ''' Testing restful api scraper to see if it returns expected value from the FightPoverty api ''' # Will store response in temp file temp_test_file = './temp_test_file.json' # Sample request request = 'http://api.fightpoverty.online/api/county' # Scrape response into temp_test_file scraper.restful_api_scraper(request, temp_test_file) # Read response from temp_test_file response = json_utils.read_json_file(temp_test_file) # Make sure it returns value as expected self.assertEqual(response['num_results'], 291) # Clean up and delete temp test file os.remove(temp_test_file)
def setup_lang_feeds(self): """ $ python -m sagas.graph.graph_manager setup_lang_feeds :return: """ print('.. setup schema') client = helper.reset(''' name: string @index(exact, term) . nsubj: string @index(exact, term) . dobj: string @index(exact) . pobj: string @index(exact) . attr: string @index(exact) . sents: string @index(fulltext) @lang . lemmas: string @index(term) . verbs: string @index(term) . zh_SBV: string @index(term) @lang . zh_VOB: string @index(term) @lang . ''') files = list_with_suffix('data/graph', '_feed.json') for file in tqdm(files): feed_json = json_utils.read_json_file(file) _ = helper.set_json(client, feed_json)
def run_ab3dmot( classname: str, pose_dir: str, dets_dump_dir: str, tracks_dump_dir: str, min_conf: float = 0.3, match_algorithm: str = 'h', #hungarian match_threshold: float = 4, match_distance: float = 'iou', p: np.ndarray = np.eye(10), thr_estimate: float = 0.8, thr_prune: float = 0.1, ps: float = 0.9) -> None: """ #path to argoverse tracking dataset test set, we will add our predicted labels into per_sweep_annotations_amodal/ #inside this folder Filtering occurs in the city frame, not the egovehicle frame. Args: - classname: string, either 'VEHICLE' or 'PEDESTRIAN' - pose_dir: string - dets_dump_dir: string - tracks_dump_dir: string - max_age: integer - min_hits: integer Returns: - None """ dl = SimpleArgoverseTrackingDataLoader(data_dir=pose_dir, labels_dir=dets_dump_dir) am = ArgoverseMap() for log_id in tqdm(dl.sdb.get_valid_logs()): print(log_id) city_name = dl.get_city_name(log_id) labels_folder = dets_dump_dir + "/" + log_id + "/per_sweep_annotations_amodal/" lis = os.listdir(labels_folder) lidar_timestamps = [ int(file.split(".")[0].split("_")[-1]) for file in lis ] lidar_timestamps.sort() previous_frame_bbox = [] ab3dmot = AB3DMOT(thr_estimate=thr_estimate, thr_prune=thr_prune, ps=ps) print(labels_folder) tracked_labels_copy = [] for j, current_lidar_timestamp in enumerate(lidar_timestamps): dets = dl.get_labels_at_lidar_timestamp(log_id, current_lidar_timestamp) dets_copy = dets transforms = [] city_SE3_egovehicle = dl.get_city_to_egovehicle_se3( log_id, current_lidar_timestamp) egovehicle_SE3_city = city_SE3_egovehicle.inverse() transformed_labels = [] conf = [] for l_idx, l in enumerate(dets): if l['label_class'] != classname: # will revisit in other tracking pass continue if l["score"] < min_conf: # print('Skipping det with confidence ', l["score"]) continue det_obj = json_label_dict_to_obj_record(l) det_corners_egovehicle_fr = det_obj.as_3d_bbox() transforms += [city_SE3_egovehicle] if city_SE3_egovehicle is None: print('Was None') # convert detection from egovehicle frame to city frame det_corners_city_fr = city_SE3_egovehicle.transform_point_cloud( det_corners_egovehicle_fr) ego_xyz = np.mean(det_corners_city_fr, axis=0) # Check the driveable/roi area #da = am.remove_non_driveable_area_points(np.array([ego_xyz]), city_name=city_name) # if len(da) == 0 and l['label_class'] == 'VEHICLE': # continue # roi = am.remove_non_roi_points(np.array([ego_xyz]), city_name=city_name) # if len(roi) == 0: # continue yaw = yaw_from_bbox_corners(det_corners_city_fr) transformed_labels += [[ ego_xyz[0], ego_xyz[1], ego_xyz[2], yaw, l["length"], l["width"], l["height"] ]] conf += [l["score"]] if len(transformed_labels) > 0: transformed_labels = np.array(transformed_labels) else: transformed_labels = np.empty((0, 7)) dets_all = { "dets": transformed_labels, "info": np.zeros(transformed_labels.shape), "conf": conf } # perform measurement update in the city frame. dets_with_object_id = ab3dmot.update(dets_all, match_distance, match_threshold, match_algorithm, p) tracked_labels = [] for det in dets_with_object_id: # move city frame tracks back to ego-vehicle frame xyz_city = np.array( [det[0].item(), det[1].item(), det[2].item()]).reshape(1, 3) city_yaw_object = det[3] city_se2_object = SE2(rotation=rotmat2d(city_yaw_object), translation=xyz_city.squeeze()[:2]) city_se2_egovehicle, city_yaw_ego = get_B_SE2_A( city_SE3_egovehicle) ego_se2_city = city_se2_egovehicle.inverse() egovehicle_se2_object = ego_se2_city.right_multiply_with_se2( city_se2_object) # recreate all 8 points # transform them # compute yaw from 8 points once more egovehicle_SE3_city = city_SE3_egovehicle.inverse() xyz_ego = egovehicle_SE3_city.transform_point_cloud( xyz_city).squeeze() # update for new yaw # transform all 8 points at once, then compute yaw on the fly ego_yaw_obj = se2_to_yaw(egovehicle_se2_object) qx, qy, qz, qw = yaw_to_quaternion3d(ego_yaw_obj) tracked_labels.append({ "center": { "x": xyz_ego[0], "y": xyz_ego[1], "z": xyz_ego[2] }, "rotation": { "x": qx, "y": qy, "z": qz, "w": qw }, "length": det[4], "width": det[5], "height": det[6], "track_label_uuid": uuid_gen.get_uuid(det[7]), "timestamp": current_lidar_timestamp, "label_class": classname }) tracked_labels_copy = copy.deepcopy(tracked_labels) label_dir = os.path.join(tracks_dump_dir, log_id, "per_sweep_annotations_amodal") check_mkdir(label_dir) json_fname = f"tracked_object_labels_{current_lidar_timestamp}.json" json_fpath = os.path.join(label_dir, json_fname) if Path(json_fpath).exists(): # accumulate tracks of another class together prev_tracked_labels = read_json_file(json_fpath) tracked_labels.extend(prev_tracked_labels) save_json_dict(json_fpath, tracked_labels)
sys.path.insert(0, "../python_utils") # pylint: disable=import-error, wrong-import-position, wrong-import-order from mysql_utils import connect_to_mysql_db from json_utils import read_json_file # Connect to SQL db (CNX, CUR) = connect_to_mysql_db(None) CUR.execute("SELECT * FROM county") COUNTIES = CUR.fetchall() ALL_COUNTY_ARRAY = read_json_file('./counties.json') with open('county_codes.tsv', 'w') as tsvfile: WRITER = csv.writer(tsvfile, delimiter='\t') WRITER.writerow(["id", "rate"]) for county in COUNTIES: (county_id, county_name, county_state, poverty_per, poverty_pop, fp_mult) = county # Search through COUNTY_ARRAY for county in db fips_code = '' for fips_county in ALL_COUNTY_ARRAY: (fips_name, fips_per, fips_pop, not_used, fips_state, fips_county_code) = fips_county
def load_json_data(self, schema_head, file_name, reset=True): import json_utils if reset: self.setup_schema_head(schema_head) ps = json_utils.read_json_file(file_name) self.gm.add_object(ps)
if label not in self.classes: continue elif word not in self.emo_dict.keys(): self.emo_dict[word] = {} self.emo_dict[word][label] = score # add missing scores for word in self.emo_dict.keys(): for c in self.classes: if c not in self.emo_dict[word].keys(): self.emo_dict[word][c] = 0 """ save the dictionary as a json file """ def save_as_json(self): dump_path = "../models/lexicon.json" with open(dump_path, "w") as jsonfile: json.dump(self.emo_dict, jsonfile) d = EmotionDictionaryBuilder("../models/emotion_lex.txt") d.build_dict() d.save_as_json() # test json json_data = read_json_file("../models/lexicon.json") print(json_data)