def merge_json(): final_dict = {} dept_dict = read_json("year_dept.json") category_dict = read_json("year_category.json") for each in dept_dict: each["CREATION YEAR"] = str(each["CREATION YEAR"]) each["ZIP CODE"] = str(int(each["ZIP CODE"])) if each["CREATION YEAR"] not in final_dict.keys(): final_dict[each["CREATION YEAR"]] = {} if each["ZIP CODE"] not in final_dict[each["CREATION YEAR"]].keys(): final_dict[each["CREATION YEAR"]][each["ZIP CODE"]] = {} if "DEPARTMENT" not in final_dict[each["CREATION YEAR"]][each["ZIP CODE"]].keys(): final_dict[each["CREATION YEAR"]][each["ZIP CODE"]]["DEPARTMENT"] = [] dept = {"name": each["DEPARTMENT"], "count": each["COUNT"]} final_dict[each["CREATION YEAR"]][each["ZIP CODE"]]["DEPARTMENT"].append(dept) for each in category_dict: each["CREATION YEAR"] = str(each["CREATION YEAR"]) each["ZIP CODE"] = str(int(each["ZIP CODE"])) if each["CREATION YEAR"] not in final_dict.keys(): final_dict[each["CREATION YEAR"]] = {} if each["ZIP CODE"] not in final_dict[each["CREATION YEAR"]].keys(): final_dict[each["CREATION YEAR"]][each["ZIP CODE"]] = {} if "CATEGORY" not in final_dict[each["CREATION YEAR"]][each["ZIP CODE"]].keys(): final_dict[each["CREATION YEAR"]][each["ZIP CODE"]]["CATEGORY"] = [] dept = {"name": each["CATEGORY"], "count": each["COUNT"]} final_dict[each["CREATION YEAR"]][each["ZIP CODE"]]["CATEGORY"].append(dept) write_json(final_dict, "final.json")
def analyze_predictions(): dept_category = read_json(dept_category_file) prob_category = read_json(prob_category_file) merged_df = pd.read_csv(merged_file) merged_df['DEPT_LABEL'] = merged_df['DEPT_LABEL'].apply( lambda x: dept_category[str(x)]) merged_df['PROB_LABEL'] = merged_df['PROB_LABEL'].apply( lambda x: prob_category[str(x)]) master_df = pd.read_csv(master_dataframe_file) new_master = master_df.merge(merged_df, on='CASE ID') new_master.drop(columns=['Unnamed: 0', 'Unnamed: 0.1'], inplace=True) new_master.to_csv(master_dataframe_file_pred, header=True, index=False)
def remove_item(self, *args): data = read_json(self.filename) or [] try: data.remove(list(args)) except: log.err("Can't remove " + str(list(args)) + " from " + str(data)) with open(self.filename, 'w+') as file: file.write(json.dumps(data))
def testFeatures(): data = read_json(test_file) data = [Recipe(id=recipe["id"], cuisine=None, ingredients=recipe["ingredients"]) for recipe in data] for r in data: c = getPerceptronCuisineFeatures(r) print "{},{}".format(r.id, c)
def remove(self, index): data = read_json(self.filename) or [] if index >= len(data): log.err("Trying to remove something out of index? size: {}, index: {}".format(len(data), index)) return del data[index] with open(self.filename, 'w+') as file: file.write(json.dumps(data))
def remove(self, index): data = read_json(self.filename) or [] if index >= len(data): log.err( "Trying to remove something out of index? size: {}, index: {}". format(len(data), index)) return del data[index] with open(self.filename, 'w+') as file: file.write(json.dumps(data))
def read(self, callback): if os.path.isfile(self.filename): BACKUP = self.filename + ".backup" # On Linux, os.rename will silently overwrite, but that is not the # case on Windows, so just remove the old backup if it exists: if os.path.isfile(BACKUP): os.remove(BACKUP) os.rename(self.filename, BACKUP) if os.path.isfile(self.filename): log.err("os.rename did not move file, removing it manually") os.remove(self.filename) data = read_json(BACKUP) or [] for line in data: try: callback(*line) except Exception,e: log.err("Error while reading: " + str(e))
def read(self, callback): if os.path.isfile(self.filename): BACKUP = self.filename + ".backup" # On Linux, os.rename will silently overwrite, but that is not the # case on Windows, so just remove the old backup if it exists: if os.path.isfile(BACKUP): os.remove(BACKUP) os.rename(self.filename, BACKUP) if os.path.isfile(self.filename): log.err("os.rename did not move file, removing it manually") os.remove(self.filename) data = read_json(BACKUP) or [] for line in data: try: callback(*line) except Exception, e: log.err("Error while reading: " + str(e))
def get_gantt_tasks_by_week(self, week_number: int): tasks = read_json('data/tasks.json') queried_tasks = [] for task in tasks: start = DATE_HELPER.get_datetime_from_task(task["START DATE"]) end = DATE_HELPER.get_datetime_from_task(task["END DATE"]) start_week = DATE_HELPER.get_week(start, output_week_number=True)[1] end_week = DATE_HELPER.get_week(end, output_week_number=True)[1] in_curr_week = start_week == week_number or end_week == week_number if in_curr_week: queried_tasks.append(task) after_curr_week = DATE_HELPER.get_week( start, output_week_number=True)[1] > week_number if after_curr_week: break return queried_tasks
def process_json(): time_series_dict = read_json("time_series.json") final_dict = {} for each in time_series_dict: each["CREATION YEAR"] = str(each["CREATION YEAR"]) each["ZIP CODE"] = str(int(each["ZIP CODE"])) each["CREATION MONTH"] = str(each["CREATION MONTH"]) if each["CREATION YEAR"] not in final_dict.keys(): final_dict[each["CREATION YEAR"]] = {} if each["ZIP CODE"] not in final_dict[each["CREATION YEAR"]].keys(): final_dict[each["CREATION YEAR"]][each["ZIP CODE"]] = {} if "DEPARTMENT" not in final_dict[each["CREATION YEAR"]][each["ZIP CODE"]].keys(): final_dict[each["CREATION YEAR"]][each["ZIP CODE"]]["DEPARTMENT"] = [] dept = { "CREATION MONTH": each["CREATION MONTH"], "NAME": each["DEPARTMENT"], "COUNT": each["DAYS TO CLOSE"] } final_dict[each["CREATION YEAR"]][each["ZIP CODE"]]["DEPARTMENT"].append(dept) write_json(final_dict, "time_series_final.json")
import os from utils.json_utils import read_json import pandas as pd root_dir_path = "/home/charan/Documents/workspaces/python_workspaces/Data/ADL_Project" prob_category = os.path.join(root_dir_path, "problem/prob_category.json") dept_category = os.path.join(root_dir_path, "dept/dept_category.json") prob_category = read_json(prob_category) dept_category = read_json(dept_category) def apply_dept(input_string): return list(dept_category.values()).index(input_string) re_map = { "Parks & Rec": "Parks and Rec", "Parks & Recreation": "Parks and Rec", "Information Technology": "DROP", "NCS": "DROP", "Housing Community Dev": "DROP", "IT": "DROP", "Municipal Court": "DROP" } def apply_process_dept(input_string): if input_string in list(re_map.keys()): return re_map[input_string] else: return input_string
import pandas as pd import os from utils.json_utils import write_json, read_json root_dir = "/home/charan/Documents/workspaces/python_workspaces/Data/ADL_Project" prob_file = os.path.join(root_dir, "problem/prob_classification_df.csv") dept_file = os.path.join(root_dir, "dept/dept_classification_df.csv") final_df = os.path.join(root_dir, "final_data/311_Cases_master_with_desc.csv") dept_json = os.path.join(root_dir, "dept/dept_category.json") prob_json = os.path.join(root_dir, "problem/prob_category.json") map_prob_json = os.path.join( root_dir, "problem/new_category.json") ## Used for mapping to parent category. map_prob_json = read_json(map_prob_json) master_dataframe_file_pred = os.path.join(root_dir, "final_data/Data_with_no_desc.csv") timeseries_data = os.path.join(root_dir, 'time-series/time_series.csv') final_data = pd.read_csv(final_df) prob_category = {} dept_category = {} def apply_dept_label(input_text): return list(dept_category.values()).index(input_text) def apply_prob_label(input_string): return list(prob_category.values()).index(input_string)
def get_settings(settings_file=DEFAULT_SETTINGS_FILE): if not path.isfile(settings_file): create_default_settings(settings_file) settings = read_json(settings_file) return settings
def load_from_file(self, filename: str = ""): try: data = read_json(filename) drones_dict = data.get("drones", None) assert not drones_dict is None, f"Drones not specified in file {filename}. Failed loading from file." env_dict = data.get("environment", None) assert not env_dict is None, f"Environment not specified in file {filename}. Failed loading from file." assert "trajectory" in dict(next(iter(drones_dict.values()))).keys( ), f"Trajectories not specified in file {filename}. Failed loading from file." traj_dict = { id: drone_dict["trajectory"] for id, drone_dict in drones_dict.items() } if "path" in env_dict.keys(): env_dict = read_json(env_dict["path"]) self.log["environment"] = Environment([ Obstacle(obj["shape"], np.array(obj["points"])) for obj in env_dict["obstacles"] ]) drones = [] for id, drone in drones_dict.items(): d_class = getattr( importlib.import_module(drone["info"]["module"]), drone["info"]["cls"]) d_args = dict( filter( lambda elem: elem[0] != "module" and elem[0] != "cls" and elem[0] != "id", drone["info"].items())) sensors = [] for s in drone["info"]["sensors"]: s_class = getattr(importlib.import_module(s["module"]), s["cls"]) s_args = { k: np.array(v) if isinstance(v, list) else v for k, v in dict( filter( lambda elem: elem[0] != "module" and elem[0] != "cls", s.items())).items() } sensors.append(s_class(**s_args)) drones.append( d_class(id=id, state=np.array( next(iter( drone["trajectory"].values()))["state"]), **d_args)) self.__log_info(drones) for id, drone_traj in traj_dict.items(): for time, state_meas_dict in drone_traj.items(): drone = list(filter(lambda d: d.id == id, drones))[0] self.log_time_step( drone.generate_time_entry( np.array(state_meas_dict["state"]), [ float(m["measurement"]) for m in state_meas_dict["measurements"] ]), float(time)) except FileNotFoundError: print( f"Could not find file named {filename}. Loading log file failed. Exiting." ) exit(0)
def save(self, *args): data = read_json(self.filename) or [] data.append(args) with open(self.filename, 'w+') as file: file.write(json.dumps(data))
ws = {} for ing in recipe.ingredients: for w in ing.split(): ws[w] = ws.get(w, 0.0) + 1.0 for k, v in ws.iteritems(): features.append(["w_{}".format(k), v]) return features def testFeatures(): data = read_json(test_file) data = [Recipe(id=recipe["id"], cuisine=None, ingredients=recipe["ingredients"]) for recipe in data] for r in data: c = getPerceptronCuisineFeatures(r) print "{},{}".format(r.id, c) def participate(data): ingest(data) initWeights() for x in range(0, iterations): random.shuffle(data) perceptronFeatures(data) print "@@@ Iteration: " + str(x + 1) + " @@@" testFeatures() if __name__ == '__main__': raw_data = read_json(training_file) data = [Recipe(id=recipe["id"], cuisine=recipe["cuisine"], ingredients=recipe["ingredients"]) for recipe in raw_data] participate(data)