def evaluate(rec_output_type, user_list, rec_size, user_2_true, user_2_pred): user_2_tp_and_fp_and_prec_and_ndcg = {} for user in user_list: if user == "user37" and rec_output_type == "Single": "stop" tp = 0.0 fp = rec_size prec = 0.0 ndcg = 0.0 if user in user_2_true and user in user_2_pred: y_true = list(unique_everseen(user_2_true[user])) y_pred = list(unique_everseen(user_2_pred[user])) tp = count_tp(rec_output_type, y_true, y_pred) #fp = len(y_pred) - tp # NOTE Assumed if pred_size< rec_size(k), the missing part is fp fp = max(len(y_pred), rec_size) - tp prec = calculatePrecision(tp, fp) dcg = calculateDcg(rec_size, y_true, y_pred) idcg = calculateIdealDcg(rec_size, y_true) ndcg = dcg / idcg user_2_tp_and_fp_and_prec_and_ndcg[user] = (tp, fp, prec, ndcg) # // write precs of eah user # printer.printPrecisionEvalResult(folder + outputPrecfileName, controlUserIdList, userPrecMap, userTPMap, userFPMap); # # // calculate precision for overall results # Double precisionTotal = EvaluateCheckin2011DB.precision(truePosTotal, falsePosTotal); # String str = "Overall -- tpTotal: " + truePosTotal # + " fpTotal: " + falsePosTotal # + " prec: " + precisionTotal; # printer.printString(folder + outputPrecfileName, str); return user_2_tp_and_fp_and_prec_and_ndcg
def combine(cls, obj, perms, limit=None): ret = [] limit = limit and int(limit) if isinstance(obj, iterables): head = List it = obj else: head = obj.__class__ it = obj.args if limit: for per in perms: if not is_integer(per): raise FunctionException("Subsets::int") ret += [ head(*x) for x in islice(combinations(it, int(per)), limit - len(ret)) ] if len(ret) >= limit: break else: for per in perms: if not is_integer(per): raise FunctionException("Subsets::int") ret += [head(*x) for x in combinations(it, int(per))] return List(*unique_everseen(ret))
def step(self, actions): self.players = [ limit_to_size(move(player, action, self.prob_right_direction), self.grid_size) for player, action in zip(self.players, actions) ] states = self.__get_state() is_at_goal = [ player == goal for player, goal in zip(self.players, self.goals) ] reward_is_at_goal = [-1 if x == False else 20 for x in is_at_goal] # detect a crash dup = list(unique_everseen(duplicates(self.players))) # if a player's position appears twice, add -20 to the current reward reward_is_crash = [ -19 if (player in dup) else 0 for player in self.players ] rewards = [a + b for a, b in zip(reward_is_at_goal, reward_is_crash)] done = True in is_at_goal if self.render_board: print("{}".format(self.render())) return (states, rewards, done)
def test_traverse(): """To test the traverse implementation we call gc.collect() while instances of all the C objects are still valid.""" acc = iteration_utilities.accumulate([]) app = iteration_utilities.applyfunc(lambda x: x, 1) cha = iteration_utilities.chained(int, float) cla = iteration_utilities.clamp([], 0, 1) com = iteration_utilities.complement(int) con = iteration_utilities.constant(1) dee = iteration_utilities.deepflatten([]) dup = iteration_utilities.duplicates([]) fli = iteration_utilities.flip(int) gro = iteration_utilities.grouper([], 2) ine = iteration_utilities.intersperse([], 1) iik = iteration_utilities.ItemIdxKey(10, 2) ite = iteration_utilities.iter_except(int, TypeError) mer = iteration_utilities.merge([]) nth = iteration_utilities.nth(1) pac = iteration_utilities.packed(int) par = iteration_utilities.partial(int, 10) rep = iteration_utilities.replicate([], 3) rou = iteration_utilities.roundrobin([]) see = iteration_utilities.Seen() sid = iteration_utilities.sideeffects([], lambda x: x) spl = iteration_utilities.split([], lambda x: True) sta = iteration_utilities.starfilter(lambda x: True, []) suc = iteration_utilities.successive([]) tab = iteration_utilities.tabulate(int) une = iteration_utilities.unique_everseen([]) unj = iteration_utilities.unique_justseen([]) gc.collect()
def removeDuplicates(data): from iteration_utilities import unique_everseen permissionkeys = ["readLabels", "readUsers", "writeLabels", "writeUsers"] for key in permissionkeys: if data.get(key): data[key] = list(unique_everseen(data.get(key))) return data
def valid(string): if 8 in string or 11 in string or 14 in string: return False if not any(starmap(straight, successive(string, 3))): return False pairs = unique_everseen(starfilter(eq, successive(string))) return 1 < count_items(pairs)
def find_files(data_path, directory, search_for_mm): files_array = [] for x in directory: if search_for_mm: name = file_name_result.search(x) else: name = file_name.search(x) if name is not None and name not in files_array: files_array.append(name[0]) files_array = list(unique_everseen(files_array)) files_array.sort() return files_array
def _check_no_duplicate_object_ids(self): """ Checks if all object IDs are unique. Raises: ValueError: if duplicate IDs were found """ duplicate_obj_ids = list(unique_everseen(duplicates(self.object_ids))) if duplicate_obj_ids: raise PVConfigurationException( f'User configuration entries contain duplicate' f'object IDs: {duplicate_obj_ids}')
def test_uniqueeverseen_unhashable1(): assert list(unique_everseen([{ T(1): T(1) }, { T(2): T(2) }, { T(1): T(1) }])) == [{ T(1): T(1) }, { T(2): T(2) }]
def limite_z(self, position_coeur, position_sein): List_z_coeur = [] List_z_Sein = [] List_tot_z = [] for i in position_coeur: List_z_coeur.append(i[2]) for j in position_sein: List_z_Sein.append(j[2]) List_z_coeur = list(unique_everseen(duplicates(List_z_coeur))) List_z_Sein = list(unique_everseen(duplicates(List_z_Sein))) slice_barycenter = int( np.ceil((np.min(List_z_Sein) + np.max(List_z_Sein)) / 2)) List_tot_z = List_z_Sein + List_z_coeur mini = np.min(List_tot_z) maxi = np.max(List_tot_z) # print("Mini = ", mini, "Maxi = ",maxi) return mini, maxi, slice_barycenter
def pairs(actual, k): all_act_pairs = [] if len(actual) >= k: for pair in itertools.combinations(actual[:k], 2): all_act_pairs.append(pair) else: if len(actual) == 1: all_act_pairs.append((actual[0], actual[0])) else: for pair in itertools.combinations(actual, 2): all_act_pairs.append(pair) all_act_pairs = list(unique_everseen(all_act_pairs, key=set)) return float(len(all_act_pairs))
def test_uniqueeverseen_getter1(): t = unique_everseen([T(1), T([0, 0]), T(3)]) assert not t.seen assert t.key is None assert next(t) == T(1) assert t.seen == Seen({T(1)}) assert t.key is None assert next(t) == T([0, 0]) assert T(1) in t.seen assert T([0, 0]) in t.seen assert t.key is None assert next(t) == T(3) assert t.seen == Seen({T(1), T(3)}, [T([0, 0])]) assert t.key is None
def pairsu(actual, predicted, k): actual = list(dict.fromkeys(actual)) predicted = list(dict.fromkeys(predicted)) all_act_pairs = [] for pair in itertools.combinations(actual[:k], 2): all_act_pairs.append(pair) all_pred_pairs = [] for pair in itertools.combinations(predicted[:k], 2): all_pred_pairs.append(pair) # The order is not important: all_act_pairs = list(unique_everseen(all_act_pairs, key=set)) all_pred_pairs = list(unique_everseen(all_pred_pairs, key=set)) result = 0 for el in all_act_pairs: if el in all_pred_pairs: result = result + 1 return float(result)
def test_uniqueeverseen_getter2(): t = unique_everseen([T(1), T([0, 0]), T(3)], iteration_utilities.return_identity) assert not t.seen assert t.key is iteration_utilities.return_identity assert next(t) == T(1) assert t.seen == Seen({T(1)}) assert t.key is iteration_utilities.return_identity assert next(t) == T([0, 0]) assert T(1) in t.seen assert T([0, 0]) in t.seen assert t.key is iteration_utilities.return_identity assert next(t) == T(3) assert t.seen == Seen({T(1), T(3)}, [T([0, 0])]) assert t.key is iteration_utilities.return_identity
def permute(cls, obj, perms): ret = [] if isinstance(obj, iterables): head = List it = obj else: head = obj.__class__ it = obj.args for per in perms: if not is_integer(per): raise FunctionException("Permutations::int") ret += [head(*x) for x in permutations(it, int(per))] return List(*unique_everseen(ret))
def find_repeated_info(df): global merged global info_with_multiple_userid_ls # find duplicate entries with many userid df_colunm1_ls = df[df.columns[1]].to_list() info_with_multiple_userid_ls = list( unique_everseen(duplicates(df_colunm1_ls))) # construct new df, to cut down on the number of rows info_with_multiple_userid = pd.DataFrame(info_with_multiple_userid_ls, columns=["info"]) info_with_multiple_userid.head() merged = pd.merge(info_with_multiple_userid, df, how="left", left_on="info", right_on=df.columns[1]).drop(df.columns[1], axis=1)
def printOverallNetworkStatistic(): #Global Variables: global tbs global mfb #Compute the total similarity average of all social troll bot networks used in the 2016 elections: tsa = sum(tbs) / (len(tbs)) #Obtain the list of most frequent commonly used social troll bots used in the 2016 elections: mfb = list(unique_everseen(duplicates(mfb))) #Print Statements: print( "The average % of similarty of all politically social troll bot networks used in the 2016 elections = " + str(tsa) + "%") print( "The list of the most commonly used social troll bots used in the 2016 elections: " + str(mfb)) print( "The no. of most commonly used social troll bots used in the 2016 elections: " + str(len(mfb))) return True
def query(self, vector, radius=1, top_k=5): res_indices = [] ## Need to improve index calculations indices = vector.dot(self.base_vector.T).reshape(self.num_tables, -1) > 0 if radius == 0: res_indices = indices.dot(2**np.arange( self.n_vectors)) + np.arange( self.num_tables) * 2**self.n_vectors elif radius == 1: clone_indices = indices.repeat(axis=0, repeats=self.n_vectors) rel_indices = (np.arange(self.num_tables) * 2**self.n_vectors).repeat(axis=0, repeats=self.n_vectors) translate = np.tile(np.eye(self.n_vectors), (self.num_tables, 1)) res_indices = (np.abs(clone_indices - translate).dot(2**np.arange( self.n_vectors)) + rel_indices).astype(int) res_indices = np.concatenate([ res_indices, indices.dot(2**np.arange(self.n_vectors)) + np.arange(self.num_tables) * 2**self.n_vectors ]) start = time.time() lst = self.hash_table[res_indices].tolist() self.lookup_index_times.append(time.time() - start) start = time.time() res = list(unique_everseen(duplicates(flatten(lst)))) sim_scores = vector.dot(self.vectors[res].T) max_sim_indices = sim_scores.argsort()[-top_k:][::-1] max_sim_scores = sim_scores[max_sim_indices] return [(self.names[res[i]], score) for i, score in zip(max_sim_indices, max_sim_scores)]
def test_uniqueeverseen_empty1(): assert list(unique_everseen([])) == []
def test_uniqueeverseen_failure_setstate3(): _hf.iterator_setstate_empty_fail(unique_everseen(toT([1, 1])))
def test_uniqueeverseen_key2(): assert list(unique_everseen([T(1), T(1), T(-1)], abs)) == [T(1)]
def test_uniqueeverseen_failure1(): with pytest.raises(_hf.FailIter.EXC_TYP, match=_hf.FailIter.EXC_MSG): unique_everseen(_hf.FailIter())
def test_uniqueeverseen_pickle1(protocol): uqe = unique_everseen([T(1), T(2), T(1), T(2)]) assert next(uqe) == T(1) x = pickle.dumps(uqe, protocol=protocol) assert list(pickle.loads(x)) == [T(2)]
def test_uniqueeverseen_unhashable2(): assert list(unique_everseen([[T(1)], [T(2)], [T(1)]])) == [[T(1)], [T(2)]]
def test_uniqueeverseen_normal1(): assert list(unique_everseen([T(1), T(2), T(1)])) == [T(1), T(2)]
def test_uniqueeverseen_failure2(): with pytest.raises(TypeError): list(unique_everseen([T(1), T(2), T(3), T('a')], abs))
def finaljson(request): try: engine = create_engine('postgresql://*****:*****@localhost:5432/postgres') # l3.to_sql('legacytable3', engine, if_exists='append') l1 = pd.read_sql_query('select * from "legacytable1"', con=engine) l2 = pd.read_sql_query('select * from "legacytable2"', con=engine) l3 = pd.read_sql_query('select * from "legacytable3"', con=engine) merg = pd.merge(l1, l2, on="Emp#", how="outer") merg1 = pd.merge(merg, l3, on="Emp#", how="outer") merg1["Full Name"] = merg1["First Name"] + " " + merg1["Last Name"] def digit_tracker(x): d = str(x) k = len(d) spd = "0" * (11 - k) + d return spd merg1['uniqueid'] = merg1['Emp#'].apply(digit_tracker) final = merg1[["uniqueid", "Full Name", "Skills", "Technology", "Tax ID", "City", "Country Code", "Phone", "Email"]] final # final.to_sql('new_table', engine, if_exists='append') # final.to_csv("new_table.csv") print("your new table is created and stored.") mergedata1 = [ {"id": "03", "name": "ankit"}, {"id": "03", "address": "yyyy"}, {"id": "02", "phone": "3434343"}, {"id": "03", "phone": "23344"}, {"id": "01", "address": "rrrr"}, ] tablex = final.to_json(orient='records') # code for reshape json and remove redundancy plusdata=mergedata1 print(plusdata) plusdata.sort(key=lambda item: item.get("id")) print(plusdata) for i in range(len(plusdata)-1): if (plusdata[i]["id"] == plusdata[i + 1]["id"]): plusdata[i].update(plusdata[i + 1]) for i in range(len(plusdata)-1): if (plusdata[i]["id"] == plusdata[i + 1]["id"]): plusdata[i].update(plusdata[i + 1]) plusdata[i + 1].update(plusdata[i]) fitdata=list(unique_everseen(plusdata, key=lambda item: frozenset(item.items()))) print(fitdata) return render(request, 'dmac/product.html', {'tablex': tablex, 'fitdata': fitdata}) except: mergedata1 = [ {"id": "03", "name": "ankit"}, {"id": "03", "address": "yyyy"}, {"id": "02", "phone": "3434343"}, {"id": "03", "phone": "23344"}, {"id": "01", "address": "rrrr"}, ] # tablex = final.to_json(orient='records') # code for reshape json and remove redundancy plusdata = mergedata1 print(plusdata) plusdata.sort(key=lambda item: item.get("id")) print(plusdata) for i in range(len(plusdata) - 1): if (plusdata[i]["id"] == plusdata[i + 1]["id"]): plusdata[i].update(plusdata[i + 1]) for i in range(len(plusdata) - 1): if (plusdata[i]["id"] == plusdata[i + 1]["id"]): plusdata[i].update(plusdata[i + 1]) plusdata[i + 1].update(plusdata[i]) fitdata = list(unique_everseen(plusdata, key=lambda item: frozenset(item.items()))) print(fitdata) return render(request, 'dmac/product.html', {'fitdata': fitdata})
def thankyou(request): engine = create_engine('postgresql://*****:*****@localhost:5432/postgres') # engine = create_engine('postgresql://*****:*****@localhost:5432/postgres') df_aws2 = pd.read_sql_query("select relname from pg_class where relkind='r' and relname !~ '^(pg_|sql_)';", con=engine) tablename_list = list(df_aws2["relname"]) # # l3.to_sql('legacytable3', engine, if_exists='append') l1 = pd.read_sql_query('select * from "saptable1" ', con=engine) mkt = l1.iloc[0:, 1:] data=mkt # response = requests.get('http://127.0.0.1:8000/rest/') # tables = pd.read_html(r.text) # print('Tables found:', len(tables)) # print(tables) # print(r.text) # Parse the HTML pages # from bs4 import BeautifulSoup # soup = BeautifulSoup(response.text, 'html.parser') # det = soup.find_all("table") # tables = pd.read_html(response.text) # print(tables) # You can extract the page title as string as well # Find all the h3 elements # print(f"{tutorialpoints_page.find_all('h2')}") # tags = tutorialpoints_page.find( # lambda elm: elm.name == "h2" or elm.name == "h3" or elm.name == "h4" or elm.name == "h5" or elm.name == "h6") # for sibling in tags.find_next_siblings(): # if sibling.name == "table": # my_table = sibling # df = pd.read_html(str(my_table)) # print(df) if request.method == "POST": tabl = request.POST.get('jsonData') rules = request.POST.getlist('rules') print(rules) sit = json.loads(tabl) frame = pd.DataFrame(sit) frame = frame.iloc[0:, 1:] plusdata1 = frame.to_json(orient='records') plusdata2 = json.loads(plusdata1) # git = {'lispr':sit} # print(git) def del_none(d): """ Delete keys with the value ``None`` in a dictionary, recursively. This alters the input so you may wish to ``copy`` the dict first. """ # For Python 3, write `list(d.items())`; `d.items()` won’t work # For Python 2, write `d.items()`; `d.iteritems()` won’t work for key, value in list(d.items()): if value == "": del d[key] elif isinstance(value, dict): del_none(value) return d # For convenience listdf = [] for f in plusdata2: listdf.append(del_none(f.copy())) print(del_none(f.copy())) # def del_none(f): # for key, value in list(f.items()): # if value is None: # del f[key] # elif isinstance(value, dict): # del_none(value) # lift.append(f) # # return d # # del_none(f.copy()) print(listdf) # print(frame) # print(frame.info) # print(plusdata) # tablu = JSONParser().parse('jsonData') # desc = request.POST.get('desc') # table_name = json.loads(table_name.body) # print(type(plusdata)) plusdata = listdf plusdata.sort(key=lambda item: item.get("NAME")) # print(plusdata) for i in range(len(plusdata) - 1): if (plusdata[i]["NAME"] == plusdata[i + 1]["NAME"]): plusdata[i].update(plusdata[i + 1]) for i in range(len(plusdata) - 1): if (plusdata[i]["NAME"] == plusdata[i + 1]["NAME"]): plusdata[i].update(plusdata[i + 1]) plusdata[i + 1].update(plusdata[i]) fitdata = list(unique_everseen(plusdata, key=lambda item: frozenset(item.items()))) # print(fitdata) l1 = pd.DataFrame(fitdata) print(l1) try: if rules[0]=="name_spliter": def string_split(x): fd = x.split(" ") return fd[0] l1["first_name"] = l1["NAME"].apply(string_split) def string_last(x): try: fd = x.split(" ") return fd[1] except: pass l1["last_name"] = l1["NAME"].apply(string_last) table1 = l1.to_html() elif rules[0]=="string resizer": def digit_tracker(x): d = str(x) k = len(d) if k <= 8: spd = "0" * (8 - k) + d return spd else: spd = d[0:8] return spd l1['PHONE_new'] = l1['PHONE'].apply(digit_tracker) table1 = l1.to_html() elif rules[0]=="first+last": l1["merge_two_column"] = l1["NAME"]+" "+l1["WERKS"] table1= l1.to_html() else: table1 = l1.to_html() except: l1.to_sql('mckinsolteam', engine, if_exists='append') table1 = l1.to_html() # print(rules[0]) return HttpResponse(table1)
def test_uniqueeverseen_unhashable3(): assert list( unique_everseen([[T(1), T(1)], [T(1), T(2)], [T(1), T(3)]], operator.itemgetter(0))) == [[T(1), T(1)]]
def test_uniqueeverseen_normal2(): # key=None is identical to no key assert list(unique_everseen([T(1), T(2), T(1)], None)) == [T(1), T(2)]